From 0defb73291b15ac03ea0f72c9ec656870bdf0bec Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 9 Feb 2024 16:54:52 +0100 Subject: [PATCH 01/99] xenopsd: fix config to match install location (#5444) Signed-off-by: Yann Dirson --- ocaml/xenopsd/scripts/make-custom-xenopsd.conf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ocaml/xenopsd/scripts/make-custom-xenopsd.conf b/ocaml/xenopsd/scripts/make-custom-xenopsd.conf index 09fe4559268..b49610f0e9a 100755 --- a/ocaml/xenopsd/scripts/make-custom-xenopsd.conf +++ b/ocaml/xenopsd/scripts/make-custom-xenopsd.conf @@ -37,12 +37,12 @@ supported-vbd-backend-kinds=vbd,qdisk,9pfs xenguest=${LIBEXECDIR}/xenguest network-conf=${ETCDIR}/xapi/network.conf -vif-script=${LIBEXECDIR}/vif -vif-xl-script=${LIBEXECDIR}/vif -vbd-script=${LIBEXECDIR}/block -vbd-xl-script=${LIBEXECDIR}/block -qemu-vif-script=${LIBEXECDIR}/qemu-vif-script -setup-vif-rules=${LIBEXECDIR}/setup-vif-rules +vif-script=${XENOPSD_LIBEXECDIR}/vif +vif-xl-script=${XENOPSD_LIBEXECDIR}/vif +vbd-script=${XENOPSD_LIBEXECDIR}/block +vbd-xl-script=${XENOPSD_LIBEXECDIR}/block +qemu-vif-script=${XENOPSD_LIBEXECDIR}/qemu-vif-script +setup-vif-rules=${XENOPSD_LIBEXECDIR}/setup-vif-rules sockets-group=$group qemu-wrapper=${QEMU_WRAPPER_DIR}/qemu-wrapper swtpm-wrapper=${QEMU_WRAPPER_DIR}/qemu-wrapper From 221e86e6084c318dd96949f094c2d7d7f68884a1 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Mon, 12 Feb 2024 10:51:10 +0000 Subject: [PATCH 02/99] CP-47754: Do not report errors attempting to read PCI vendor:product Different network card (like virtual ones or USB ones) do not have a vendor:product causing networkd daemon to report a lot of errors like Error in read one line of file: /sys/class/net/eth0/device/device, exception Unix.Unix_error(Unix.ENOENT, "open", "/sys/class/net/eth0/device/device") Raised by primitive operation at Xapi_stdext_unix__Unixext.with_file in file "lib/xapi-stdext-unix/unixext.ml", line 68, characters 11-40 Called from Xapi_stdext_unix__Unixext.buffer_of_file in file "lib/xapi-stdext-unix/unixext.ml" (inlined), line 155, characters 31-83 Called from Xapi_stdext_unix__Unixext.string_of_file in file "lib/xapi-stdext-unix/unixext.ml", line 157, characters 47-73 Called from Network_utils.Sysfs.read_one_line in file "ocaml/networkd/lib/network_utils.ml", line 156, characters 6-33 Do not report these missing file as errors. Signed-off-by: Frediano Ziglio --- ocaml/networkd/lib/network_utils.ml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ocaml/networkd/lib/network_utils.ml b/ocaml/networkd/lib/network_utils.ml index 27d23486a51..1c92735f259 100644 --- a/ocaml/networkd/lib/network_utils.ml +++ b/ocaml/networkd/lib/network_utils.ml @@ -225,15 +225,15 @@ module Sysfs = struct let get_pci_ids name = let read_id_from path = - try - let l = read_one_line path in - (* trim 0x *) - String.sub l 2 (String.length l - 2) - with _ -> "" + let l = path |> Unixext.string_of_file |> String.trim in + (* trim 0x *) + String.sub l 2 (String.length l - 2) in - ( read_id_from (getpath name "device/vendor") - , read_id_from (getpath name "device/device") - ) + try + ( read_id_from (getpath name "device/vendor") + , read_id_from (getpath name "device/device") + ) + with _ -> ("", "") (** Returns the name of the driver for network device [dev] *) let get_driver_name dev = From 7222f808d39f6b95dae138dd38990a9e1b1b1bdf Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 13 Feb 2024 08:29:58 +0100 Subject: [PATCH 03/99] Add .codecov.yml: It configures the Codecov PR comment and checks - This is a first shot on configuring Codecov for the Xen-API repo. - It might need further tweaks, but it is a first step towards what I think we want Codecov to do. Steps taken: - Only add the Codecov comment to the PR when coverage changes - Remove not needed noise from the Codecov PR comment - Give examples on how to configure how much code must be covered for Codecov to pass the coverage check of a PR - Disable adding coverage annotations to the code in the GitHub Code Review for now: - The coverage can be visited using the Codecov link at all times. https://app.codecov.io/gh/xapi-project/xen-api/pulls - The annotations consume a lot of space in the PR code review, and can make it hard to review files that are not covered yet. - The annotations can be hidden in GitHub PR code review by pressing the "a" key or by deselecting the "Show comments" checkbox, but they are shown by default. - The Codecov Chrome and Firefox extension is a much nicer way to indicate coverage: Link: https://github.com/codecov/codecov-browser-extension - How to enable: You need to log in to Codecov "using Github". For Firefox, enable the needed permissions: https://github.com/codecov/codecov-browser-extension/issues/50 - Show the Codecov status without waiting for other status to pass Signed-off-by: Bernhard Kaindl --- .codecov.yml | 244 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000000..d49bbb420c5 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,244 @@ +# For more configuration details: +# https://docs.codecov.io/docs/codecov-yaml + +# After making edits, check if this file is valid by running: +# curl -X POST --data-binary @.codecov.yml https://codecov.io/validate + +# +# Coverage configuration +# ---------------------- +# +codecov: + # + # Show the Codecov status without waiting for other status to pass: + # + require_ci_to_pass: no + notify: + wait_for_ci: no + +github_checks: + # + # Disable adding coverage annotations to the code in the GitHub + # Code Review for now: + # + # - The annotations consume a lot of space in the PR code review, + # and can make it hard to review files that are not covered yet. + # + # - The coverage can be visited using the Codecov link at all times. + # https://app.codecov.io/gh/xapi-project/xen-api/pulls + # + # - The annotations can be hidden in GitHub PR code review by + # pressing the "a" key or by deselecting the "Show comments" + # checkbox but they are shown by default. + # + # - The Codecov Chrome and Firefox extension is a much nicer + # way to indicate coverage: + # + # Link: https://github.com/codecov/codecov-browser-extension + # + # - How to enable: You need to log in to Codecov using Github. + # For Firefox, enable the needed permissions: + # https://github.com/codecov/codecov-browser-extension/issues/50 + # + # Reference: + # http://docs.codecov.com/docs/common-recipe-list#disable-github-check-run-annotations + # + annotations: false + + +# +# Pull request comments: +# ---------------------- +# This feature adds the code coverage summary as a comment on each PR. +# See https://docs.codecov.io/docs/pull-request-comments +# This same information is available from the Codecov checks in the PR's +# "Checks" tab in GitHub even when this feature is disabled. +# +comment: + # + # Legend: + # "diff" is the Coverage Diff of the pull request. + # "files" are the files impacted by the pull request + # "flags" are the coverage status of the pull request + # + # For an even shorter layout, this may be used: + # layout: "condensed_header, diff, files, flags" + # + layout: "header, diff, files, flags" + + # + # Only add the Codecov comment to the PR when coverage changes + # + require_changes: true + # + # The overall project coverage is secondary to the individual coverage + # and it is always shown in the repository at: + # - https://app.codecov.io/gh/xapi-project/xen-api + # + hide_project_coverage: true + + +# +# Coverage limits and display details: +# ------------------------------------ +# +coverage: + + # + # Number of precision digits when showing coverage percentage e.g. 82.1%: + # + precision: 1 + + # + # Commit status checks and display: + # --------------------------------- + # https://docs.codecov.io/docs/commit-status + # + # target: Fail the PR if coverage is below that + # threshold: Allow reducing coverage by this amount + # + # - The values added are a very generous, friendly limit to not block most PRs + # + # - XAPI maintainers may tighten these screws more to require better tests + # + status: # global coverage status and limits + + # + # Patch limits + # ------------ + # These checks look at only the diff of the PR as basis for them. + # + patch: + scripts: + + # + # The scripts limit applies to: + # ----------------------------- + # + # - scripts/** + # - excluding: **/test_*.py + # + paths: ["scripts/**", "!**/test_*.py"] + + # + # For scripts/** (excluding tests): + # + # For scripts, coverage should not be reduced compared to its base: + # + target: auto + + # + # Exception: the threshold value given is allowed + # + # Allows for not covering 20% if the changed lines of the PR: + # + threshold: 20% + + ocaml: + # + # The ocaml limit applies to: + # ----------------------------- + # + # - ocaml/** + # - excluding: **/test_*.py + # + paths: ["ocaml/**", "!**/test_*.py"] + + # + # For scripts/** (excluding tests): + # + # For scripts, coverage should not be reduced compared to its base: + # + target: auto + + # + # Exception: the threshold value given is allowed + # + # Allows for not covering 20% if the changed lines of the PR: + # + threshold: 20% + + # Checks each Python version separately: + python-3.11: + flags: ["python3.11"] + python-2.7: + flags: ["python2.7"] + + # + # Project limits + # -------------- + # These checks are relative to all code, not the changes (not the diff of the PR) + # + project: + + # + # Python modules and scripts below scripts/ (excluding tests) + # + scripts: + target: 48% + threshold: 2% + paths: ["scripts/**", "!**/test_*.py"] + + # + # Python modules and scripts below ocaml/ + # + ocaml: + paths: ["ocaml/**", "!**/test_*.py"] + target: 51% + threshold: 3% + + # + # Test files + # + tests: + # Ensure that all tests are executed (tests themselves must be 100% covered) + target: 100% + paths: ["**/test_*.py"] + + +# +# Components: +# ----------- +# Components can be selected in the Codecov Web interface then looking at one PR: +# https://app.codecov.io/gh/xapi-project/xen-api/pulls +# +component_management: + + default_rules: # default rules that will be inherited by all components + statuses: + + - type: project + # `auto` will use the coverage from the base commit (pull request base + # or parent commit) coverage to compare against. + target: auto + threshold: 2% + + - type: patch + target: auto + threshold: 10% + + individual_components: + + - component_id: scripts # this is an identifier that should not be changed + name: scripts # this is a display name, and can be changed freely + # The list of paths that should be in- and excluded in this component: + paths: ["scripts/**", "!scripts/examples/**", "!**/test_*.py"] + + - component_id: scripts/examples + name: scripts/examples + paths: ["scripts/examples/**", "!scripts/**/test_*.py"] + + - component_id: ocaml + name: ocaml + paths: ["ocaml/**", "!**/test_*.py"] + + - component_id: ocaml/xapi-storage + name: ocaml/xapi-storage + paths: + - "ocaml/xapi-storage/**" + - "ocaml/xapi-storage-script/**" + - "!**/test_*.py" + + - component_id: test_cases + name: test_cases + paths: ["**/test_*.py"] From f60c526a2896829ffbbdf12ffcae5417de7bf87f Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 14 Feb 2024 11:10:27 +0000 Subject: [PATCH 04/99] rrd_updates: output JSON in the same structure as XML the data field is not part of meta. see the XML generator at ocaml/libs/xapi-rrd/lib/rrd_updates.ml line 124 The issue was introduced when serialization was changed to use yojson: https://github.com/xapi-project/xcp-rrd/commit/048b0e33b992e45ba8a465b24bbbfe10aa5d65c2 Signed-off-by: Pau Ruiz Safont --- ocaml/libs/xapi-rrd/lib/rrd_updates.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocaml/libs/xapi-rrd/lib/rrd_updates.ml b/ocaml/libs/xapi-rrd/lib/rrd_updates.ml index e1e3a98f88d..d9de5b045b5 100644 --- a/ocaml/libs/xapi-rrd/lib/rrd_updates.ml +++ b/ocaml/libs/xapi-rrd/lib/rrd_updates.ml @@ -194,9 +194,9 @@ let json_of_t t = ; ("rows", int (Array.length t.data)) ; ("columns", int (Array.length t.legend)) ; ("legend", array (map_to_list string t.legend)) - ; ("data", array (map_to_list data_record t.data)) ] ) + ; ("data", array (map_to_list data_record t.data)) ] in Yojson.to_string meta From dae6ea2d81463efeec9c0e5152d39a213e31e124 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 6 Feb 2024 09:23:38 +0000 Subject: [PATCH 05/99] CP-47431: Replace patched `Newtonsoft.Json.CH` with `Newtonsoft.Json` in C# SDK Also: - Rename `README.dist` and `README-NuGet.dist` to `README.md` and `README-NuGet.md` - Change content of aforementioned READMEs to use markdown - Remove mentions of patched Json.NET library - Update copyright notice year - Do not add `.txt` extension to `LICENSE` file Signed-off-by: Danilo Del Busso --- .../csharp/{README.dist => autogen/README.md} | 33 ++++++++----------- ocaml/sdk-gen/csharp/autogen/dune | 25 ++------------ .../src/README-NuGet.md} | 25 +++++--------- .../templates/XenServer.csproj.mustache | 24 +++----------- 4 files changed, 30 insertions(+), 77 deletions(-) rename ocaml/sdk-gen/csharp/{README.dist => autogen/README.md} (77%) rename ocaml/sdk-gen/csharp/{README-NuGet.dist => autogen/src/README-NuGet.md} (74%) diff --git a/ocaml/sdk-gen/csharp/README.dist b/ocaml/sdk-gen/csharp/autogen/README.md similarity index 77% rename from ocaml/sdk-gen/csharp/README.dist rename to ocaml/sdk-gen/csharp/autogen/README.md index e841c8f1d48..c0637502919 100644 --- a/ocaml/sdk-gen/csharp/README.dist +++ b/ocaml/sdk-gen/csharp/autogen/README.md @@ -1,7 +1,6 @@ -XenServer.NET -============= +# XenServer.NET -Copyright (c) 2007-2023 Cloud Software Group, Inc. All Rights Reserved. +Copyright (c) 2007-2024 Cloud Software Group, Inc. All Rights Reserved. XenServer.NET is a complete SDK for XenServer, exposing the XenServer API as .NET classes. It is written in C#. @@ -15,8 +14,7 @@ XenServer.NET is free software. You can redistribute and modify it under the terms of the BSD 2-Clause license. See LICENSE.txt for details. -Reference ---------- +## Reference For XenServer documentation see https://docs.xenserver.com @@ -30,40 +28,35 @@ A number of examples to help you get started with the SDK is available at https://github.com/xenserver/xenserver-samples For community content, blogs, and downloads, visit -https://www.xenserver.com/blogs and https://www.citrix.com/community/ +https://www.xenserver.com/blogs and https://www.citrix.com/community To network with other developers using XenServer visit -https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver/ +https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver -Prerequisites -------------- +## Prerequisites This library requires .NET Standard 2.0. -Dependencies ------------- +## Dependencies XenServer.NET is dependent upon the following libraries: -- Newtonsoft JSON.NET by James Newton-King (see https://www.newtonsoft.com/). - JSON.NET is licensed under the MIT license; see LICENSE.Newtonsoft.Json.txt - for details. A patched version of the library (Newtonsoft.Json.CH.dll) is - shipped with XenServer.NET. Its source code and license file can be found in - https://github.com/xenserver/dotnet-packages. +- Newtonsoft JSON.NET by James Newton-King (see https://www.newtonsoft.com). + JSON.NET is licensed under the MIT license. -Downloads ---------- +## Downloads + This archive contains the following folders that are relevant to .NET developers: - XenServer.NET: contains the ready compiled binaries in the form of a NuGet package. - XenServer.NET\src: contains the source code shipped as a Visual Studio project. -Getting Started ---------------- +## Getting Started + Extract the contents of this archive. diff --git a/ocaml/sdk-gen/csharp/autogen/dune b/ocaml/sdk-gen/csharp/autogen/dune index e65414c7297..d5e542936ad 100644 --- a/ocaml/sdk-gen/csharp/autogen/dune +++ b/ocaml/sdk-gen/csharp/autogen/dune @@ -1,34 +1,15 @@ (rule - (targets LICENSE.txt) + (targets LICENSE) (deps ../../LICENSE ) (action (copy %{deps} %{targets})) ) -(rule - (targets README.txt) - (deps - ../README.dist - ) - (action (copy %{deps} %{targets})) -) - -(rule - (targets README-NuGet.md) - (deps - ../README-NuGet.dist - ) - (action (copy %{deps} %{targets})) -) - (alias (name generate) (deps - LICENSE.txt - README.txt - README-NuGet.md + LICENSE (source_tree .) ) -) - +) \ No newline at end of file diff --git a/ocaml/sdk-gen/csharp/README-NuGet.dist b/ocaml/sdk-gen/csharp/autogen/src/README-NuGet.md similarity index 74% rename from ocaml/sdk-gen/csharp/README-NuGet.dist rename to ocaml/sdk-gen/csharp/autogen/src/README-NuGet.md index 6994c933cab..47ccad8037b 100644 --- a/ocaml/sdk-gen/csharp/README-NuGet.dist +++ b/ocaml/sdk-gen/csharp/autogen/src/README-NuGet.md @@ -1,7 +1,6 @@ -XenServer.NET -============= +# XenServer.NET -Copyright (c) 2007-2023 Cloud Software Group, Inc. All Rights Reserved. +Copyright (c) 2007-2024 Cloud Software Group, Inc. All Rights Reserved. XenServer.NET is a complete SDK for XenServer, exposing the XenServer API as .NET classes. It is written in C#. @@ -15,8 +14,7 @@ XenServer.NET is free software. You can redistribute and modify it under the terms of the BSD 2-Clause license. See LICENSE.txt for details. -Reference ---------- +## Reference For XenServer documentation see https://docs.xenserver.com @@ -30,25 +28,20 @@ A number of examples to help you get started with the SDK is available at https://github.com/xenserver/xenserver-samples For community content, blogs, and downloads, visit -https://www.xenserver.com/blogs and https://www.citrix.com/community/ +https://www.xenserver.com/blogs and https://www.citrix.com/community To network with other developers using XenServer visit -https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver/ +https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver -Prerequisites -------------- +## Prerequisites This library requires .NET Standard 2.0. -Dependencies ------------- +## Dependencies XenServer.NET is dependent upon the following libraries: -- Newtonsoft JSON.NET by James Newton-King (see https://www.newtonsoft.com/). - JSON.NET is licensed under the MIT license; see LICENSE.Newtonsoft.Json.txt - for details. A patched version of the library (Newtonsoft.Json.CH.dll) is - shipped with XenServer.NET. Its source code and license file can be found in - https://github.com/xenserver/dotnet-packages. +- Newtonsoft JSON.NET by James Newton-King (see https://www.newtonsoft.com). + JSON.NET is licensed under the MIT license. \ No newline at end of file diff --git a/ocaml/sdk-gen/csharp/templates/XenServer.csproj.mustache b/ocaml/sdk-gen/csharp/templates/XenServer.csproj.mustache index 4006046d879..bf387509141 100644 --- a/ocaml/sdk-gen/csharp/templates/XenServer.csproj.mustache +++ b/ocaml/sdk-gen/csharp/templates/XenServer.csproj.mustache @@ -11,7 +11,7 @@ $(AssemblyName).NET $(AssemblyName).NET .NET wrapper for the XenServer API - Copyright (c) 2007-2023 Cloud Software Group, Inc. All Rights Reserved. + Copyright (c) 2007-2024 Cloud Software Group, Inc. All Rights Reserved. citrix hypervisor virtualization sdk jsonrpc .net c# xen xenserver BSD-2-Clause https://github.com/xapi-project/xen-api @@ -20,40 +20,26 @@ README-NuGet.md - - - - + true - - - False - .\lib\netstandard2.0\Newtonsoft.Json.CH.dll - - - - - False - .\lib\net45\Newtonsoft.Json.CH.dll - + + - True True + True FriendlyErrorNames.resx - Designer ResXFileCodeGenerator - XenAPI FriendlyErrorNames.Designer.cs From 9fa57f8756f3bc4431206b9cf63063a6fe3668dd Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 6 Feb 2024 09:23:38 +0000 Subject: [PATCH 06/99] Add `.gitignore` to C# SDK source Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/csharp/autogen/.gitignore | 423 ++++++++++++++++++++++++ 1 file changed, 423 insertions(+) create mode 100644 ocaml/sdk-gen/csharp/autogen/.gitignore diff --git a/ocaml/sdk-gen/csharp/autogen/.gitignore b/ocaml/sdk-gen/csharp/autogen/.gitignore new file mode 100644 index 00000000000..e60f54f7f19 --- /dev/null +++ b/ocaml/sdk-gen/csharp/autogen/.gitignore @@ -0,0 +1,423 @@ +# Created by https://www.toptal.com/developers/gitignore/api/dotnetcore,visualstudio,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=dotnetcore,visualstudio,visualstudiocode + +### DotnetCore ### +# .NET Core build folders +bin/ +obj/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +### VisualStudio ### +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +*.code-workspace + +# Local History for Visual Studio Code + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml + +### VisualStudio Patch ### +# Additional files built by Visual Studio + +# End of https://www.toptal.com/developers/gitignore/api/dotnetcore,visualstudio,visualstudiocode \ No newline at end of file From 462cb408e0295ff7517e7b1f5dc831a410280b06 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 6 Feb 2024 09:23:38 +0000 Subject: [PATCH 07/99] Use correct naming in `FriendlyErrorNames.resx` Hyphens are converted when using `ResGen.exe` but they shouldn't be used to keep a consistent style Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/csharp/FriendlyErrorNames.resx | 16 ++++++++-------- ocaml/sdk-gen/csharp/autogen/src/Failure.cs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ocaml/sdk-gen/csharp/FriendlyErrorNames.resx b/ocaml/sdk-gen/csharp/FriendlyErrorNames.resx index 133e457a3c4..737b0a20d65 100644 --- a/ocaml/sdk-gen/csharp/FriendlyErrorNames.resx +++ b/ocaml/sdk-gen/csharp/FriendlyErrorNames.resx @@ -114,7 +114,7 @@ Server cannot attach network (in the case of NIC bonding, this may be because attaching the network on this server would require other networks [that are currently active] to be taken down). - + Cannot attach network @@ -123,7 +123,7 @@ The VM could not start because the CD drive is empty. - + Disabled @@ -147,13 +147,13 @@ Not enough server memory is available to perform this operation - + Not enough free memory The restore could not be performed because the server is not in recovery mode - + Unreachable @@ -806,7 +806,7 @@ Authorized Roles: {1} The VM's Virtual Hardware Platform version is incompatible with this host. - + HVM not supported @@ -824,19 +824,19 @@ Authorized Roles: {1} You attempted to run a VM on a host which does not have a GPU available in the GPU group needed by the VM. - + GPU not available This VM needs a network that cannot be seen from that server - + Cannot see required network This VM needs storage that cannot be seen from that server - + Cannot see required storage diff --git a/ocaml/sdk-gen/csharp/autogen/src/Failure.cs b/ocaml/sdk-gen/csharp/autogen/src/Failure.cs index d165fa93947..b877dfa6de1 100644 --- a/ocaml/sdk-gen/csharp/autogen/src/Failure.cs +++ b/ocaml/sdk-gen/csharp/autogen/src/Failure.cs @@ -145,7 +145,7 @@ where trimmed.Length > 0 try { - shortError = errorDescriptions.GetString(ErrorDescription[0] + "-SHORT") ?? errorText; + shortError = errorDescriptions.GetString(ErrorDescription[0] + "_SHORT") ?? errorText; } catch (Exception) { From a1705eb4c48c894b04d810e6da57b13df7db2672 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 6 Feb 2024 09:23:38 +0000 Subject: [PATCH 08/99] Generate `FriendlyErrorNames.Designer.cs` with templates Enables building without the use of `ResGen.exe` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/csharp/dune | 1 + ocaml/sdk-gen/csharp/friendly_error_names.ml | 12 ++- .../FriendlyErrorNames.Designer.mustache | 75 +++++++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 ocaml/sdk-gen/csharp/templates/FriendlyErrorNames.Designer.mustache diff --git a/ocaml/sdk-gen/csharp/dune b/ocaml/sdk-gen/csharp/dune index 8a0ba5d6179..417dca4d4b1 100644 --- a/ocaml/sdk-gen/csharp/dune +++ b/ocaml/sdk-gen/csharp/dune @@ -22,6 +22,7 @@ mustache xapi-datamodel xmllight2 + str ) ) diff --git a/ocaml/sdk-gen/csharp/friendly_error_names.ml b/ocaml/sdk-gen/csharp/friendly_error_names.ml index 4fe041a298f..adad6f4feef 100644 --- a/ocaml/sdk-gen/csharp/friendly_error_names.ml +++ b/ocaml/sdk-gen/csharp/friendly_error_names.ml @@ -95,6 +95,7 @@ let parse_resx filename = let _ = let resx_file = "FriendlyErrorNames.resx" in + let designer_file = "FriendlyErrorNames.Designer.cs" in parse_resx resx_file ; parse_sr_xml sr_xml ; let errors = friendly_names_all Datamodel.errors in @@ -109,6 +110,12 @@ let _ = [ ("i18n_error_key", `String x) ; ("i18n_error_description", `String y) + ; ( "i8in_error_description_replace_newlines" + , `String + (Str.global_replace (Str.regexp_string "\n") + "\n /// " y + ) + ) ] ) errors @@ -116,4 +123,7 @@ let _ = ) ] in - render_file ("FriendlyErrorNames.mustache", resx_file) json templdir destdir + render_file ("FriendlyErrorNames.mustache", resx_file) json templdir destdir ; + render_file + ("FriendlyErrorNames.Designer.mustache", designer_file) + json templdir destdir diff --git a/ocaml/sdk-gen/csharp/templates/FriendlyErrorNames.Designer.mustache b/ocaml/sdk-gen/csharp/templates/FriendlyErrorNames.Designer.mustache new file mode 100644 index 00000000000..f73b3aa50d1 --- /dev/null +++ b/ocaml/sdk-gen/csharp/templates/FriendlyErrorNames.Designer.mustache @@ -0,0 +1,75 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace XenAPI { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + public class FriendlyErrorNames { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal FriendlyErrorNames() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("XenAPI.FriendlyErrorNames", typeof(FriendlyErrorNames).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + {{#i18n_errors}} + /// + /// Looks up a localized string similar to {{i8in_error_description_replace_newlines}}. + /// + public static string {{i18n_error_key}} { + get { + return ResourceManager.GetString("{{i18n_error_key}}", resourceCulture); + } + } + + {{/i18n_errors}} + } +} From 79a0ab5609786c7bc94719b83538b10e3a982b3d Mon Sep 17 00:00:00 2001 From: Benjamin Reis Date: Wed, 21 Feb 2024 15:27:31 +0100 Subject: [PATCH 09/99] Add 'threads_per_core' in 'Host.cpu_info' Discussed here: https://github.com/xapi-project/xen-api/issues/5462 Signed-off-by: Benjamin Reis --- ocaml/tests/common/test_common.ml | 2 ++ ocaml/xapi-idl/xen/xenops_interface.ml | 1 + ocaml/xapi/cpuid_helpers.ml | 2 ++ ocaml/xapi/cpuid_helpers.mli | 2 ++ ocaml/xapi/create_misc.ml | 6 ++++-- ocaml/xenopsd/lib/xenops_server_skeleton.ml | 1 + ocaml/xenopsd/xc/xenops_server_xen.ml | 2 ++ 7 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ocaml/tests/common/test_common.ml b/ocaml/tests/common/test_common.ml index 786a6eaca61..d1433868fdc 100644 --- a/ocaml/tests/common/test_common.ml +++ b/ocaml/tests/common/test_common.ml @@ -42,6 +42,7 @@ let default_cpu_info = * localhost to be counted *) ("cpu_count", "0") ; ("socket_count", "0") + ; ("threads_per_core", "0") ; ("vendor", "Abacus") ; ("speed", "") ; ("modelname", "") @@ -77,6 +78,7 @@ let make_localhost ~__context ?(features = Features.all_features) () = { cpu_count= 1 ; socket_count= 1 + ; threads_per_core= 1 ; vendor= "" ; speed= "" ; modelname= "" diff --git a/ocaml/xapi-idl/xen/xenops_interface.ml b/ocaml/xapi-idl/xen/xenops_interface.ml index ab240a53cb5..083c345f149 100644 --- a/ocaml/xapi-idl/xen/xenops_interface.ml +++ b/ocaml/xapi-idl/xen/xenops_interface.ml @@ -461,6 +461,7 @@ module Host = struct type cpu_info = { cpu_count: int ; socket_count: int + ; threads_per_core: int ; vendor: string ; speed: string ; modelname: string diff --git a/ocaml/xapi/cpuid_helpers.ml b/ocaml/xapi/cpuid_helpers.ml index db346fcef27..571a7f073b6 100644 --- a/ocaml/xapi/cpuid_helpers.ml +++ b/ocaml/xapi/cpuid_helpers.ml @@ -43,6 +43,8 @@ let cpu_count = Map_check.(field "cpu_count" int) let socket_count = Map_check.(field "socket_count" int) +let threads_per_core = Map_check.(field "threads_per_core" int) + let vendor = Map_check.(field "vendor" string) let get_flags_for_vm ~__context vm cpu_info = diff --git a/ocaml/xapi/cpuid_helpers.mli b/ocaml/xapi/cpuid_helpers.mli index ee1fbeb0d26..4d5f091d7f6 100644 --- a/ocaml/xapi/cpuid_helpers.mli +++ b/ocaml/xapi/cpuid_helpers.mli @@ -28,6 +28,8 @@ val cpu_count : int Map_check.field val socket_count : int Map_check.field +val threads_per_core : int Map_check.field + val features : [`vm] Xenops_interface.CPU_policy.t Map_check.field val features_pv : [`host] Xenops_interface.CPU_policy.t Map_check.field diff --git a/ocaml/xapi/create_misc.ml b/ocaml/xapi/create_misc.ml index a0d14ae94b9..7a2630ea57f 100644 --- a/ocaml/xapi/create_misc.ml +++ b/ocaml/xapi/create_misc.ml @@ -567,6 +567,7 @@ let create_host_cpu ~__context host_info = [ ("cpu_count", string_of_int cpu_info.cpu_count) ; ("socket_count", string_of_int cpu_info.socket_count) + ; ("threads_per_core", string_of_int cpu_info.threads_per_core) ; ("vendor", cpu_info.vendor) ; ("speed", cpu_info.speed) ; ("modelname", cpu_info.modelname) @@ -592,10 +593,11 @@ let create_host_cpu ~__context host_info = let old_cpu_info = Db.Host.get_cpu_info ~__context ~self:host in debug "create_host_cpuinfo: setting host cpuinfo: socket_count=%d, \ - cpu_count=%d, features_hvm=%s, features_pv=%s, features_hvm_host=%s, \ - features_pv_host=%s" + cpu_count=%d, threads_per_core=%d, features_hvm=%s, features_pv=%s, \ + features_hvm_host=%s, features_pv_host=%s" (Map_check.getf socket_count cpu) (Map_check.getf cpu_count cpu) + (Map_check.getf threads_per_core cpu) (Map_check.getf features_hvm cpu |> CPU_policy.to_string) (Map_check.getf features_pv cpu |> CPU_policy.to_string) (Map_check.getf features_hvm_host cpu |> CPU_policy.to_string) diff --git a/ocaml/xenopsd/lib/xenops_server_skeleton.ml b/ocaml/xenopsd/lib/xenops_server_skeleton.ml index c688aa79233..dc1b826f85e 100644 --- a/ocaml/xenopsd/lib/xenops_server_skeleton.ml +++ b/ocaml/xenopsd/lib/xenops_server_skeleton.ml @@ -26,6 +26,7 @@ module HOST = struct { Host.cpu_count= 0 ; socket_count= 0 + ; threads_per_core= 0 ; vendor= "unknown" ; speed= "" ; modelname= "" diff --git a/ocaml/xenopsd/xc/xenops_server_xen.ml b/ocaml/xenopsd/xc/xenops_server_xen.ml index a04dabe98c6..4de4bbc3573 100644 --- a/ocaml/xenopsd/xc/xenops_server_xen.ml +++ b/ocaml/xenopsd/xc/xenops_server_xen.ml @@ -985,6 +985,7 @@ module HOST = struct let socket_count = p.nr_cpus / (p.threads_per_core * p.cores_per_socket) in + let threads_per_core = p.threads_per_core in let features = get_cpu_featureset xc Featureset_host in (* this is Default policy in Xen's terminology, used on boot for new VMs *) let features_pv_host = get_cpu_featureset xc Featureset_pv in @@ -1012,6 +1013,7 @@ module HOST = struct { Host.cpu_count ; socket_count + ; threads_per_core ; vendor ; speed ; modelname From c31eb143e5c73a6ddc261bebecebe6613e97b8b7 Mon Sep 17 00:00:00 2001 From: Benjamin Reis Date: Wed, 21 Feb 2024 14:43:27 +0100 Subject: [PATCH 10/99] Filter out link IPv6 when migrating VMs - New pif helper: `get_non_link_ipv6` to get the 1st non link IPv6 of a PIF - Use the helper in `migrate_receive` and `get_primary_address` used in host evacuation to have a valid IPv6 of the destination host Co-authored-by: Pau Ruiz Safont Signed-off-by: Benjamin Reis --- ocaml/xapi/xapi_host.ml | 9 +++------ ocaml/xapi/xapi_pif_helpers.ml | 14 +++++++++++++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ocaml/xapi/xapi_host.ml b/ocaml/xapi/xapi_host.ml index 875b3be5cf2..768f33aba7b 100644 --- a/ocaml/xapi/xapi_host.ml +++ b/ocaml/xapi/xapi_host.ml @@ -20,6 +20,7 @@ let with_lock = Xapi_stdext_threads.Threadext.Mutex.execute module Unixext = Xapi_stdext_unix.Unixext open Xapi_host_helpers +open Xapi_pif_helpers open Db_filter_types open Workload_balancing @@ -2555,14 +2556,10 @@ let migrate_receive ~__context ~host ~network ~options:_ = let configuration_mode = Db.PIF.get_ipv6_configuration_mode ~__context ~self:pif in - match Db.PIF.get_IPv6 ~__context ~self:pif with + match Xapi_pif_helpers.get_non_link_ipv6 ~__context ~pif with | [] -> ("", configuration_mode) - | ip :: _ -> - (* The CIDR is also stored in the IPv6 field of a PIF. *) - let ipv6 = - match String.split_on_char '/' ip with hd :: _ -> hd | _ -> "" - in + | ipv6 :: _ -> (ipv6, configuration_mode) ) in diff --git a/ocaml/xapi/xapi_pif_helpers.ml b/ocaml/xapi/xapi_pif_helpers.ml index fc6c9708511..ae29c3c366e 100644 --- a/ocaml/xapi/xapi_pif_helpers.ml +++ b/ocaml/xapi/xapi_pif_helpers.ml @@ -255,10 +255,22 @@ let is_device_underneath_same_type ~__context pif1 pif2 = in get_device_info pif1 = get_device_info pif2 +let get_non_link_ipv6 ~__context ~pif = + let valid_nonlink ipv6 = + let open Ipaddr.V6 in + ipv6 + |> Prefix.of_string + |> Result.to_option + |> Fun.flip Option.bind @@ fun cidr -> + let addr = Prefix.address cidr in + match scope addr with Ipaddr.Link -> None | _ -> Some (to_string addr) + in + List.filter_map valid_nonlink (Db.PIF.get_IPv6 ~__context ~self:pif) + let get_primary_address ~__context ~pif = match Db.PIF.get_primary_address_type ~__context ~self:pif with | `IPv4 -> ( match Db.PIF.get_IP ~__context ~self:pif with "" -> None | ip -> Some ip ) | `IPv6 -> - List.nth_opt (Db.PIF.get_IPv6 ~__context ~self:pif) 0 + List.nth_opt (get_non_link_ipv6 ~__context ~pif) 0 From 1877fe07107d2117b27711e7294003364131dd6f Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Fri, 23 Feb 2024 17:33:20 +0000 Subject: [PATCH 11/99] Adjust the codcov target for python unit tests Signed-off-by: Steven Woods --- .codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codecov.yml b/.codecov.yml index d49bbb420c5..8380434a2a5 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -192,7 +192,7 @@ coverage: # tests: # Ensure that all tests are executed (tests themselves must be 100% covered) - target: 100% + target: 98% paths: ["**/test_*.py"] From 7e05103e435286d32154eb6df8c1ff5ad718b628 Mon Sep 17 00:00:00 2001 From: Guillaume Date: Mon, 26 Feb 2024 09:58:19 +0100 Subject: [PATCH 12/99] Xapi service depends on systemd-tmpfiles-setup Signed-off-by: Guillaume --- scripts/xapi.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/xapi.service b/scripts/xapi.service index e08cbff1067..58923c0a92c 100644 --- a/scripts/xapi.service +++ b/scripts/xapi.service @@ -1,6 +1,8 @@ [Unit] Description=XenAPI server (XAPI) +Requires=systemd-tmpfiles-setup.service +After=systemd-tmpfiles-setup.service After=attach-static-vdis.service After=forkexecd.service After=message-switch.service From 0f140cf3241a77db321bf67727928e42e75d7a20 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 7 Feb 2024 15:02:36 +0000 Subject: [PATCH 13/99] CP-47431: Use NuGet references in PowerShell SDK project Also: - Update copyright notices - Update `README` files to use markdown - Add `.gitignore` file - Drop extension from `LICENSE.txt` - Add `.sln` file since it's needed for NuGet usage in VS22 - Add .NET 8.0 to list of target frameworks for the SDK project - Ensure all DLLs are placed in `bin/` when running `dotnet build`. Without `CopyLocalLockFileAssemblies = true` this is not the case for .NET builds. Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/csharp/autogen/README.md | 24 +- ocaml/sdk-gen/powershell/README.dist | 171 ------- ocaml/sdk-gen/powershell/README_51.dist | 163 ------- ocaml/sdk-gen/powershell/autogen/.gitignore | 434 ++++++++++++++++++ ocaml/sdk-gen/powershell/autogen/README.md | 172 +++++++ ocaml/sdk-gen/powershell/autogen/README_51.md | 164 +++++++ .../powershell/autogen/XenServerPSModule.psd1 | 11 +- ocaml/sdk-gen/powershell/autogen/dune | 23 +- .../src/XenServerPowerShell.csproj} | 17 +- 9 files changed, 798 insertions(+), 381 deletions(-) delete mode 100644 ocaml/sdk-gen/powershell/README.dist delete mode 100644 ocaml/sdk-gen/powershell/README_51.dist create mode 100644 ocaml/sdk-gen/powershell/autogen/.gitignore create mode 100644 ocaml/sdk-gen/powershell/autogen/README.md create mode 100644 ocaml/sdk-gen/powershell/autogen/README_51.md rename ocaml/sdk-gen/powershell/{templates/XenServerPowerShell.csproj.mustache => autogen/src/XenServerPowerShell.csproj} (73%) diff --git a/ocaml/sdk-gen/csharp/autogen/README.md b/ocaml/sdk-gen/csharp/autogen/README.md index c0637502919..bd75fb5ffd2 100644 --- a/ocaml/sdk-gen/csharp/autogen/README.md +++ b/ocaml/sdk-gen/csharp/autogen/README.md @@ -13,58 +13,54 @@ are ideal for developers wishing to use XenServer.NET. XenServer.NET is free software. You can redistribute and modify it under the terms of the BSD 2-Clause license. See LICENSE.txt for details. - ## Reference -For XenServer documentation see https://docs.xenserver.com +For XenServer documentation see The XenServer Management API Reference is available at -https://docs.xenserver.com/en-us/xenserver/8/developer/management-api + The XenServer Software Development Kit Guide is available at -https://docs.xenserver.com/en-us/xenserver/8/developer/sdk-guide + A number of examples to help you get started with the SDK is available at -https://github.com/xenserver/xenserver-samples + For community content, blogs, and downloads, visit -https://www.xenserver.com/blogs and https://www.citrix.com/community + and To network with other developers using XenServer visit -https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver - + ## Prerequisites This library requires .NET Standard 2.0. - ## Dependencies XenServer.NET is dependent upon the following libraries: -- Newtonsoft JSON.NET by James Newton-King (see https://www.newtonsoft.com). +- Newtonsoft JSON.NET by James Newton-King (see ). JSON.NET is licensed under the MIT license. - ## Downloads - This archive contains the following folders that are relevant to .NET developers: + - XenServer.NET: contains the ready compiled binaries in the form of a NuGet package. - XenServer.NET\src: contains the source code shipped as a Visual Studio project. - ## Getting Started - Extract the contents of this archive. A. To build the source code: + 1. Open the project XenServer.csproj in Visual Studio. 2. You should now be ready to build the source code. B. To use the NuGet package in your code (offline): + 1. Add the location of the package as a NuGet source: ```pwsh diff --git a/ocaml/sdk-gen/powershell/README.dist b/ocaml/sdk-gen/powershell/README.dist deleted file mode 100644 index e343f9778a8..00000000000 --- a/ocaml/sdk-gen/powershell/README.dist +++ /dev/null @@ -1,171 +0,0 @@ -XenServer PowerShell Module -=========================== - -Copyright (c) 2013-2023 Cloud Software Group, Inc. All Rights Reserved. - -The XenServer PowerShell Module is a complete SDK for XenServer, -exposing the XenServer API as Windows PowerShell cmdlets. - -The XenServer PowerShell Module includes a cmdlet for each API call, -so API documentation and examples written for other languages will apply equally -well to PowerShell. In particular, the SDK Guide and the Management API Guide -are ideal for developers wishing to use this module. - -This module is free software. You can redistribute and modify it under the -terms of the BSD 2-Clause license. See LICENSE.txt for details. - - -Reference ---------- - -For XenServer documentation see https://docs.xenserver.com - -The XenServer Management API Reference is available at -https://docs.xenserver.com/en-us/xenserver/8/developer/management-api - -The XenServer Software Development Kit Guide is available at -https://docs.xenserver.com/en-us/xenserver/8/developer/sdk-guide - -A number of examples to help you get started with the SDK is available at -https://github.com/xenserver/xenserver-samples - -For community content, blogs, and downloads, visit -https://www.xenserver.com/blogs and https://www.citrix.com/community/ - -To network with other developers using XenServer visit -https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver/ - - -Prerequisites -------------- - -This library requires .NET 6.0 and PowerShell 7.2 or greater. - - -Dependencies ------------- - -The XenServer PowerShell Module is dependent upon the following libraries: - -- Newtonsoft JSON.NET by James Newton-King (see https://www.newtonsoft.com/). - JSON.NET is licensed under the MIT license; see LICENSE.Newtonsoft.Json.txt - for details. A patched version of the library (Newtonsoft.Json.CH.dll) is - shipped with the XenServer PowerShell Module. - -- XenServer.NET by Cloud Software Group, Inc. - XenServer.NET is a complete SDK for XenServer, exposing the XenServer - API as .NET classes. It is written in C#. - - -Folder Structure ----------------- - -This archive contains the following folders that are relevant to PowerShell users: - -- XenServerPowerShell\XenServerPSModule: this is the XenServer PowerShell - Module -- XenServerPowerShell\src: contains the C# source code for the XenServer - cmdlets shipped as a Visual Studio project. - - -Getting Started ---------------- - -1. Extract the contents of this archive. - - Note that some web browsers may mark the SDK ZIP file as "blocked" during - the download. To import the module successfully you will need to unblock the - archive before extracting its contents. To unblock the archive, right-click - on it and launch the Properties dialog. Click the Unblock button, then the - Apply or OK button. - -2. Navigate to the extracted XenServerPowerShell directory and copy the whole - folder XenServerPSModule into your PowerShell modules directory. - - - On Windows this will normally be $env:UserProfile\Documents\PowerShell\Modules - for per-user configuration, or $env:ProgramFiles\PowerShell\7\Modules for - system-wide configuration (note that $env:ProgramFiles\WindowsPowerShell\Modules - is also acceptable). - - - On Linux this will be ~/.local/share/powershell/Modules for per-user - configuration, or /usr/local/share/powershell/Modules for system-wide - configuration. - - For more information see PowerShell's documentation on module paths: - - PS> Get-Help about_PSModulePath - -3. Open a PowerShell 7 prompt as administrator. - - To do this, open the Windows Start menu by clicking the Start icon, find - the item PowerShell 7, right click it and select Run as administrator. - -4. Determine the current execution policy: - - PS> Get-ExecutionPolicy - - If the current policy is Restricted, you need to set it to RemoteSigned: - - PS> Set-ExecutionPolicy RemoteSigned - - You should understand the security implications of this change. If you are - unsure, see PowerShell's documentation on execution policies: - - PS> Get-Help about_Execution_Policies - - If the current policy is AllSigned, you will be able to use the XenServer - PowerShell module, but it will be inconvenient because this policy requires - even scripts that you write on the local computer to be signed. You may want - to change it to RemoteSigned, as above. - - If the current policy is RemoteSigned, ByPass, or Unrestricted there is - nothing to do. - -5. Exit the privileged instance of PowerShell. - -6. Open a PowerShell 7 prompt as a regular user (click Start > PowerShell 7) - and import the XenServer PowerShell Module: - - PS> Import-Module XenServerPSModule - -7. If you wish to load specific environment settings when the XenServer - PowerShell module is loaded, create the file XenServerProfile.ps1 and put it - in the folder containing your $PROFILE file for per-user configuration, or - in $PSHOME for system-wide configuration. - - - On Windows these will normally be $env:UserProfile\Documents\PowerShell - for per-user configuration, or $env:ProgramFiles\PowerShell\7 for - system-wide configuration. - - - On Linux these will be ~/.config/powershell for per-user configuration, - or /opt/microsoft/powershell/7 for system-wide configuration. - -8. For an overview of the XenServer PowerShell Module type: - - PS> Get-Help about_XenServer - - You can obtain a list of all available cmdlets by typing: - - PS> Get-Command -Module XenServerPSModule - - For help with a specific command use: - - PS> Get-Help [CommandName] - -9. Here is a quick example of opening a session and making a call to a server: - - PS> Connect-XenServer -Url https:// - PS> Get-XenVM - PS> Disconnect-XenServer - - -Building and Debugging the Source Code --------------------------------------- - -1. Open the project XenServerPowerShell.csproj in Visual Studio 2022. You should - now be ready to build the source code. - -2. If in Debug mode, clicking Start will launch a PowerShell 7 prompt as an - external process, and import the compiled XenServerPowerShell.dll as a module - (without, however, processing the scripts, types, and formats shipped within - the XenServerPSModule). You should now be ready to debug the cmdlets. diff --git a/ocaml/sdk-gen/powershell/README_51.dist b/ocaml/sdk-gen/powershell/README_51.dist deleted file mode 100644 index 5806368881c..00000000000 --- a/ocaml/sdk-gen/powershell/README_51.dist +++ /dev/null @@ -1,163 +0,0 @@ -XenServer PowerShell Module -=========================== - -Copyright (c) 2013-2023 Cloud Software Group, Inc. All Rights Reserved. - -The XenServer PowerShell Module is a complete SDK for XenServer, -exposing the XenServer API as Windows PowerShell cmdlets. - -The XenServer PowerShell Module includes a cmdlet for each API call, -so API documentation and examples written for other languages will apply equally -well to PowerShell. In particular, the SDK Guide and the Management API Guide -are ideal for developers wishing to use this module. - -This module is free software. You can redistribute and modify it under the -terms of the BSD 2-Clause license. See LICENSE.txt for details. - - -Reference ---------- - -For XenServer documentation see https://docs.xenserver.com - -The XenServer Management API Reference is available at -https://docs.xenserver.com/en-us/xenserver/8/developer/management-api - -The XenServer Software Development Kit Guide is available at -https://docs.xenserver.com/en-us/xenserver/8/developer/sdk-guide - -A number of examples to help you get started with the SDK is available at -https://github.com/xenserver/xenserver-samples - -For community content, blogs, and downloads, visit -https://www.xenserver.com/blogs and https://www.citrix.com/community/ - -To network with other developers using XenServer visit -https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver/ - - -Prerequisites -------------- - -This library requires .NET Framework 4.5 or greater and PowerShell 5.1 or greater. - - -Dependencies ------------- - -The XenServer PowerShell Module is dependent upon the following libraries: - -- Newtonsoft JSON.NET by James Newton-King (see https://www.newtonsoft.com/). - JSON.NET is licensed under the MIT license; see LICENSE.Newtonsoft.Json.txt - for details. A patched version of the library (Newtonsoft.Json.CH.dll) is - shipped with the XenServer PowerShell Module. - -- XenServer.NET by Cloud Software Group, Inc. - XenServer.NET is a complete SDK for XenServer, exposing the XenServer - API as .NET classes. It is written in C#. - - -Folder Structure ----------------- - -This archive contains the following folders that are relevant to PowerShell users: - -- XenServerPowerShell\XenServerPSModule: this is the XenServer PowerShell - Module -- XenServerPowerShell\src: contains the C# source code for the XenServer - cmdlets shipped as a Visual Studio project. - - -Getting Started ---------------- - -1. Extract the contents of this archive. - - Note that some web browsers may mark the SDK ZIP file as "blocked" during - the download. To import the module successfully you will need to unblock the - archive before extracting its contents. To unblock the archive, right-click - on it and launch the Properties dialog. Click the Unblock button, then the - Apply or OK button. - -2. Navigate to the extracted XenServerPowerShell directory and copy the whole - folder XenServerPSModule into your PowerShell modules directory. - - - On Windows this will normally be $env:UserProfile\Documents\WindowsPowerShell\Modules - for per-user configuration, or $env:windir\system32\WindowsPowerShell\v1.0\Modules for - system-wide configuration. - - For more information see PowerShell's documentation on module paths: - - PS> Get-Help about_PSModulePath - -3. Open a PowerShell prompt as administrator. - - To do this, open the Windows Start menu by clicking the Start icon, find - the item PowerShell, right click it and select Run as administrator. - -4. Determine the current execution policy: - - PS> Get-ExecutionPolicy - - If the current policy is Restricted, you need to set it to RemoteSigned: - - PS> Set-ExecutionPolicy RemoteSigned - - You should understand the security implications of this change. If you are - unsure, see PowerShell's documentation on execution policies: - - PS> Get-Help about_Execution_Policies - - If the current policy is AllSigned, you will be able to use the XenServer - PowerShell module, but it will be inconvenient because this policy requires - even scripts that you write on the local computer to be signed. You may want - to change it to RemoteSigned, as above. - - If the current policy is RemoteSigned, ByPass, or Unrestricted there is - nothing to do. - -5. Exit the privileged instance of PowerShell. - -6. Open a PowerShell prompt as a regular user (click Start > PowerShell) - and import the XenServer PowerShell Module: - - PS> Import-Module XenServerPSModule - -7. If you wish to load specific environment settings when the XenServer - PowerShell module is loaded, create the file XenServerProfile.ps1 and put it - in the folder containing your $PROFILE file for per-user configuration, or - in $PSHOME for system-wide configuration. - - - On Windows these will normally be $env:UserProfile\Documents\WindowsPowerShell - for per-user configuration, or $env:windir\system32\WindowsPowerShell\v1.0 for - system-wide configuration. - -8. For an overview of the XenServer PowerShell Module type: - - PS> Get-Help about_XenServer - - You can obtain a list of all available cmdlets by typing: - - PS> Get-Command -Module XenServerPSModule - - For help with a specific command use: - - PS> Get-Help [CommandName] - -9. Here is a quick example of opening a session and making a call to a server: - - PS> Connect-XenServer -Url https:// - PS> Get-XenVM - PS> Disconnect-XenServer - - -Building and Debugging the Source Code --------------------------------------- - -1. Open the project XenServerPowerShell.csproj in Visual Studio 2022. You should - now be ready to build the source code. - -2. If in Debug mode, clicking Start will launch a PowerShell prompt as an - external process, and import the compiled XenServerPowerShell.dll as a module - (without, however, processing the scripts, types, and formats shipped within - the XenServerPSModule). You should now be ready to debug the cmdlets. diff --git a/ocaml/sdk-gen/powershell/autogen/.gitignore b/ocaml/sdk-gen/powershell/autogen/.gitignore new file mode 100644 index 00000000000..f6a6481aa75 --- /dev/null +++ b/ocaml/sdk-gen/powershell/autogen/.gitignore @@ -0,0 +1,434 @@ +# Created by https://www.toptal.com/developers/gitignore/api/visualstudio,visualstudiocode,dotnetcore,powershell +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudio,visualstudiocode,dotnetcore,powershell + +### DotnetCore ### +# .NET Core build folders +bin/ +obj/ + +# Common node modules locations +/node_modules +/wwwroot/node_modules + +### PowerShell ### +# Exclude packaged modules +*.zip + +# Exclude .NET assemblies from source +*.dll + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +### VisualStudio ### +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +*.code-workspace + +# Local History for Visual Studio Code + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml + +### VisualStudio Patch ### +# Additional files built by Visual Studio + +# End of https://www.toptal.com/developers/gitignore/api/visualstudio,visualstudiocode,dotnetcore,powershell \ No newline at end of file diff --git a/ocaml/sdk-gen/powershell/autogen/README.md b/ocaml/sdk-gen/powershell/autogen/README.md new file mode 100644 index 00000000000..cbe06791bad --- /dev/null +++ b/ocaml/sdk-gen/powershell/autogen/README.md @@ -0,0 +1,172 @@ +# XenServer PowerShell Module + +Copyright (c) 2013-2024 Cloud Software Group, Inc. All Rights Reserved. + +The XenServer PowerShell Module is a complete SDK for XenServer, +exposing the XenServer API as Windows PowerShell cmdlets. + +The XenServer PowerShell Module includes a cmdlet for each API call, +so API documentation and examples written for other languages will apply equally +well to PowerShell. In particular, the SDK Guide and the Management API Guide +are ideal for developers wishing to use this module. + +This module is free software. You can redistribute and modify it under the +terms of the BSD 2-Clause license. See LICENSE.txt for details. + +## Reference + +For XenServer documentation see + +The XenServer Management API Reference is available at + + +The XenServer Software Development Kit Guide is available at + + +A number of examples to help you get started with the SDK is available at + + +For community content, blogs, and downloads, visit + and + +To network with other developers using XenServer visit + + +## Prerequisites + +This library requires .NET 6.0 and PowerShell 7.2 or greater. + +## Dependencies + +The XenServer PowerShell Module is dependent upon the following libraries: + +- Newtonsoft JSON.NET by James Newton-King (see ). + JSON.NET is licensed under the MIT license. + +- XenServer.NET by Cloud Software Group, Inc. + XenServer.NET is a complete SDK for XenServer, exposing the XenServer + API as .NET classes. It is written in C#. + +## Folder Structure + +This archive contains the following folders that are relevant to PowerShell users: + +- `XenServerPowerShell\XenServerPSModule`: this is the XenServer PowerShell + Module +- `XenServerPowerShell\src`: contains the C# source code for the XenServer + cmdlets shipped as a Visual Studio project. + +## Getting Started + +1. Extract the contents of this archive. + + Note that some web browsers may mark the SDK ZIP file as "blocked" during + the download. To import the module successfully you will need to unblock the + archive before extracting its contents. To unblock the archive, right-click + on it and launch the Properties dialog. Click the Unblock button, then the + Apply or OK button. + +2. Navigate to the extracted XenServerPowerShell directory and copy the whole + folder XenServerPSModule into your PowerShell modules directory. + + - On Windows this will normally be `$env:UserProfile\Documents\PowerShell\Modules` + for per-user configuration, or `$env:ProgramFiles\PowerShell\7\Modules` for + system-wide configuration (note that `$env:ProgramFiles\WindowsPowerShell\Modules` + is also acceptable). + + - On Linux this will be `~/.local/share/powershell/Modules` for per-user + configuration, or `/usr/local/share/powershell/Modules` for system-wide + configuration. + + For more information see PowerShell's documentation on module paths: + + PS> Get-Help about_PSModulePath + +3. Open a PowerShell 7 prompt as administrator. + + To do this, open the Windows Start menu by clicking the Start icon, find + the item PowerShell 7, right click it and select Run as administrator. + +4. Determine the current execution policy: + +```ps + PS> Get-ExecutionPolicy +``` + + If the current policy is Restricted, you need to set it to `RemoteSigned`: + +```ps + PS> Set-ExecutionPolicy RemoteSigned +``` + + You should understand the security implications of this change. If you are + unsure, see PowerShell's documentation on execution policies: + +```ps + PS> Get-Help about_Execution_Policies +``` + + If the current policy is AllSigned, you will be able to use the XenServer + PowerShell module, but it will be inconvenient because this policy requires + even scripts that you write on the local computer to be signed. You may want + to change it to `RemoteSigned`, as above. + + If the current policy is `RemoteSigned`, `ByPass`, or `Unrestricted` there is + nothing to do. + +5. Exit the privileged instance of PowerShell. + +6. Open a PowerShell 7 prompt as a regular user (click Start > PowerShell 7) + and import the XenServer PowerShell Module: + +```ps + PS> Import-Module XenServerPSModule +``` + +7. If you wish to load specific environment settings when the XenServer + PowerShell module is loaded, create the file `XenServerProfile.ps1` and put it + in the folder containing your `$PROFILE` file for per-user configuration, or + in `$PSHOME` for system-wide configuration. + + - On Windows these will normally be `$env:UserProfile\Documents\PowerShell` + for per-user configuration, or `$env:ProgramFiles\PowerShell\7` for + system-wide configuration. + + - On Linux these will be `~/.config/powershell` for per-user configuration, + or `/opt/microsoft/powershell/7` for system-wide configuration. + +8. For an overview of the XenServer PowerShell Module type: + +```ps + PS> Get-Help about_XenServer +``` + + You can obtain a list of all available cmdlets by typing: + +```ps + PS> Get-Command -Module XenServerPSModule +``` + + For help with a specific command use: + +```ps + PS> Get-Help [CommandName] +``` + +9. Here is a quick example of opening a session and making a call to a server: + +```ps + PS> Connect-XenServer -Url https:// + PS> Get-XenVM + PS> Disconnect-XenServer +``` + +## Building and Debugging the Source Code + +1. Open the project `XenServerPowerShell.csproj` in Visual Studio 2022. You should + now be ready to build the source code. + +2. If in Debug mode, clicking Start will launch a PowerShell 7 prompt as an + external process, and import the compiled `XenServerPowerShell.dll` as a module + (without, however, processing the scripts, types, and formats shipped within + the `XenServerPSModule`). You should now be ready to debug the cmdlets. diff --git a/ocaml/sdk-gen/powershell/autogen/README_51.md b/ocaml/sdk-gen/powershell/autogen/README_51.md new file mode 100644 index 00000000000..8088982ff47 --- /dev/null +++ b/ocaml/sdk-gen/powershell/autogen/README_51.md @@ -0,0 +1,164 @@ +# XenServer PowerShell Module + +Copyright (c) 2013-2024 Cloud Software Group, Inc. All Rights Reserved. + +The XenServer PowerShell Module is a complete SDK for XenServer, +exposing the XenServer API as Windows PowerShell cmdlets. + +The XenServer PowerShell Module includes a cmdlet for each API call, +so API documentation and examples written for other languages will apply equally +well to PowerShell. In particular, the SDK Guide and the Management API Guide +are ideal for developers wishing to use this module. + +This module is free software. You can redistribute and modify it under the +terms of the BSD 2-Clause license. See LICENSE.txt for details. + +## Reference + +For XenServer documentation see + +The XenServer Management API Reference is available at + + +The XenServer Software Development Kit Guide is available at + + +A number of examples to help you get started with the SDK is available at + + +For community content, blogs, and downloads, visit + and + +To network with other developers using XenServer visit + + +## Prerequisites + +This library requires .NET Framework 4.5 or greater and PowerShell 5.1 or greater. + +## Dependencies + +The XenServer PowerShell Module is dependent upon the following libraries: + +- Newtonsoft JSON.NET by James Newton-King (see ). + JSON.NET is licensed under the MIT license. + +- XenServer.NET by Cloud Software Group, Inc. + XenServer.NET is a complete SDK for XenServer, exposing the XenServer + API as .NET classes. It is written in C#. + +## Folder Structure + +This archive contains the following folders that are relevant to PowerShell users: + +- `XenServerPowerShell\XenServerPSModule`: this is the XenServer PowerShell + Module +- `XenServerPowerShell\src`: contains the C# source code for the XenServer + cmdlets shipped as a Visual Studio project. + +## Getting Started + +1. Extract the contents of this archive. + + Note that some web browsers may mark the SDK ZIP file as "blocked" during + the download. To import the module successfully you will need to unblock the + archive before extracting its contents. To unblock the archive, right-click + on it and launch the Properties dialog. Click the Unblock button, then the + Apply or OK button. + +2. Navigate to the extracted XenServerPowerShell directory and copy the whole + folder XenServerPSModule into your PowerShell modules directory. + + - On Windows this will normally be `$env:UserProfile\Documents\WindowsPowerShell\Modules` + for per-user configuration, or `$env:windir\system32\WindowsPowerShell\v1.0\Modules` for + system-wide configuration. + + For more information see PowerShell's documentation on module paths: + + PS> Get-Help about_PSModulePath + +3. Open a PowerShell 5.1 prompt as administrator. + + To do this, open the Windows Start menu by clicking the Start icon, find + the item PowerShell 5.1, right click it and select Run as administrator. + +4. Determine the current execution policy: + +```ps + PS> Get-ExecutionPolicy +``` + + If the current policy is Restricted, you need to set it to `RemoteSigned`: + +```ps + PS> Set-ExecutionPolicy RemoteSigned +``` + + You should understand the security implications of this change. If you are + unsure, see PowerShell's documentation on execution policies: + +```ps + PS> Get-Help about_Execution_Policies +``` + + If the current policy is AllSigned, you will be able to use the XenServer + PowerShell module, but it will be inconvenient because this policy requires + even scripts that you write on the local computer to be signed. You may want + to change it to `RemoteSigned`, as above. + + If the current policy is `RemoteSigned`, `ByPass`, or `Unrestricted` there is + nothing to do. + +5. Exit the privileged instance of PowerShell. + +6. Open a PowerShell 5.1 prompt as a regular user (click Start > PowerShell) + and import the XenServer PowerShell Module: + +```ps + PS> Import-Module XenServerPSModule +``` + +7. If you wish to load specific environment settings when the XenServer + PowerShell module is loaded, create the file `XenServerProfile.ps1` and put it + in the folder containing your `$PROFILE` file for per-user configuration, or + in `$PSHOME` for system-wide configuration. + + - On Windows these will normally be `$env:UserProfile\Documents\WindowsPowerShell` + for per-user configuration, or `$env:windir\system32\WindowsPowerShell\v1.0` for + system-wide configuration. + +8. For an overview of the XenServer PowerShell Module type: + +```ps + PS> Get-Help about_XenServer +``` + + You can obtain a list of all available cmdlets by typing: + +```ps + PS> Get-Command -Module XenServerPSModule +``` + + For help with a specific command use: + +```ps + PS> Get-Help [CommandName] +``` + +9. Here is a quick example of opening a session and making a call to a server: + +```ps + PS> Connect-XenServer -Url https:// + PS> Get-XenVM + PS> Disconnect-XenServer +``` + +## Building and Debugging the Source Code + +1. Open the project `XenServerPowerShell.csproj` in Visual Studio 2022. You should + now be ready to build the source code. + +2. If in Debug mode, clicking Start will launch a PowerShell prompt as an + external process, and import the compiled `XenServerPowerShell.dll` as a module + (without, however, processing the scripts, types, and formats shipped within + the `XenServerPSModule`). You should now be ready to debug the cmdlets. diff --git a/ocaml/sdk-gen/powershell/autogen/XenServerPSModule.psd1 b/ocaml/sdk-gen/powershell/autogen/XenServerPSModule.psd1 index 153595f4afe..87edaa0214f 100644 --- a/ocaml/sdk-gen/powershell/autogen/XenServerPSModule.psd1 +++ b/ocaml/sdk-gen/powershell/autogen/XenServerPSModule.psd1 @@ -37,7 +37,7 @@ GUID = 'D695A8B9-039A-443C-99A4-0D48D7C6AD76' #Copyright Author = '' CompanyName = 'Cloud Software Group, Inc' -Copyright = 'Copyright (c) 2013-2023 Cloud Software Group, Inc. All rights reserved.' +Copyright = 'Copyright (c) 2013-2024 Cloud Software Group, Inc. All rights reserved.' # Requirements PowerShellVersion = '@PS_VERSION@' @@ -49,17 +49,16 @@ ProcessorArchitecture = 'None' RootModule = 'XenServerPowerShell.dll' RequiredModules = @() NestedModules = @() -RequiredAssemblies = @('Newtonsoft.Json.CH.dll', +RequiredAssemblies = @('Newtonsoft.Json.dll', 'XenServer.dll') ScriptsToProcess = @('Initialize-Environment.ps1') TypesToProcess = @('XenServer.types.ps1xml') FormatsToProcess = @('XenServer.format.ps1xml') FileList = @('about_XenServer.help.txt', - 'Newtonsoft.Json.CH.dll', + 'Newtonsoft.Json.dll', 'Initialize-Environment.ps1', - 'LICENSE.Newtonsoft.Json.txt', - 'LICENSE.txt', - 'README.txt', + 'LICENSE', + 'README.md', 'XenServer.dll', 'XenServer.format.ps1xml', 'XenServer.types.ps1xml', diff --git a/ocaml/sdk-gen/powershell/autogen/dune b/ocaml/sdk-gen/powershell/autogen/dune index 96bedf2fd99..56eb4c8480a 100644 --- a/ocaml/sdk-gen/powershell/autogen/dune +++ b/ocaml/sdk-gen/powershell/autogen/dune @@ -1,34 +1,15 @@ (rule - (targets LICENSE.txt) + (targets LICENSE) (deps ../../LICENSE ) (action (copy %{deps} %{targets})) ) -(rule - (targets README.txt) - (deps - ../README.dist - ) - (action (copy %{deps} %{targets})) -) - -(rule - (targets README_51.txt) - (deps - ../README_51.dist - ) - (action (copy %{deps} %{targets})) -) - (alias (name generate) (deps - LICENSE.txt - README.txt - README_51.txt + LICENSE (source_tree .) ) ) - diff --git a/ocaml/sdk-gen/powershell/templates/XenServerPowerShell.csproj.mustache b/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj similarity index 73% rename from ocaml/sdk-gen/powershell/templates/XenServerPowerShell.csproj.mustache rename to ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj index 17498994073..85ea0dc72b4 100644 --- a/ocaml/sdk-gen/powershell/templates/XenServerPowerShell.csproj.mustache +++ b/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj @@ -1,10 +1,14 @@ 0.0.0 - net6.0;net45 + net8.0;net6.0;net45 Library True + + + true + True Program @@ -17,8 +21,11 @@ C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe -NoLogo -NoExit -Command "Import-Module '.\XenServerPowerShell.dll'" + + + - + @@ -27,9 +34,7 @@ - - False - .\XenServer.dll - + + From 36f852dbe2721779966ab4acb4f278859f1acec9 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 7 Feb 2024 15:02:36 +0000 Subject: [PATCH 14/99] Add reusable workflow for generating and building all SDKs Makes use of new composite action that sets up a XenAPI environment in a Ubuntu node. Signed-off-by: Danilo Del Busso --- .github/workflows/generate-and-build-sdks.yml | 208 ++++++++++++++++++ .../setup-xapi-environment/action.yml | 59 +++++ 2 files changed, 267 insertions(+) create mode 100644 .github/workflows/generate-and-build-sdks.yml create mode 100644 .github/workflows/setup-xapi-environment/action.yml diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml new file mode 100644 index 00000000000..c5bff2cad77 --- /dev/null +++ b/.github/workflows/generate-and-build-sdks.yml @@ -0,0 +1,208 @@ +name: Generate and Build SDKs + +on: + workflow_call: + inputs: + xapi_version: + required: true + type: string + +jobs: + generate-sdk-sources: + name: Generate SDK sources + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup XenAPI environment + uses: ./.github/workflows/setup-xapi-environment + with: + xapi_version: ${{ inputs.xapi_version }} + + - name: Generate SDKs + shell: bash + run: opam exec -- make sdk + + - name: Store C# SDK source + uses: actions/upload-artifact@v3 + with: + name: SDK_Source_CSharp + path: _build/install/default/xapi/sdk/csharp/* + + - name: Store PowerShell SDK source + uses: actions/upload-artifact@v3 + with: + name: SDK_Source_PowerShell + path: _build/install/default/xapi/sdk/powershell/* + + build-csharp-sdk: + name: Build C# SDK + runs-on: windows-2022 + needs: generate-sdk-sources + steps: + - name: Strip 'v' prefix from xapi version + shell : pwsh + run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Retrieve C# SDK source + uses: actions/download-artifact@v3 + with: + name: SDK_Source_CSharp + path: source/ + + - name: Build C# SDK + shell: pwsh + run: | + dotnet build source/src ` + --disable-build-servers ` + --configuration Release ` + -p:Version=${{ env.XAPI_VERSION_NUMBER }}-prerelease-unsigned ` + --verbosity=normal + + - name: Store C# SDK + uses: actions/upload-artifact@v3 + with: + name: SDK_Binaries_CSharp + path: source/src/bin/Release/XenServer.NET.${{ env.XAPI_VERSION_NUMBER }}-prerelease-unsigned.nupkg + + build-powershell-5x-sdk: + name: Build PowerShell 5.x SDK (.NET Framework 4.5) + needs: build-csharp-sdk + # PowerShell SDK for PowerShell 5.x needs to run on windows-2019 because + # windows-2022 doesn't contain .NET Framework 4.x dev tools + runs-on: windows-2019 + steps: + - name: Strip 'v' prefix from xapi version + shell : pwsh + run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Retrieve PowerShell SDK source + uses: actions/download-artifact@v3 + with: + name: SDK_Source_PowerShell + path: source/ + + - name: Retrieve C# SDK binaries + uses: actions/download-artifact@v3 + with: + name: SDK_Binaries_CSharp + path: csharp/ + + # Following needed for restoring packages + # when calling dotnet add package + - name: Set up dotnet CLI (.NET 6.0 and 8.0) + uses: actions/setup-dotnet@v3 + with: + dotnet-version: | + 6 + 8 + + - name: Setup project and dotnet CLI + shell: pwsh + run: | + dotnet nuget add source --name local ${{ github.workspace }}\csharp + dotnet add source/src package XenServer.NET --version ${{ env.XAPI_VERSION_NUMBER }}-prerelease-unsigned + + - name: Build PowerShell SDK (.NET Framework 4.5) + shell: pwsh + run: | + dotnet build source/src/XenServerPowerShell.csproj ` + --disable-build-servers ` + --configuration Release ` + -p:Version=${{ env.XAPI_VERSION_NUMBER }}-prerelease-unsigned ` + -p:TargetFramework=net45 ` + --verbosity=normal` + + - name: Update SDK and PS versions in "XenServerPSModule.psd1" + shell: pwsh + run: | + (Get-Content "source\XenServerPSModule.psd1") -replace "@SDK_VERSION@","${{ env.XAPI_VERSION_NUMBER }}" | Set-Content -Path "source\XenServerPSModule.psd1" + (Get-Content "source\XenServerPSModule.psd1") -replace "@PS_VERSION@","5.0" | Set-Content -Path "source\XenServerPSModule.psd1" + + - name: Move binaries to destination folder + shell: pwsh + run: | + New-Item -Path "." -Name "output" -ItemType "directory" + Copy-Item -Verbose "source\README_51.md" -Destination "output" -Force + Copy-Item -Verbose "source\LICENSE" -Destination "output" -Force + Copy-Item -Path "source\src\bin\Release\net45\*" -Include ".dll" "output\" + Get-ChildItem -Path "source" |` + Where-Object { $_.Extension -eq ".ps1" -or $_.Extension -eq ".ps1xml" -or $_.Extension -eq ".psd1" -or $_.Extension -eq ".txt" } |` + ForEach-Object -Process { Copy-Item -Verbose $_.FullName -Destination "output" } + + - name: Store PowerShell SDK (.NET Framework 4.5) + uses: actions/upload-artifact@v3 + with: + name: XenServerPowerShell_NET45 + path: output/**/* + + build-powershell-7x-sdk: + name: Build PowerShell 7.x SDK + strategy: + fail-fast: false + matrix: + dotnet: ["6", "8"] + needs: build-csharp-sdk + runs-on: windows-2022 + + steps: + - name: Strip 'v' prefix from xapi version + shell : pwsh + run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Retrieve PowerShell SDK source + uses: actions/download-artifact@v3 + with: + name: SDK_Source_PowerShell + path: source/ + + - name: Retrieve C# SDK binaries + uses: actions/download-artifact@v3 + with: + name: SDK_Binaries_CSharp + path: csharp/ + + - name: Set up dotnet CLI (.NET ${{ matrix.dotnet }}) + uses: actions/setup-dotnet@v3 + with: + dotnet-version: ${{ matrix.dotnet }} + + - name: Setup project and NuGet source + shell: pwsh + run: | + dotnet nuget add source --name local ${{ github.workspace }}\csharp + dotnet add source/src package XenServer.NET --version ${{ env.XAPI_VERSION_NUMBER }}-prerelease-unsigned + + - name: Build PowerShell SDK (.NET ${{ matrix.dotnet }}) + shell: pwsh + run: | + dotnet build source/src/XenServerPowerShell.csproj ` + --disable-build-servers ` + --configuration Release ` + -p:Version=${{ env.XAPI_VERSION_NUMBER }}-prerelease-unsigned ` + -p:TargetFramework=net${{ matrix.dotnet }}.0 ` + --verbosity=normal` + + - name: Update SDK and PS versions in "XenServerPSModule.psd1" + shell: pwsh + run: | + (Get-Content "source\XenServerPSModule.psd1") -replace "@SDK_VERSION@","${{ env.XAPI_VERSION_NUMBER }}" | Set-Content -Path "source\XenServerPSModule.psd1" + (Get-Content "source\XenServerPSModule.psd1") -replace "@PS_VERSION@","7.0" | Set-Content -Path "source\XenServerPSModule.psd1" + + - name: Move binaries to destination folder + shell: pwsh + run: | + New-Item -Path "." -Name "output" -ItemType "directory" + Copy-Item -Verbose "source\README.md" -Destination "output" -Force + Copy-Item -Verbose "source\LICENSE" -Destination "output" -Force + Copy-Item -Path "source\src\bin\Release\net${{ matrix.dotnet }}.0\*" -Include ".dll" "output\" + Get-ChildItem -Path "source" |` + Where-Object { $_.Extension -eq ".ps1" -or $_.Extension -eq ".ps1xml" -or $_.Extension -eq ".psd1" -or $_.Extension -eq ".txt" } |` + ForEach-Object -Process { Copy-Item -Verbose $_.FullName -Destination "output" } + + - name: Store PowerShell SDK (.NET ${{ matrix.dotnet }}) + uses: actions/upload-artifact@v3 + with: + name: XenServerPowerShell_NET${{ matrix.dotnet }} + path: output/**/* diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml new file mode 100644 index 00000000000..ca0863dfe32 --- /dev/null +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -0,0 +1,59 @@ +name: Setup XenAPI environment +description: Setup a XenAPI environment for making opam calls. + +inputs: + xapi_version: + description: "XenAPI version, pass to configure as --xapi_version=" + required: true +runs: + using: "composite" + steps: + - name: Free space + shell: bash + run: sudo rm -rf /usr/local/lib/android + + - name: Pull configuration from xs-opam + shell: bash + run: | + curl --fail --silent https://raw.githubusercontent.com/xapi-project/xs-opam/master/tools/xs-opam-ci.env | cut -f2 -d " " > .env + + - name: Download XE_SR_ERRORCODES.xml + shell : bash + run: | + mkdir -p /opt/xensource/sm + wget -O /opt/xensource/sm/XE_SR_ERRORCODES.xml https://raw.githubusercontent.com/xapi-project/sm/master/drivers/XE_SR_ERRORCODES.xml + + - name: Load environment file + id: dotenv + uses: falti/dotenv-action@v1.0.4 + + - name: Update Ubuntu repositories + shell: bash + run: sudo apt-get update + + - name: Use disk with more space for TMPDIR and XDG_CACHE_HOME + shell: bash + run: | + df -h || true + export TMPDIR="/mnt/build/tmp" + export XDG_CACHE_HOME="/mnt/build/cache" + sudo mkdir -p "${TMPDIR}" "${XDG_CACHE_HOME}" + sudo chown "$(id -u):$(id -g)" "${TMPDIR}" "${XDG_CACHE_HOME}" + echo "TMPDIR=${TMPDIR}" >>"$GITHUB_ENV" + echo "XDG_CACHE_HOME=${XDG_CACHE_HOME}" >>"$GITHUB_ENV" + + - name: Use ocaml + uses: ocaml/setup-ocaml@v2 + with: + ocaml-compiler: ${{ steps.dotenv.outputs.ocaml_version_full }} + opam-repositories: | + xs-opam: ${{ steps.dotenv.outputs.repository }} + dune-cache: true + + - name: Install dependencies + shell: bash + run: opam install . --deps-only --with-test -v + + - name: Configure + shell: bash + run: opam exec -- ./configure --xapi_version="${{ inputs.xapi_version }}" From b8dcdf81fa7660428e2d66e2c6051fd3f7b8961a Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 7 Feb 2024 15:02:36 +0000 Subject: [PATCH 15/99] Remove unused logic in `gen_powershell_binding.ml` Signed-off-by: Danilo Del Busso --- .../sdk-gen/powershell/gen_powershell_binding.ml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 2701e190767..2e1fb55e5fb 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -79,21 +79,7 @@ let rec main () = |> List.map gen_http_action in let all_cmdlets = cmdlets @ http_cmdlets in - List.iter (fun x -> write_file x.filename x.content) all_cmdlets ; - - let fnames = all_cmdlets |> List.map (fun x -> x.filename) in - let sorted_members = List.sort String.compare fnames in - let json = - `O - [ - ( "cmdlets" - , `A (List.map (fun x -> `O [("cmdlet", `String x)]) sorted_members) - ) - ] - in - render_file - ("XenServerPowerShell.csproj.mustache", "XenServerPowerShell.csproj") - json templdir destdir + List.iter (fun x -> write_file x.filename x.content) all_cmdlets (****************) (* Http actions *) From 7153b19e41f193af3a1a78f192fe9d2d0259e9b7 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Mon, 26 Feb 2024 14:32:26 +0000 Subject: [PATCH 16/99] Exposed GFS2_CAPACITY in the known message types (for the purpose of providing user friendlier messages on the client side). Signed-off-by: Konstantina Chremmou --- ocaml/sdk-gen/csharp/templates/Message2.mustache | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ocaml/sdk-gen/csharp/templates/Message2.mustache b/ocaml/sdk-gen/csharp/templates/Message2.mustache index 0cf943f17a7..4661d815146 100644 --- a/ocaml/sdk-gen/csharp/templates/Message2.mustache +++ b/ocaml/sdk-gen/csharp/templates/Message2.mustache @@ -38,6 +38,7 @@ namespace XenAPI UPDATES_FEATURE_EXPIRING_WARNING, UPDATES_FEATURE_EXPIRING_MAJOR, UPDATES_FEATURE_EXPIRING_CRITICAL, + GFS2_CAPACITY, LEAF_COALESCE_START_MESSAGE, LEAF_COALESCE_COMPLETED, LEAF_COALESCE_FAILED, @@ -62,6 +63,8 @@ namespace XenAPI return MessageType.UPDATES_FEATURE_EXPIRING_MAJOR; case "UPDATES_FEATURE_EXPIRING_CRITICAL": return MessageType.UPDATES_FEATURE_EXPIRING_CRITICAL; + case "GFS2_CAPACITY": + return MessageType.GFS2_CAPACITY; case "LEAF_COALESCE_START_MESSAGE": return MessageType.LEAF_COALESCE_START_MESSAGE; case "LEAF_COALESCE_COMPLETED": From 5ead1d1afa04f38c887a7edfeb19cc2bd90aaeb8 Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Thu, 1 Feb 2024 15:54:48 +0000 Subject: [PATCH 17/99] Store trace_log_dir in XS_EXPORTER_BUGTOOL_ENDPOINT of the observer.conf This can then be used by observer.py, whereas the value 'bugtool' was useless Signed-off-by: Steven Woods --- ocaml/libs/tracing/tracing.ml | 2 ++ ocaml/libs/tracing/tracing.mli | 2 ++ ocaml/xapi/xapi_observer.ml | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ocaml/libs/tracing/tracing.ml b/ocaml/libs/tracing/tracing.ml index e128c510522..05700596116 100644 --- a/ocaml/libs/tracing/tracing.ml +++ b/ocaml/libs/tracing/tracing.ml @@ -722,6 +722,8 @@ module Export = struct let set_trace_log_dir dir = trace_log_dir := dir + let get_trace_log_dir () = !trace_log_dir + let set_max_file_size size = max_file_size := size let set_compress_tracing_files enabled = compress_tracing_files := enabled diff --git a/ocaml/libs/tracing/tracing.mli b/ocaml/libs/tracing/tracing.mli index 8b5940fb3cf..ee30b29f041 100644 --- a/ocaml/libs/tracing/tracing.mli +++ b/ocaml/libs/tracing/tracing.mli @@ -153,6 +153,8 @@ module Export : sig val set_trace_log_dir : string -> unit + val get_trace_log_dir : unit -> string + val set_compress_tracing_files : bool -> unit end diff --git a/ocaml/xapi/xapi_observer.ml b/ocaml/xapi/xapi_observer.ml index 3c5992290cd..5f73beeb073 100644 --- a/ocaml/xapi/xapi_observer.ml +++ b/ocaml/xapi/xapi_observer.ml @@ -235,7 +235,7 @@ module ObserverConfig = struct let rec bugtool_endpoint endpoints = match endpoints with | x :: _ when x = Tracing.bugtool_name -> - Some x + Some (Tracing.Export.Destination.File.get_trace_log_dir ()) | _ :: t -> bugtool_endpoint t | [] -> From 9e0d3da8ae059020e283c5bc02c05e9abec855d5 Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Thu, 8 Feb 2024 14:22:35 +0000 Subject: [PATCH 18/99] Set traceparent trace flag to 01 This enables the sampled flag (https://www.w3.org/TR/trace-context/#sampled-flag). This indicates that we may have recorded trace data. In the future if we use sampling, this flag can be set with more granularity. Signed-off-by: Steven Woods --- ocaml/libs/tracing/tracing.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocaml/libs/tracing/tracing.ml b/ocaml/libs/tracing/tracing.ml index 05700596116..87326fb65cf 100644 --- a/ocaml/libs/tracing/tracing.ml +++ b/ocaml/libs/tracing/tracing.ml @@ -143,7 +143,7 @@ end module SpanContext = struct type t = {trace_id: string; span_id: string} [@@deriving rpcty] - let to_traceparent t = Printf.sprintf "00-%s-%s-00" t.trace_id t.span_id + let to_traceparent t = Printf.sprintf "00-%s-%s-01" t.trace_id t.span_id let of_traceparent traceparent = let elements = String.split_on_char '-' traceparent in From 10d38c6ca23f67465a8a9ff37aea126fb2fdb1f8 Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Mon, 5 Feb 2024 11:41:18 +0000 Subject: [PATCH 19/99] Add service.name attribute as a default observer attribute. This is necessary for the python observers to set their local endpoint value. It also allows filtering traces by component Signed-off-by: Steven Woods --- ocaml/tests/test_observer.ml | 9 ++++++++- ocaml/xapi/xapi_observer.ml | 8 ++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ocaml/tests/test_observer.ml b/ocaml/tests/test_observer.ml index 4da9b324388..69c2f754c93 100644 --- a/ocaml/tests/test_observer.ml +++ b/ocaml/tests/test_observer.ml @@ -82,7 +82,13 @@ module TracerProvider = struct () with _ -> Alcotest.failf "Missing mandatory attribute: %s" x ) - ["xs.pool.uuid"; "xs.host.name"; "xs.host.uuid"; "xs.observer.name"] + [ + "xs.pool.uuid" + ; "xs.host.name" + ; "xs.host.uuid" + ; "xs.observer.name" + ; "service.name" + ] let check_endpoints ~name ~endpoints = let provider = find_provider_exn ~name in @@ -287,6 +293,7 @@ let verify_json_fields_and_values ~json = ; ("xs.observer.name", `String "test-observer") ; ("xs.host.uuid", `String _) ; ("xs.host.name", `String _) + ; ("service.name", `String _) ] ) ; ("annotations", `List _) diff --git a/ocaml/xapi/xapi_observer.ml b/ocaml/xapi/xapi_observer.ml index 5f73beeb073..355cb8dfd0c 100644 --- a/ocaml/xapi/xapi_observer.ml +++ b/ocaml/xapi/xapi_observer.ml @@ -429,7 +429,7 @@ let assert_valid_attributes attributes = ) attributes -let default_attributes ~__context ~host ~name_label = +let default_attributes ~__context ~host ~name_label ~component = let pool = Helpers.get_pool ~__context in let host_label = Db.Host.get_name_label ~__context ~self:host in let host_uuid = Db.Host.get_uuid ~__context ~self:host in @@ -439,12 +439,13 @@ let default_attributes ~__context ~host ~name_label = ; ("xs.host.name", host_label) ; ("xs.host.uuid", host_uuid) ; ("xs.observer.name", name_label) + ; ("service.name", to_string component) ] let register_component ~__context ~self ~host ~component = let name_label = Db.Observer.get_name_label ~__context ~self in let attributes = - default_attributes ~__context ~host ~name_label + default_attributes ~__context ~host ~name_label ~component @ Db.Observer.get_attributes ~__context ~self in let uuid = Db.Observer.get_uuid ~__context ~self in @@ -603,13 +604,12 @@ let set_attributes ~__context ~self ~value = let uuid = Db.Observer.get_uuid ~__context ~self in let host = Helpers.get_localhost ~__context in let name_label = Db.Observer.get_name_label ~__context ~self in - let default_attributes = default_attributes ~__context ~host ~name_label in let observation_fn () = List.iter (fun c -> let module Forwarder = (val get_forwarder c : ObserverInterface) in Forwarder.set_attributes ~__context ~uuid - ~attributes:(default_attributes @ value) + ~attributes:(default_attributes ~__context ~host ~name_label ~component:c @ value) ) (Db.Observer.get_components ~__context ~self |> List.map of_string From 7a0d3e40706d1ff568b539fa3b62459e427fd644 Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Thu, 15 Feb 2024 17:46:49 +0000 Subject: [PATCH 20/99] Add the default_attributes to Dom0ObserverConfig Observers The default_attributes are not added to the DB and so these are missing in the observer.conf. Dom0ObserverConfig currently lacks a single source of truth outside of the DB (and default_attributes is not given to the DB) and so attributes outside of the DB are lost whenever a different setter function is used. This should be reworked in future so that observer.conf is read from/updated instead of regenerated each time. Signed-off-by: Steven Woods --- ocaml/xapi/xapi_observer.ml | 44 ++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/ocaml/xapi/xapi_observer.ml b/ocaml/xapi/xapi_observer.ml index 355cb8dfd0c..eff01624156 100644 --- a/ocaml/xapi/xapi_observer.ml +++ b/ocaml/xapi/xapi_observer.ml @@ -220,6 +220,19 @@ module Xapi_cluster = struct end end +let default_attributes ~__context ~host ~name_label ~component = + let pool = Helpers.get_pool ~__context in + let host_label = Db.Host.get_name_label ~__context ~self:host in + let host_uuid = Db.Host.get_uuid ~__context ~self:host in + let pool_uuid = Db.Pool.get_uuid ~__context ~self:pool in + [ + ("xs.pool.uuid", pool_uuid) + ; ("xs.host.name", host_label) + ; ("xs.host.uuid", host_uuid) + ; ("xs.observer.name", name_label) + ; ("service.name", to_string component) + ] + module ObserverConfig = struct type t = { otel_service_name: string @@ -247,12 +260,18 @@ module ObserverConfig = struct attrs let config_of_observer ~__context ~component ~observer = + (* In the future this should be updated so that the config is read + from and updated instead of being regenerated. *) let endpoints = Db.Observer.get_endpoints ~__context ~self:observer in + let host = Helpers.get_localhost ~__context in + let name_label = Db.Observer.get_name_label ~__context ~self:observer in { - otel_service_name= component + otel_service_name= to_string component ; otel_resource_attributes= attributes_to_W3CBaggage - (Db.Observer.get_attributes ~__context ~self:observer) + (Db.Observer.get_attributes ~__context ~self:observer + @ default_attributes ~__context ~host ~name_label ~component + ) ; xs_exporter_zipkin_endpoints= zipkin_endpoints endpoints ; xs_exporter_bugtool_endpoint= bugtool_endpoint endpoints } @@ -292,8 +311,7 @@ module Dom0ObserverConfig (ObserverComponent : OBSERVER_COMPONENT) : if Db.Observer.get_enabled ~__context ~self:observer then ( let observer_config = ObserverConfig.config_of_observer ~__context - ~component:(to_string ObserverComponent.component) - ~observer + ~component:ObserverComponent.component ~observer in Xapi_stdext_unix.Unixext.mkdir_rec dir_name 0o755 ; let file_name = observer_conf_path_of ~uuid in @@ -429,19 +447,6 @@ let assert_valid_attributes attributes = ) attributes -let default_attributes ~__context ~host ~name_label ~component = - let pool = Helpers.get_pool ~__context in - let host_label = Db.Host.get_name_label ~__context ~self:host in - let host_uuid = Db.Host.get_uuid ~__context ~self:host in - let pool_uuid = Db.Pool.get_uuid ~__context ~self:pool in - [ - ("xs.pool.uuid", pool_uuid) - ; ("xs.host.name", host_label) - ; ("xs.host.uuid", host_uuid) - ; ("xs.observer.name", name_label) - ; ("service.name", to_string component) - ] - let register_component ~__context ~self ~host ~component = let name_label = Db.Observer.get_name_label ~__context ~self in let attributes = @@ -609,7 +614,10 @@ let set_attributes ~__context ~self ~value = (fun c -> let module Forwarder = (val get_forwarder c : ObserverInterface) in Forwarder.set_attributes ~__context ~uuid - ~attributes:(default_attributes ~__context ~host ~name_label ~component:c @ value) + ~attributes: + (default_attributes ~__context ~host ~name_label ~component:c + @ value + ) ) (Db.Observer.get_components ~__context ~self |> List.map of_string From 98b55d6af5126de8b7f01ee1a60d08e6782e3808 Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 27 Feb 2024 10:30:00 +0800 Subject: [PATCH 21/99] Revert "CP-45572: Print update guidance in xe host-apply-updates" This reverts commit 6668d5f5609eb75a50b7dd4138412e6bf9a732bd. Signed-off-by: Ming Lu --- ocaml/xapi-cli-protocol/cli_protocol.ml | 10 +------ ocaml/xapi-cli-server/cli_frontend.ml | 2 +- ocaml/xapi-cli-server/cli_operations.ml | 35 ++++++++----------------- ocaml/xe-cli/newcli.ml | 17 ------------ 4 files changed, 13 insertions(+), 51 deletions(-) diff --git a/ocaml/xapi-cli-protocol/cli_protocol.ml b/ocaml/xapi-cli-protocol/cli_protocol.ml index 261bc11b187..38d0f76871c 100644 --- a/ocaml/xapi-cli-protocol/cli_protocol.ml +++ b/ocaml/xapi-cli-protocol/cli_protocol.ml @@ -35,7 +35,6 @@ type command = | Load of string (* filename *) | HttpGet of string * string (* filename * path *) | PrintHttpGetJson of string (* path *) - | PrintUpdateGuidance of string (* path *) | HttpPut of string * string (* filename * path *) | HttpConnect of string (* path *) | Prompt (* request the user enter some text *) @@ -70,8 +69,6 @@ let string_of_command = function "HttpGet " ^ path ^ " -> " ^ filename | PrintHttpGetJson path -> "PrintHttpGetJson " ^ path ^ " -> stdout" - | PrintUpdateGuidance path -> - "PrintUpdateGuidance " ^ path ^ " -> stdout" | HttpPut (filename, path) -> "HttpPut " ^ path ^ " -> " ^ filename | HttpConnect path -> @@ -161,7 +158,7 @@ let unmarshal_list pos f = (*****************************************************************************) (* Marshal/Unmarshal higher-level messages *) -(* Highest command id: 19 *) +(* Highest command id: 18 *) let marshal_command = function | Print x -> @@ -174,8 +171,6 @@ let marshal_command = function marshal_int 12 ^ marshal_string a ^ marshal_string b | PrintHttpGetJson a -> marshal_int 18 ^ marshal_string a - | PrintUpdateGuidance a -> - marshal_int 19 ^ marshal_string a | HttpPut (a, b) -> marshal_int 13 ^ marshal_string a ^ marshal_string b | HttpConnect a -> @@ -229,9 +224,6 @@ let unmarshal_command pos = | 18 -> let a, pos = unmarshal_string pos in (PrintHttpGetJson a, pos) - | 19 -> - let a, pos = unmarshal_string pos in - (PrintUpdateGuidance a, pos) | n -> raise (Unknown_tag ("command", n)) diff --git a/ocaml/xapi-cli-server/cli_frontend.ml b/ocaml/xapi-cli-server/cli_frontend.ml index 55d884db9d8..09ce4f79dfb 100644 --- a/ocaml/xapi-cli-server/cli_frontend.ml +++ b/ocaml/xapi-cli-server/cli_frontend.ml @@ -1032,7 +1032,7 @@ let rec cmdtable_data : (string * cmd_spec) list = reqd= ["hash"] ; optn= [] ; help= "Apply updates from enabled repository on specified host." - ; implementation= With_fd Cli_operations.host_apply_updates + ; implementation= No_fd Cli_operations.host_apply_updates ; flags= [Host_selectors] } ) diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index 6766a5161f0..c3876aff009 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -7641,32 +7641,19 @@ let print_avail_updates ~rpc ~session_id ~fd ~host = PrintHttpGetJson (get_avail_updates_uri ~session_id ~task ~host) ) -let print_update_guidance ~rpc ~session_id ~fd ~host = - command_in_task ~rpc ~session_id ~fd ~obj:host - ~label:"Print update guidance for host" ~quiet_on_success:true - (fun session_id task host -> - PrintUpdateGuidance (get_avail_updates_uri ~session_id ~task ~host) - ) - -let host_apply_updates fd printer rpc session_id params = +let host_apply_updates _printer rpc session_id params = let hash = List.assoc "hash" params in - do_host_op rpc session_id ~multiple:false - (fun _ host -> - let host = host.getref () in - printer (Cli_printer.PMsg "Guidance of updates:") ; - print_update_guidance ~rpc ~session_id ~fd ~host ; - printer (Cli_printer.PMsg "Applying updates ...") ; - match Client.Host.apply_updates ~rpc ~session_id ~self:host ~hash with - | [] -> - printer (Cli_printer.PMsg "Updated.") - | warnings -> - printer (Cli_printer.PMsg "Updated with warnings:") ; - List.iter - (fun l -> printer (Cli_printer.PMsg (String.concat "; " l))) - warnings + ignore + (do_host_op rpc session_id ~multiple:false + (fun _ host -> + let host = host.getref () in + Client.Host.apply_updates ~rpc ~session_id ~self:host ~hash + |> List.iter (fun l -> + _printer (Cli_printer.PMsg (String.concat "; " l)) + ) + ) + params ["hash"] ) - params ["hash"] - |> ignore let host_updates_show_available fd _printer rpc session_id params = do_host_op rpc session_id ~multiple:false diff --git a/ocaml/xe-cli/newcli.ml b/ocaml/xe-cli/newcli.ml index 7c07c199512..7dddf2ee359 100644 --- a/ocaml/xe-cli/newcli.ml +++ b/ocaml/xe-cli/newcli.ml @@ -773,23 +773,6 @@ let main_loop ifd ofd permitted_filenames = |> print_endline ; flush stdout ) - | Command (PrintUpdateGuidance url) -> - do_http_get ofd url exit_code (fun ic -> - while input_line ic <> "\r" do - () - done ; - Yojson.Basic.from_channel ic |> Yojson.Basic.Util.member "hosts" - |> function - | `List [] -> - raise (ClientSideError "No host data returned") - | `List (host :: _) -> - Yojson.Basic.Util.member "guidance" host - |> Yojson.Basic.pretty_to_string - |> print_endline ; - flush stdout - | _ -> - raise (ClientSideError "Unknown data format") - ) | Command Prompt -> let data = input_line stdin in marshal ofd (Blob (Chunk (Int32.of_int (String.length data)))) ; From f0b43f678cbdb32d1f0f11f75ad84200340158de Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Mon, 26 Feb 2024 20:45:15 +0800 Subject: [PATCH 22/99] CA-389206: Add comment for changing the CLI interface Signed-off-by: Ming Lu --- ocaml/xapi-cli-protocol/cli_protocol.ml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ocaml/xapi-cli-protocol/cli_protocol.ml b/ocaml/xapi-cli-protocol/cli_protocol.ml index 38d0f76871c..1fc8d95d11d 100644 --- a/ocaml/xapi-cli-protocol/cli_protocol.ml +++ b/ocaml/xapi-cli-protocol/cli_protocol.ml @@ -26,6 +26,13 @@ let minor = 2 a totally different kind of server (eg a standard HTTP server) *) let prefix = "XenSource thin CLI protocol" +(* Be careful to add/remove/modify the commands. The CLI interface is expected + to be extreme stable. Please keep following in mind when doing the changes: + 1. backwards compatibility support. E.g. a very old CLI client may need to + be supported still, + 2. a new command should be one for general purpose only rather than for a + specific usage. *) + (** Command sent by the server to the client. If the command is "Save" then the server waits for "OK" from the client and then streams a list of data chunks to the client. *) From 5d41bfe30fdb2c251bf2d7ae6fe721be63cfbc67 Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 27 Feb 2024 19:10:33 +0800 Subject: [PATCH 23/99] Revert "Fixup: Don't change behaviour in download_file when filename is empty" This reverts commit 397d47aacb55fc7eac292faac1f785e5e5872fa2. Signed-off-by: Ming Lu --- ocaml/xapi-cli-server/cli_operations.ml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index c3876aff009..e426a53566a 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -5500,10 +5500,12 @@ let download_file rpc session_id task fd filename uri label = response := unmarshal fd done ; let ok = - match !response with - | Response OK -> + match (!response, filename <> "") with + | Response OK, true -> true - | Response Failed -> + | Response OK, false -> + false + | Response Failed, _ -> (* Need to check whether the thin cli managed to contact the server or not. If not, we need to mark the task as failed *) if Client.Task.get_progress ~rpc ~session_id ~self:task < 0.0 then @@ -5515,8 +5517,7 @@ let download_file rpc session_id task fd filename uri label = wait_for_task_complete rpc session_id task ; (* Check the server status -- even if the client thinks it's ok, we need to check that the server does too. *) - let quiet_on_success = if filename = "" then true else false in - check_task_status ~rpc ~session_id ~task ~fd ~label ~ok ~quiet_on_success () + check_task_status ~rpc ~session_id ~task ~fd ~label ~ok () let download_file_with_task fd rpc session_id filename uri query label task_name = From 6d4c35e4b11694cd35bbc66e28b40693c135fc77 Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 27 Feb 2024 19:12:04 +0800 Subject: [PATCH 24/99] Revert "CP-45572,CP-45573: Refine to use 'command_in_task' more" This reverts commit 533a000b19dfd0e8ad1807ee3e8e0788075ef2b0. Signed-off-by: Ming Lu --- ocaml/xapi-cli-server/cli_operations.ml | 134 +++++++++++++++++------- 1 file changed, 94 insertions(+), 40 deletions(-) diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index e426a53566a..bfe362cc2d1 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -5681,17 +5681,24 @@ let vm_import fd _printer rpc session_id params = in marshal fd (Command (Print (String.concat "," uuids))) -let command_in_task ~rpc ~session_id ~fd ~obj ~label ~quiet_on_success f = - let task = +let blob_get fd _printer rpc session_id params = + let blob_uuid = List.assoc "uuid" params in + let blob_ref = Client.Blob.get_by_uuid ~rpc ~session_id ~uuid:blob_uuid in + let filename = List.assoc "filename" params in + let blobtask = Client.Task.create ~rpc ~session_id - ~label:(Printf.sprintf "%s (ref=%s)" label (Ref.string_of obj)) + ~label:(Printf.sprintf "Obtaining blob, ref=%s" (Ref.string_of blob_ref)) ~description:"" in - Client.Task.set_progress ~rpc ~session_id ~self:task ~value:(-1.0) ; - let command = f session_id task obj in + Client.Task.set_progress ~rpc ~session_id ~self:blobtask ~value:(-1.0) ; + let bloburi = + Printf.sprintf "%s?session_id=%s&task_id=%s&ref=%s" Constants.blob_uri + (Ref.string_of session_id) (Ref.string_of blobtask) + (Ref.string_of blob_ref) + in finally (fun () -> - marshal fd (Command command) ; + marshal fd (Command (HttpGet (filename, bloburi))) ; let response = ref (Response Wait) in while !response = Response Wait do response := unmarshal fd @@ -5701,48 +5708,61 @@ let command_in_task ~rpc ~session_id ~fd ~obj ~label ~quiet_on_success f = | Response OK -> true | Response Failed -> - (* Need to check whether the thin cli managed to contact the server - * or not. If not, we need to mark the task as failed. - *) - if Client.Task.get_progress ~rpc ~session_id ~self:task < 0.0 then - Client.Task.set_status ~rpc ~session_id ~self:task ~value:`failure ; + if Client.Task.get_progress ~rpc ~session_id ~self:blobtask < 0.0 + then + Client.Task.set_status ~rpc ~session_id ~self:blobtask + ~value:`failure ; false | _ -> false in - wait_for_task_complete rpc session_id task ; - check_task_status ~rpc ~session_id ~task ~fd ~label ~ok ~quiet_on_success - () + wait_for_task_complete rpc session_id blobtask ; + check_task_status ~rpc ~session_id ~task:blobtask ~fd ~label:"Blob get" + ~ok () ) - (fun () -> Client.Task.destroy ~rpc ~session_id ~self:task) - -let blob_uri ~session_id ~task ~blob = - let query = - [ - ("session_id", [Ref.string_of session_id]) - ; ("task_id", [Ref.string_of task]) - ; ("ref", [Ref.string_of blob]) - ] - in - Uri.make ~path:Constants.blob_uri ~query () |> Uri.to_string - -let blob_get fd _printer rpc session_id params = - let blob_uuid = List.assoc "uuid" params in - let blob_ref = Client.Blob.get_by_uuid ~rpc ~session_id ~uuid:blob_uuid in - let filename = List.assoc "filename" params in - command_in_task ~rpc ~session_id ~fd ~obj:blob_ref ~label:"GET blob" - ~quiet_on_success:false (fun session_id task blob -> - HttpGet (filename, blob_uri ~session_id ~task ~blob) - ) + (fun () -> Client.Task.destroy ~rpc ~session_id ~self:blobtask) let blob_put fd _printer rpc session_id params = let blob_uuid = List.assoc "uuid" params in let blob_ref = Client.Blob.get_by_uuid ~rpc ~session_id ~uuid:blob_uuid in let filename = List.assoc "filename" params in - command_in_task ~rpc ~session_id ~fd ~obj:blob_ref ~label:"PUT blob" - ~quiet_on_success:false (fun session_id task blob -> - HttpPut (filename, blob_uri ~session_id ~task ~blob) - ) + let blobtask = + Client.Task.create ~rpc ~session_id + ~label:(Printf.sprintf "Blob PUT, ref=%s" (Ref.string_of blob_ref)) + ~description:"" + in + Client.Task.set_progress ~rpc ~session_id ~self:blobtask ~value:(-1.0) ; + let bloburi = + Printf.sprintf "%s?session_id=%s&task_id=%s&ref=%s" Constants.blob_uri + (Ref.string_of session_id) (Ref.string_of blobtask) + (Ref.string_of blob_ref) + in + finally + (fun () -> + marshal fd (Command (HttpPut (filename, bloburi))) ; + let response = ref (Response Wait) in + while !response = Response Wait do + response := unmarshal fd + done ; + let ok = + match !response with + | Response OK -> + true + | Response Failed -> + if Client.Task.get_progress ~rpc ~session_id ~self:blobtask < 0.0 + then + Client.Task.set_status ~rpc ~session_id ~self:blobtask + ~value:`failure ; + false + | _ -> + false + in + wait_for_task_complete rpc session_id blobtask ; + (* if the client thinks it's ok, check that the server does too *) + check_task_status ~rpc ~session_id ~task:blobtask ~fd ~label:"Blob put" + ~ok () + ) + (fun () -> Client.Task.destroy ~rpc ~session_id ~self:blobtask) let blob_create printer rpc session_id params = let name = List.assoc "name" params in @@ -7635,9 +7655,43 @@ let get_avail_updates_uri ~session_id ~task ~host = in Uri.make ~path:Constants.get_updates_uri ~query () |> Uri.to_string +let command_in_task ~rpc ~session_id ~fd ~host ~label f = + let task = + Client.Task.create ~rpc ~session_id + ~label:(Printf.sprintf "%s for host (ref=%s)" label (Ref.string_of host)) + ~description:"" + in + Client.Task.set_progress ~rpc ~session_id ~self:task ~value:(-1.0) ; + let command = f session_id task host in + finally + (fun () -> + marshal fd (Command command) ; + let response = ref (Response Wait) in + while !response = Response Wait do + response := unmarshal fd + done ; + let ok = + match !response with + | Response OK -> + true + | Response Failed -> + (* Need to check whether the thin cli managed to contact the server + * or not. If not, we need to mark the task as failed. + *) + if Client.Task.get_progress ~rpc ~session_id ~self:task < 0.0 then + Client.Task.set_status ~rpc ~session_id ~self:task ~value:`failure ; + false + | _ -> + false + in + wait_for_task_complete rpc session_id task ; + check_task_status ~rpc ~session_id ~task ~fd ~label ~ok + ~quiet_on_success:true () + ) + (fun () -> Client.Task.destroy ~rpc ~session_id ~self:task) + let print_avail_updates ~rpc ~session_id ~fd ~host = - command_in_task ~rpc ~session_id ~fd ~obj:host - ~label:"Print available updates for host" ~quiet_on_success:true + command_in_task ~rpc ~session_id ~fd ~host ~label:"Print available updates" (fun session_id task host -> PrintHttpGetJson (get_avail_updates_uri ~session_id ~task ~host) ) From 6d48842095e4e3a7c26dfd7cb82a1615d28c7953 Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 27 Feb 2024 19:12:42 +0800 Subject: [PATCH 25/99] Revert "CP-45573: Add 'xe host-updates-show-available' CLI" This reverts commit 8ca4b43a9b6fbadb8f2832e05366de33fc859f6d. Signed-off-by: Ming Lu --- ocaml/xapi-cli-protocol/cli_protocol.ml | 10 +--- ocaml/xapi-cli-server/cli_frontend.ml | 9 --- ocaml/xapi-cli-server/cli_operations.ml | 73 ++----------------------- ocaml/xe-cli/newcli.ml | 10 ---- 4 files changed, 6 insertions(+), 96 deletions(-) diff --git a/ocaml/xapi-cli-protocol/cli_protocol.ml b/ocaml/xapi-cli-protocol/cli_protocol.ml index 1fc8d95d11d..bf58263eabd 100644 --- a/ocaml/xapi-cli-protocol/cli_protocol.ml +++ b/ocaml/xapi-cli-protocol/cli_protocol.ml @@ -41,7 +41,6 @@ type command = | Debug of string (* debug message to optionally display *) | Load of string (* filename *) | HttpGet of string * string (* filename * path *) - | PrintHttpGetJson of string (* path *) | HttpPut of string * string (* filename * path *) | HttpConnect of string (* path *) | Prompt (* request the user enter some text *) @@ -74,8 +73,6 @@ let string_of_command = function "Load " ^ x | HttpGet (filename, path) -> "HttpGet " ^ path ^ " -> " ^ filename - | PrintHttpGetJson path -> - "PrintHttpGetJson " ^ path ^ " -> stdout" | HttpPut (filename, path) -> "HttpPut " ^ path ^ " -> " ^ filename | HttpConnect path -> @@ -165,7 +162,7 @@ let unmarshal_list pos f = (*****************************************************************************) (* Marshal/Unmarshal higher-level messages *) -(* Highest command id: 18 *) +(* Highest command id: 17 *) let marshal_command = function | Print x -> @@ -176,8 +173,6 @@ let marshal_command = function marshal_int 1 ^ marshal_string x | HttpGet (a, b) -> marshal_int 12 ^ marshal_string a ^ marshal_string b - | PrintHttpGetJson a -> - marshal_int 18 ^ marshal_string a | HttpPut (a, b) -> marshal_int 13 ^ marshal_string a ^ marshal_string b | HttpConnect a -> @@ -228,9 +223,6 @@ let unmarshal_command pos = | 16 -> let body, pos = unmarshal_string pos in (PrintStderr body, pos) - | 18 -> - let a, pos = unmarshal_string pos in - (PrintHttpGetJson a, pos) | n -> raise (Unknown_tag ("command", n)) diff --git a/ocaml/xapi-cli-server/cli_frontend.ml b/ocaml/xapi-cli-server/cli_frontend.ml index 09ce4f79dfb..f8aa043eb5a 100644 --- a/ocaml/xapi-cli-server/cli_frontend.ml +++ b/ocaml/xapi-cli-server/cli_frontend.ml @@ -1047,15 +1047,6 @@ let rec cmdtable_data : (string * cmd_spec) list = ; flags= [Neverforward] } ) - ; ( "host-updates-show-available" - , { - reqd= [] - ; optn= [] - ; help= "Show available updates for a specified host." - ; implementation= With_fd Cli_operations.host_updates_show_available - ; flags= [Host_selectors] - } - ) ; ( "patch-upload" , { reqd= ["file-name"] diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index bfe362cc2d1..b93276fba98 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -5462,14 +5462,11 @@ let wait_for_task_complete rpc session_id task_id = Thread.delay 1.0 done -let check_task_status ?(quiet_on_success = false) ~rpc ~session_id ~task ~fd - ~label ~ok () = +let check_task_status ~rpc ~session_id ~task ~fd ~label ~ok = (* if the client thinks it's ok, check that the server does too *) match Client.Task.get_status ~rpc ~session_id ~self:task with - | `success when ok && not quiet_on_success -> + | `success when ok -> marshal fd (Command (Print (Printf.sprintf "%s succeeded" label))) - | `success when ok && quiet_on_success -> - () | `success -> marshal fd (Command @@ -5517,7 +5514,7 @@ let download_file rpc session_id task fd filename uri label = wait_for_task_complete rpc session_id task ; (* Check the server status -- even if the client thinks it's ok, we need to check that the server does too. *) - check_task_status ~rpc ~session_id ~task ~fd ~label ~ok () + check_task_status ~rpc ~session_id ~task ~fd ~label ~ok let download_file_with_task fd rpc session_id filename uri query label task_name = @@ -5718,7 +5715,7 @@ let blob_get fd _printer rpc session_id params = in wait_for_task_complete rpc session_id blobtask ; check_task_status ~rpc ~session_id ~task:blobtask ~fd ~label:"Blob get" - ~ok () + ~ok ) (fun () -> Client.Task.destroy ~rpc ~session_id ~self:blobtask) @@ -5760,7 +5757,7 @@ let blob_put fd _printer rpc session_id params = wait_for_task_complete rpc session_id blobtask ; (* if the client thinks it's ok, check that the server does too *) check_task_status ~rpc ~session_id ~task:blobtask ~fd ~label:"Blob put" - ~ok () + ~ok ) (fun () -> Client.Task.destroy ~rpc ~session_id ~self:blobtask) @@ -7645,57 +7642,6 @@ let update_resync_host _printer rpc session_id params = let host = Client.Host.get_by_uuid ~rpc ~session_id ~uuid in Client.Pool_update.resync_host ~rpc ~session_id ~host -let get_avail_updates_uri ~session_id ~task ~host = - let query = - [ - ("session_id", [Ref.string_of session_id]) - ; ("task_id", [Ref.string_of task]) - ; ("host_refs", [Ref.string_of host]) - ] - in - Uri.make ~path:Constants.get_updates_uri ~query () |> Uri.to_string - -let command_in_task ~rpc ~session_id ~fd ~host ~label f = - let task = - Client.Task.create ~rpc ~session_id - ~label:(Printf.sprintf "%s for host (ref=%s)" label (Ref.string_of host)) - ~description:"" - in - Client.Task.set_progress ~rpc ~session_id ~self:task ~value:(-1.0) ; - let command = f session_id task host in - finally - (fun () -> - marshal fd (Command command) ; - let response = ref (Response Wait) in - while !response = Response Wait do - response := unmarshal fd - done ; - let ok = - match !response with - | Response OK -> - true - | Response Failed -> - (* Need to check whether the thin cli managed to contact the server - * or not. If not, we need to mark the task as failed. - *) - if Client.Task.get_progress ~rpc ~session_id ~self:task < 0.0 then - Client.Task.set_status ~rpc ~session_id ~self:task ~value:`failure ; - false - | _ -> - false - in - wait_for_task_complete rpc session_id task ; - check_task_status ~rpc ~session_id ~task ~fd ~label ~ok - ~quiet_on_success:true () - ) - (fun () -> Client.Task.destroy ~rpc ~session_id ~self:task) - -let print_avail_updates ~rpc ~session_id ~fd ~host = - command_in_task ~rpc ~session_id ~fd ~host ~label:"Print available updates" - (fun session_id task host -> - PrintHttpGetJson (get_avail_updates_uri ~session_id ~task ~host) - ) - let host_apply_updates _printer rpc session_id params = let hash = List.assoc "hash" params in ignore @@ -7710,15 +7656,6 @@ let host_apply_updates _printer rpc session_id params = params ["hash"] ) -let host_updates_show_available fd _printer rpc session_id params = - do_host_op rpc session_id ~multiple:false - (fun _ host -> - let host = host.getref () in - print_avail_updates ~rpc ~session_id ~fd ~host - ) - params [] - |> ignore - module SDN_controller = struct let introduce printer rpc session_id params = let port = diff --git a/ocaml/xe-cli/newcli.ml b/ocaml/xe-cli/newcli.ml index 7dddf2ee359..634ee4d4ebf 100644 --- a/ocaml/xe-cli/newcli.ml +++ b/ocaml/xe-cli/newcli.ml @@ -763,16 +763,6 @@ let main_loop ifd ofd permitted_filenames = (fun () -> copy_with_heartbeat ic file_ch heartbeat_fun) (fun () -> try close_out file_ch with _ -> ()) ) - | Command (PrintHttpGetJson url) -> - do_http_get ofd url exit_code (fun ic -> - while input_line ic <> "\r" do - () - done ; - Yojson.Basic.from_channel ic - |> Yojson.Basic.pretty_to_string - |> print_endline ; - flush stdout - ) | Command Prompt -> let data = input_line stdin in marshal ofd (Blob (Chunk (Int32.of_int (String.length data)))) ; From c84f43375acb528390858b4d7051a238d901c2bf Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 27 Feb 2024 19:12:52 +0800 Subject: [PATCH 26/99] Revert "CP-45572,CP-45573: Split 'do_http_get' function out" This reverts commit a53e54deb083930c2159bd50fe8a5fbd35e4ca55. Signed-off-by: Ming Lu --- ocaml/xe-cli/newcli.ml | 112 +++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 54 deletions(-) diff --git a/ocaml/xe-cli/newcli.ml b/ocaml/xe-cli/newcli.ml index 634ee4d4ebf..d197b849a94 100644 --- a/ocaml/xe-cli/newcli.ml +++ b/ocaml/xe-cli/newcli.ml @@ -435,38 +435,6 @@ let assert_filename_permitted ?(permit_cwd = false) permitted_filenames filename | _ -> () -let do_http_get ofd url exit_code f = - try - let rec doit url = - let server, path = parse_url url in - debug "Opening connection to server '%s' path '%s'\n%!" server path ; - with_open_tcp server @@ fun (ic, oc) -> - Printf.fprintf oc "GET %s HTTP/1.0\r\n\r\n" path ; - flush oc ; - (* Get the result header immediately *) - let resultline = input_line ic in - debug "Got %s\n%!" resultline ; - match http_response_code resultline with - | 200 -> - f ic ; marshal ofd (Response OK) - | 302 -> - let headers = read_rest_of_headers ic in - let newloc = List.assoc "location" headers in - (* see above about Unixfd.with_connection *) - close_in_noerr ic ; close_out_noerr oc ; doit newloc - | _ -> - failwith "Unhandled response code" - in - doit url - with - | ClientSideError msg -> - marshal ofd (Response Failed) ; - Printf.fprintf stderr "Operation failed. Error: %s\n" msg ; - exit_code := Some 1 - | e -> - debug "HTTP GET failure: %s\n%!" (Printexc.to_string e) ; - marshal ofd (Response Failed) - let main_loop ifd ofd permitted_filenames = (* Intially exchange version information *) let major', minor' = @@ -741,28 +709,64 @@ let main_loop ifd ofd permitted_filenames = the normal communication channel *) marshal ofd (Response Failed) ) - | Command (HttpGet (filename, url)) -> - do_http_get ofd url exit_code (fun ic -> - let file_ch = - if filename = "" then - Unix.out_channel_of_descr (Unix.dup Unix.stdout) - else ( - assert_filename_permitted ~permit_cwd:true permitted_filenames - filename ; - try - open_out_gen - [Open_wronly; Open_creat; Open_excl] - 0o600 filename - with e -> raise (ClientSideError (Printexc.to_string e)) - ) - in - while input_line ic <> "\r" do - () - done ; - Pervasiveext.finally - (fun () -> copy_with_heartbeat ic file_ch heartbeat_fun) - (fun () -> try close_out file_ch with _ -> ()) - ) + | Command (HttpGet (filename, url)) -> ( + try + let rec doit url = + let server, path = parse_url url in + debug "Opening connection to server '%s' path '%s'\n%!" server path ; + with_open_tcp server @@ fun (ic, oc) -> + Printf.fprintf oc "GET %s HTTP/1.0\r\n\r\n" path ; + flush oc ; + (* Get the result header immediately *) + let resultline = input_line ic in + debug "Got %s\n%!" resultline ; + match http_response_code resultline with + | 200 -> + let file_ch = + if filename = "" then + Unix.out_channel_of_descr (Unix.dup Unix.stdout) + else ( + assert_filename_permitted ~permit_cwd:true permitted_filenames + filename ; + try + open_out_gen + [Open_wronly; Open_creat; Open_excl] + 0o600 filename + with e -> raise (ClientSideError (Printexc.to_string e)) + ) + in + while input_line ic <> "\r" do + () + done ; + Pervasiveext.finally + (fun () -> + copy_with_heartbeat ic file_ch heartbeat_fun ; + marshal ofd (Response OK) + ) + (fun () -> try close_out file_ch with _ -> ()) + | 302 -> + let headers = read_rest_of_headers ic in + let newloc = List.assoc "location" headers in + (* see above about Unixfd.with_connection *) + close_in_noerr ic ; close_out_noerr oc ; doit newloc + | _ -> + failwith "Unhandled response code" + in + doit url + with + | ClientSideError msg -> + marshal ofd (Response Failed) ; + Printf.fprintf stderr "Operation failed. Error: %s\n" msg ; + exit_code := Some 1 + | e -> ( + match e with + | Filename_not_permitted _ -> + raise e + | _ -> + debug "HttpGet failure: %s\n%!" (Printexc.to_string e) ; + marshal ofd (Response Failed) + ) + ) | Command Prompt -> let data = input_line stdin in marshal ofd (Blob (Chunk (Int32.of_int (String.length data)))) ; From 7741b973af109bad90abded0a6779a03b5128964 Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 27 Feb 2024 19:13:05 +0800 Subject: [PATCH 27/99] Revert "CP-45572,CP-45573: Split 'check_task_status' function out" This reverts commit 54039f33ab8039bf56a66f0b9edf8aac3cae702a. Signed-off-by: Ming Lu --- ocaml/xapi-cli-server/cli_operations.ml | 118 ++++++++++++++++-------- 1 file changed, 80 insertions(+), 38 deletions(-) diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index b93276fba98..bc0d9ea30bc 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -5462,34 +5462,6 @@ let wait_for_task_complete rpc session_id task_id = Thread.delay 1.0 done -let check_task_status ~rpc ~session_id ~task ~fd ~label ~ok = - (* if the client thinks it's ok, check that the server does too *) - match Client.Task.get_status ~rpc ~session_id ~self:task with - | `success when ok -> - marshal fd (Command (Print (Printf.sprintf "%s succeeded" label))) - | `success -> - marshal fd - (Command - (PrintStderr (Printf.sprintf "%s failed, unknown error.\n" label)) - ) ; - raise (ExitWithError 1) - | `failure -> - let result = Client.Task.get_error_info ~rpc ~session_id ~self:task in - if result = [] then - marshal fd - (Command - (PrintStderr (Printf.sprintf "%s failed, unknown error\n" label)) - ) - else - raise (Api_errors.Server_error (List.hd result, List.tl result)) - | `cancelled -> - marshal fd (Command (PrintStderr (Printf.sprintf "%s cancelled\n" label))) ; - raise (ExitWithError 1) - | _ -> - marshal fd (Command (PrintStderr "Internal error\n")) ; - (* should never happen *) - raise (ExitWithError 1) - let download_file rpc session_id task fd filename uri label = marshal fd (Command (HttpGet (filename, uri))) ; let response = ref (Response Wait) in @@ -5497,12 +5469,10 @@ let download_file rpc session_id task fd filename uri label = response := unmarshal fd done ; let ok = - match (!response, filename <> "") with - | Response OK, true -> + match !response with + | Response OK -> true - | Response OK, false -> - false - | Response Failed, _ -> + | Response Failed -> (* Need to check whether the thin cli managed to contact the server or not. If not, we need to mark the task as failed *) if Client.Task.get_progress ~rpc ~session_id ~self:task < 0.0 then @@ -5514,7 +5484,34 @@ let download_file rpc session_id task fd filename uri label = wait_for_task_complete rpc session_id task ; (* Check the server status -- even if the client thinks it's ok, we need to check that the server does too. *) - check_task_status ~rpc ~session_id ~task ~fd ~label ~ok + match Client.Task.get_status ~rpc ~session_id ~self:task with + | `success -> + if ok then ( + if filename <> "" then + marshal fd (Command (Print (Printf.sprintf "%s succeeded" label))) + ) else ( + marshal fd + (Command + (PrintStderr (Printf.sprintf "%s failed, unknown error.\n" label)) + ) ; + raise (ExitWithError 1) + ) + | `failure -> + let result = Client.Task.get_error_info ~rpc ~session_id ~self:task in + if result = [] then + marshal fd + (Command + (PrintStderr (Printf.sprintf "%s failed, unknown error\n" label)) + ) + else + raise (Api_errors.Server_error (List.hd result, List.tl result)) + | `cancelled -> + marshal fd (Command (PrintStderr (Printf.sprintf "%s cancelled\n" label))) ; + raise (ExitWithError 1) + | _ -> + marshal fd (Command (PrintStderr "Internal error\n")) ; + (* should never happen *) + raise (ExitWithError 1) let download_file_with_task fd rpc session_id filename uri query label task_name = @@ -5714,8 +5711,31 @@ let blob_get fd _printer rpc session_id params = false in wait_for_task_complete rpc session_id blobtask ; - check_task_status ~rpc ~session_id ~task:blobtask ~fd ~label:"Blob get" - ~ok + (* if the client thinks it's ok, check that the server does too *) + match Client.Task.get_status ~rpc ~session_id ~self:blobtask with + | `success -> + if ok then + marshal fd (Command (Print "Blob get succeeded")) + else ( + marshal fd + (Command (PrintStderr "Blob get failed, unknown error.\n")) ; + raise (ExitWithError 1) + ) + | `failure -> + let result = + Client.Task.get_error_info ~rpc ~session_id ~self:blobtask + in + if result = [] then + marshal fd (Command (PrintStderr "Blob get failed, unknown error\n")) + else + raise (Api_errors.Server_error (List.hd result, List.tl result)) + | `cancelled -> + marshal fd (Command (PrintStderr "Blob get cancelled\n")) ; + raise (ExitWithError 1) + | _ -> + marshal fd (Command (PrintStderr "Internal error\n")) ; + (* should never happen *) + raise (ExitWithError 1) ) (fun () -> Client.Task.destroy ~rpc ~session_id ~self:blobtask) @@ -5756,8 +5776,30 @@ let blob_put fd _printer rpc session_id params = in wait_for_task_complete rpc session_id blobtask ; (* if the client thinks it's ok, check that the server does too *) - check_task_status ~rpc ~session_id ~task:blobtask ~fd ~label:"Blob put" - ~ok + match Client.Task.get_status ~rpc ~session_id ~self:blobtask with + | `success -> + if ok then + marshal fd (Command (Print "Blob put succeeded")) + else ( + marshal fd + (Command (PrintStderr "Blob put failed, unknown error.\n")) ; + raise (ExitWithError 1) + ) + | `failure -> + let result = + Client.Task.get_error_info ~rpc ~session_id ~self:blobtask + in + if result = [] then + marshal fd (Command (PrintStderr "Blob put failed, unknown error\n")) + else + raise (Api_errors.Server_error (List.hd result, List.tl result)) + | `cancelled -> + marshal fd (Command (PrintStderr "Blob put cancelled\n")) ; + raise (ExitWithError 1) + | _ -> + marshal fd (Command (PrintStderr "Internal error\n")) ; + (* should never happen *) + raise (ExitWithError 1) ) (fun () -> Client.Task.destroy ~rpc ~session_id ~self:blobtask) From cfc5b6334ce140850626829a3baac0d3fb510de1 Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 27 Feb 2024 19:14:20 +0800 Subject: [PATCH 28/99] Revert "Decrease the usage count of List.hd from 317 to 315" This reverts commit db91ddf593d7b0b368535874ef45605492d04e8c. Signed-off-by: Ming Lu --- quality-gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quality-gate.sh b/quality-gate.sh index d33edacff02..224e852aa32 100755 --- a/quality-gate.sh +++ b/quality-gate.sh @@ -3,7 +3,7 @@ set -e list-hd () { - N=316 + N=318 LIST_HD=$(git grep -r --count 'List.hd' -- **/*.ml | cut -d ':' -f 2 | paste -sd+ - | bc) if [ "$LIST_HD" -eq "$N" ]; then echo "OK counted $LIST_HD List.hd usages" From b5a76a30ce2fea9a9e91a119c1bd364a585c6d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Mon, 26 Feb 2024 17:15:04 +0000 Subject: [PATCH 29/99] ci: add differential ShellCheck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should only report errors on lines that are changed in a PR, and not block merges for pre-existing bugs. Signed-off-by: Edwin Török --- .github/workflows/main.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cadf84c35c4..7f52439cd69 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,28 @@ concurrency: # On new push, cancel old workflows from the same PR, branch or ta cancel-in-progress: true jobs: + # https://www.shellcheck.net/wiki/GitHub-Actions + # https://github.com/redhat-plumbers-in-action/differential-shellcheck?tab=readme-ov-file#usage + shell-test: + name: Differential ShellCheck + runs-on: ubuntu-latest + + permissions: + security-events: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + +# If needed severity levels can be controlled here +# severity: warning + - name: Differential ShellCheck + uses: redhat-plumbers-in-action/differential-shellcheck@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + python-test: name: Python tests runs-on: ubuntu-22.04 From ab24f6da65ea5b6ce72417b6fac4301943f67570 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 20 Feb 2024 13:46:20 +0000 Subject: [PATCH 30/99] CA-383867: Share code for handling Tpm requests This will allow to handle serialization of key as well as states in server_interface and the write cache Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/server_interface.ml | 69 +++------------------ ocaml/xapi-guard/lib/types.ml | 77 ++++++++++++++++++++++++ ocaml/xapi-guard/lib/types.mli | 29 +++++++++ 3 files changed, 116 insertions(+), 59 deletions(-) diff --git a/ocaml/xapi-guard/lib/server_interface.ml b/ocaml/xapi-guard/lib/server_interface.ml index 4bcaae8a387..37e639f1192 100644 --- a/ocaml/xapi-guard/lib/server_interface.ml +++ b/ocaml/xapi-guard/lib/server_interface.ml @@ -18,6 +18,7 @@ open Lwt.Syntax module D = Debug.Make (struct let name = __MODULE__ end) open D +module Tpm = Xapi_guard.Types.Tpm type rpc_t = Rpc.t @@ -99,59 +100,6 @@ let serve_forever_lwt_callback rpc_fn path _ req body = in Cohttp_lwt_unix.Server.respond_string ~status:`Method_not_allowed ~body () -(* The TPM has 3 kinds of states *) -type state = { - permall: string (** permanent storage *) - ; savestate: string (** for ACPI S3 *) - ; volatilestate: string (** for snapshot/migration/etc. *) -} - -let split_char = ' ' - -let join_string = String.make 1 split_char - -let deserialize t = - match String.split_on_char split_char t with - | [permall] -> - (* backwards compat with reading tpm2-00.permall *) - {permall; savestate= ""; volatilestate= ""} - | [permall; savestate; volatilestate] -> - {permall; savestate; volatilestate} - | splits -> - Fmt.failwith "Invalid state: too many splits %d" (List.length splits) - -let serialize t = - (* it is assumed that swtpm has already base64 encoded this *) - String.concat join_string [t.permall; t.savestate; t.volatilestate] - -let lookup_key key t = - match key with - | "/tpm2-00.permall" -> - t.permall - | "/tpm2-00.savestate" -> - t.savestate - | "/tpm2-00.volatilestate" -> - t.volatilestate - | s -> - Fmt.invalid_arg "Unknown TPM state key: %s" s - -let update_key key state t = - if String.contains state split_char then - Fmt.invalid_arg - "State to be stored (%d bytes) contained forbidden separator: %c" - (String.length state) split_char ; - match key with - | "/tpm2-00.permall" -> - {t with permall= state} - | "/tpm2-00.savestate" -> - {t with savestate= state} - | "/tpm2-00.volatilestate" -> - {t with volatilestate= state} - | s -> - Fmt.invalid_arg "Unknown TPM state key: %s" s - -let empty = "" - let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = let get_vtpm_ref () = let* vm = @@ -179,29 +127,32 @@ let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = Lwt_mutex.with_lock mutex @@ fun () -> (* TODO: some logging *) match (Cohttp.Request.meth req, Uri.path uri) with - | `GET, key when key <> "/" -> + | `GET, path when path <> "/" -> let* self = get_vtpm_ref () in let* contents = with_xapi ~cache @@ VTPM.get_contents ~self in - let body = contents |> deserialize |> lookup_key key in + let key = Tpm.key_of_swtpm path in + let body = Tpm.(contents |> deserialize |> lookup ~key) in let headers = Cohttp.Header.of_list [("Content-Type", "application/octet-stream")] in Cohttp_lwt_unix.Server.respond_string ~headers ~status:`OK ~body () - | `PUT, key when key <> "/" -> + | `PUT, path when path <> "/" -> let* body = Cohttp_lwt.Body.to_string body in let* self = get_vtpm_ref () in let* contents = with_xapi ~cache @@ VTPM.get_contents ~self in + let key = Tpm.key_of_swtpm path in let contents = - contents |> deserialize |> update_key key body |> serialize + Tpm.(contents |> deserialize |> update key body |> serialize) in let* () = with_xapi ~cache @@ VTPM.set_contents ~self ~contents in Cohttp_lwt_unix.Server.respond ~status:`No_content ~body:Cohttp_lwt.Body.empty () - | `DELETE, key when key <> "/" -> + | `DELETE, path when path <> "/" -> let* self = get_vtpm_ref () in let* contents = with_xapi ~cache @@ VTPM.get_contents ~self in + let key = Tpm.key_of_swtpm path in let contents = - contents |> deserialize |> update_key key empty |> serialize + Tpm.(contents |> deserialize |> update key empty_state |> serialize) in let* () = with_xapi ~cache @@ VTPM.set_contents ~self ~contents in Cohttp_lwt_unix.Server.respond ~status:`No_content diff --git a/ocaml/xapi-guard/lib/types.ml b/ocaml/xapi-guard/lib/types.ml index 7b705c89a01..3f2b41c7682 100644 --- a/ocaml/xapi-guard/lib/types.ml +++ b/ocaml/xapi-guard/lib/types.ml @@ -3,3 +3,80 @@ module Service = struct let to_string = function Varstored -> "Varstored" | Swtpm -> "Swtpm" end + +module Tpm = struct + (* The TPM has 3 kinds of states *) + type t = { + permall: string (** permanent storage *) + ; savestate: string (** for ACPI S3 *) + ; volatilestate: string (** for snapshot/migration/etc. *) + } + + type key = Perm | Save | Volatile + + let key_of_swtpm = function + | "/tpm2-00.permall" -> + Perm + | "/tpm2-00.savestate" -> + Save + | "/tpm2-00.volatilestate" -> + Volatile + | s -> + Fmt.invalid_arg "Unknown TPM state key: %s" s + + let serialize_key = function Perm -> 0 | Save -> 1 | Volatile -> 2 + + let deserialize_key = function + | 0 -> + Perm + | 1 -> + Save + | 2 -> + Volatile + | s -> + Fmt.invalid_arg "Unknown TPM state key: %i" s + + let empty_state = "" + + let empty = {permall= ""; savestate= ""; volatilestate= ""} + + let split_char = ' ' + + let join_string = String.make 1 split_char + + let deserialize t = + match String.split_on_char split_char t with + | [permall] -> + (* backwards compat with reading tpm2-00.permall *) + {permall; savestate= ""; volatilestate= ""} + | [permall; savestate; volatilestate] -> + {permall; savestate; volatilestate} + | splits -> + Fmt.failwith "Invalid state: too many splits %d" (List.length splits) + + let serialize t = + (* it is assumed that swtpm has already base64 encoded this *) + String.concat join_string [t.permall; t.savestate; t.volatilestate] + + let lookup ~key t = + match key with + | Perm -> + t.permall + | Save -> + t.savestate + | Volatile -> + t.volatilestate + + let update key state t = + if String.contains state split_char then + Fmt.invalid_arg + "State to be stored (%d bytes) contained forbidden separator: %c" + (String.length state) split_char ; + match key with + | Perm -> + {t with permall= state} + | Save -> + {t with savestate= state} + | Volatile -> + {t with volatilestate= state} +end diff --git a/ocaml/xapi-guard/lib/types.mli b/ocaml/xapi-guard/lib/types.mli index 6bb036826d7..f210ea8c96a 100644 --- a/ocaml/xapi-guard/lib/types.mli +++ b/ocaml/xapi-guard/lib/types.mli @@ -5,3 +5,32 @@ module Service : sig val to_string : t -> string end + +module Tpm : sig + (** TPMs have 3 kind of states *) + type t + + (** key to access a single state *) + type key + + val key_of_swtpm : string -> key + (** [key_of_swtpm path] returns a state key represented by [path]. These paths + are parts of the requests generated by SWTPM and may contain slashes *) + + val deserialize_key : int -> key + + val serialize_key : key -> int + (** [serialize key] returns the state key represented by [key]. *) + + val empty : t + + val empty_state : string + + val deserialize : string -> t + + val serialize : t -> string + + val update : key -> string -> t -> t + + val lookup : key:key -> t -> string +end From a714f76f7204d12a5438329e7771f0c79fb6a73e Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 6 Dec 2023 10:08:09 +0000 Subject: [PATCH 31/99] CA-383867: Add local disk cache library for xapi guard This enables xapi-guard to decouple persistence of TPM contents from the xapi service being online. That is, when xapi is down. The contents of the TPMs will be written to disk, and when xapi is back online the contents will be uploaded. This is needed to protect VMs while xapi is being restarted, usually as part of an update. Some properties of the cache: - The cache is tried to be bypassed whenever possible, and is only used as fallback after a write fails. - The cache is handled by a thread that writes to cache and one that reads from it. They communicate through a bounded queue. - Whenever a TPM content is written to disk, previous versions of it are deleted. This helps the reading thread to catch up. - When the queue has been filled the writer stops adding elements to the queue, and the reader will try to flush the queue, and after it will try to flush the cache. After this happens both threads will transition to cache bypass operation. Signed-off-by: Pau Ruiz Safont --- doc/content/xapi-guard/_index.md | 56 +++ ocaml/xapi-guard/lib/disk_cache.ml | 449 ++++++++++++++++++++ ocaml/xapi-guard/lib/disk_cache.mli | 25 ++ ocaml/xapi-guard/lib/dune | 5 +- ocaml/xapi-guard/lib/lwt_bounded_stream.ml | 48 +++ ocaml/xapi-guard/lib/lwt_bounded_stream.mli | 34 ++ ocaml/xapi-guard/src/dune | 1 + ocaml/xapi-guard/test/cache_test.ml | 156 +++++++ ocaml/xapi-guard/test/cache_test.mli | 0 ocaml/xapi-guard/test/dune | 16 + 10 files changed, 789 insertions(+), 1 deletion(-) create mode 100644 ocaml/xapi-guard/lib/disk_cache.ml create mode 100644 ocaml/xapi-guard/lib/disk_cache.mli create mode 100644 ocaml/xapi-guard/lib/lwt_bounded_stream.ml create mode 100644 ocaml/xapi-guard/lib/lwt_bounded_stream.mli create mode 100644 ocaml/xapi-guard/test/cache_test.ml create mode 100644 ocaml/xapi-guard/test/cache_test.mli diff --git a/doc/content/xapi-guard/_index.md b/doc/content/xapi-guard/_index.md index bcafb968b07..39d64a558e5 100644 --- a/doc/content/xapi-guard/_index.md +++ b/doc/content/xapi-guard/_index.md @@ -17,6 +17,8 @@ Principles 2. Xenopsd is able to control xapi-guard through message switch, this access is not limited. 3. Listening to domain socket is restored whenever the daemon restarts to minimize disruption of running domains. +4. Disruptions to requests when xapi is unavailable is minimized. + The startup procedure is not blocked by the availability of xapi, and write requests from domains must not fail because xapi is unavailable. Overview @@ -26,3 +28,57 @@ Xapi-guard forwards calls from domains to xapi to persist UEFI variables, and up To do this, it listens to 1 socket per service (varstored, or swtpm) per domain. To create these sockets before the domains are running, it listens to a message-switch socket. This socket listens to calls from xenopsd, which orchestrates the domain creation. + +To protect the domains from xapi being unavailable transiently, xapi-guard provides an on-disk cache for vTPM writes. +This cache acts as a buffer and stores the requests temporarily until xapi can be contacted again. +This situation usually happens when xapi is being restarted as part of an update. +SWTPM, the vTPM daemon, reads the contents of the TPM from xapi-guard on startup, suspend, and resume. +During normal operation SWTPM does not send read requests from xapi-guard. + +The cache module consists of two Lwt threads, one that writes to disk, and another one that reads from disk. +The writer is triggered when a VM writes to the vTPM. +It never blocks if xapi is unreachable, but responds as soon as the data has been stored either by xapi or on the local disk, such that the VM receives a timely response to the write request. +Both try to send the requests to xapi, depending on the state, to attempt write all the cached data back to xapi, and stop using the cache. +The threads communicate through a bounded queue, this is done to limit the amount of memory used. +This queue is a performance optimisation, where the writer informs the reader precisely which are the names of the cache files, such that the reader does not need to list the cache directory. +And a full queue does not mean data loss, just a loss of performance; vTPM writes are still cached. + +This means that the cache operates in three modes: +- Direct: during normal operation the disk is not used at all +- Engaged: both threads use the queue to order events +- Disengaged: A thread dumps request to disk while the other reads the cache + until it's empty + +```mermaid +--- +title: Cache State +--- +stateDiagram-v2 + Disengaged + note right of Disengaged + Writer doesn't add requests to queue + Reader reads from cache and tries to push to xapi + end note + Direct + note left of Direct + Writer bypasses cache, send to xapi + Reader waits + end note + Engaged + note right of Engaged + Writer writes to cache and adds requests to queue + Reader reads from queue and tries to push to xapi + end note + + [*] --> Disengaged + + Disengaged --> Disengaged : Reader pushed pending TPMs to xapi, in the meantime TPMs appeared in the cache + Disengaged --> Direct : Reader pushed pending TPMs to xapi, cache is empty + + Direct --> Direct : Writer receives TPM, sent to xapi + Direct --> Engaged : Writer receives TPM, error when sent to xapi + + Engaged --> Direct : Reader sent TPM to xapi, finds an empty queue + Engaged --> Engaged : Writer receives TPM, queue is not full + Engaged --> Disengaged : Writer receives TPM, queue is full +``` diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml new file mode 100644 index 00000000000..b972654f085 --- /dev/null +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -0,0 +1,449 @@ +(* Copyright (C) Cloud Software Group, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published + by the Free Software Foundation; version 2.1 only. with the special + exception on linking described in file LICENSE. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. +*) + +module D = Debug.Make (struct let name = __MODULE__ end) + +let ( // ) = Filename.concat + +let runtime_data = "/var/lib" // "xapi-guard" + +let ( let* ) = Lwt.bind + +let ( let@ ) f x = f x + +let with_lock = Lwt_mutex.with_lock + +type t = Uuidm.t * Mtime.t * Types.Tpm.key + +let cache_of service = runtime_data // Types.Service.to_string service + +let files_in dir ~otherwise = + Lwt.catch + (fun () -> + let* listing = Lwt_unix.files_of_directory dir |> Lwt_stream.to_list in + List.filter_map + (function "." | ".." -> None | name -> Some (dir // name)) + listing + |> Lwt.return + ) + otherwise + +let unlink_safe file = + let __FUN = __FUNCTION__ in + Lwt.catch + (fun () -> Lwt_unix.unlink file) + (function + | Unix.(Unix_error (ENOENT, _, _)) -> + Lwt.pause () + | e -> + D.info "%s: error %s when deleting %s, ignoring" __FUN + (Printexc.to_string e) file ; + Lwt.pause () + ) + +type valid_file = t * string + +type file = + | Latest of valid_file + | Outdated of valid_file + | Temporary of string + | Invalid of string + +let path_of_key root (uuid, timestamp, key) = + root + // Uuidm.to_string uuid + // Types.Tpm.(serialize_key key |> string_of_int) + // Mtime.(to_uint64_ns timestamp |> Int64.to_string) + +let key_of_path path = + let ( let* ) = Option.bind in + let key_dir = Filename.(dirname path) in + let* uuid = Filename.(basename (dirname key_dir)) |> Uuidm.of_string in + let* key = + Filename.basename key_dir + |> int_of_string_opt + |> Option.map Types.Tpm.deserialize_key + in + let* timestamp = + Filename.basename path + |> Int64.of_string_opt + |> Option.map Mtime.of_uint64_ns + in + Some ((uuid, timestamp, key), path) + +let path_is_temp path = + let pathlen = String.length path in + String.ends_with ~suffix:".pre" path + && key_of_path (String.sub path 0 (pathlen - 4)) |> Option.is_some + +let temp_of_path path = path ^ ".pre" + +let get_all_contents root = + let classify contents = + let rec loop (acc, found) = function + | [] -> + List.rev acc + | latest :: others -> ( + match key_of_path latest with + | None -> + let file = + if path_is_temp latest then + Temporary latest + else + Invalid latest + in + loop (file :: acc, found) others + | Some valid_file -> + let file = + if found then Outdated valid_file else Latest valid_file + in + loop (file :: acc, false) others + ) + in + let ordered = List.fast_sort (fun x y -> String.compare y x) contents in + loop ([], false) ordered + in + let empty = Fun.const (Lwt.return []) in + let contents_of_key key = + let* contents = files_in key ~otherwise:empty in + Lwt.return (classify contents) + in + let* tpms = files_in root ~otherwise:empty in + let* files = + Lwt_list.map_p + (fun tpm -> + let* keys = files_in tpm ~otherwise:empty in + Lwt_list.map_p contents_of_key keys + ) + tpms + in + Lwt.return List.(concat (concat files)) + +let persist_to ~filename:f_path ~contents = + let atomic_write_to_file ~perm f = + let tmp_path = temp_of_path f_path in + let dirname = Filename.dirname f_path in + let flags = Unix.[O_WRONLY; O_CREAT; O_SYNC] in + let* fd_tmp = Lwt_unix.openfile tmp_path flags perm in + let* () = + Lwt.finalize + (fun () -> + (* do not close fd when closing the channel, avoids double-closing the fd *) + let close () = Lwt.return_unit in + let chan = Lwt_io.of_fd ~mode:Output ~close fd_tmp in + let* () = + Lwt.finalize (fun () -> f chan) (fun () -> Lwt_io.close chan) + in + Lwt_unix.fsync fd_tmp + ) + (fun () -> Lwt_unix.close fd_tmp) + in + let* () = Lwt_unix.rename tmp_path f_path in + let* fd_dir = Lwt_unix.openfile dirname [O_RDONLY] 0 in + Lwt.finalize + (fun () -> Lwt_unix.fsync fd_dir) + (fun () -> Lwt_unix.close fd_dir) + in + let write out_chan = Lwt_io.write out_chan contents in + atomic_write_to_file ~perm:0o600 write + +(** - Direct: request doesn't pass through the cache + - Engaged: both side coordinate through the queue, writer ends the mode + when the queue has been filled. + - Disengaged: writer ignores the queue, reader empties it and the cache; + then it changes the mode to engaged. +*) +type state = Direct | Engaged | Disengaged + +type channel = { + queue: t Lwt_bounded_stream.t + ; push: t option -> unit option + ; lock: Lwt_mutex.t (* lock for the states *) + ; mutable state: state +} + +(* + Notes: + - uses Mtime.t to force usage of monotonic time + - This means that between runs (and reboots) cached stated is lost if not + persisted first. + IDEA: carryover: read contents of cache and "convert it" to the current run + + TODO: + - Add startup step to convert existing content to new time + - Exponential backoff on xapi push error + - Limit error logging on xapi push error: once per downtime is enough + *) + +module Writer : sig + val with_cache : + direct:(t -> string -> (unit, exn) Lwt_result.t) + -> Types.Service.t + -> channel + -> ((t -> string -> unit Lwt.t) -> 'a Lwt.t) + -> 'a Lwt.t + (** [with_cache ~direct typ queue context] creates a cache for content of + type [typ]. The cache is writable through the function [context], which + is provided a writing function to persist to the cache. It uses [channel] + to push events to + + Example: + Xapi_guard.Disk_cache.(Writer.with_cache ~direct:(upload session_cache) Tpm channel) + @@ fun write_tpm -> write_tpm (uuid, time, key) contents + *) +end = struct + let mkdir_p ?(perm = 0o755) path = + let rec loop acc path = + let create_dir () = Lwt_unix.mkdir path perm in + let create_subdirs () = Lwt_list.iter_s (fun (_, f) -> f ()) acc in + Lwt.try_bind create_dir create_subdirs (function + | Unix.(Unix_error (EEXIST, _, _)) -> + (* create directories, parents first *) + create_subdirs () + | Unix.(Unix_error (ENOENT, _, _)) -> + let parent = Filename.dirname path in + loop ((path, create_dir) :: acc) parent + | exn -> + let msg = + Printf.sprintf {|Could not create directory "%s" because: %s|} + path (Printexc.to_string exn) + in + Lwt.fail (Failure msg) + ) + in + loop [] path + + let files_in_existing dir = + let create_dir = function + | Unix.(Unix_error (ENOENT, _, _)) -> + let* () = mkdir_p dir ~perm:0o700 in + Lwt.return [] + | e -> + raise e + in + files_in dir ~otherwise:create_dir + + let write_contents ~direct root queue (uuid, now, key) contents = + let __FUN = __FUNCTION__ in + + let key_str = Types.Tpm.(serialize_key key |> string_of_int) in + let key_dir = root // Uuidm.(to_string uuid) // key_str in + (* 1. Record existing requests in cache *) + let* outdated_contents = files_in_existing key_dir in + + let filename = key_dir // (Mtime.to_uint64_ns now |> Int64.to_string) in + (* 2. Write new timestamped content to cache, atomically, if needed; and + notify the other side, if needed *) + let persist () = persist_to ~filename ~contents in + let persist_and_push () = + let push () = + match queue.push (Some (uuid, now, key)) with + | Some () -> + Lwt.return_unit + | None -> + (* Queue is full, change mode to ignore queue *) + queue.state <- Disengaged ; + Lwt.return_unit + in + let* () = persist () in + push () + in + let engage_and_persist exn = + queue.state <- Engaged ; + D.info "%s: Error on push. Reason: %s" __FUN (Printexc.to_string exn) ; + persist_and_push () + in + let* () = + with_lock queue.lock (fun () -> + match queue.state with + | Direct -> + Lwt.try_bind + (fun () -> direct (uuid, now, key) contents) + (function + | Ok () -> + Lwt.return_unit + | Error exn -> + engage_and_persist exn + ) + (function exn -> engage_and_persist exn) + | Engaged -> + persist_and_push () + | Disengaged -> + persist () + ) + in + (* 4. Delete previous requests from filesystem *) + let* _ = Lwt_list.map_p unlink_safe outdated_contents in + Lwt.return_unit + + let with_cache ~direct typ queue f = + let root = cache_of typ in + let* () = mkdir_p root ~perm:0o700 in + f (write_contents ~direct root queue) +end + +module Watcher : sig + val watch : + direct:(t -> string -> (unit, exn) Lwt_result.t) + -> Types.Service.t + -> channel + -> unit + -> unit Lwt.t +end = struct + type push_cache = File of valid_file | Update_all | Wait + + (* Outdated and invalid files can be deleted, keep temporary files just in case + they need to be recovered *) + let discarder = function + | Latest _ as f -> + Either.Left f + | Temporary _ as f -> + Left f + | Outdated (_, p) -> + Right p + | Invalid p -> + Right p + + let get_latest_and_delete_rest root = + let* files = get_all_contents root in + let keep, to_delete = List.partition_map discarder files in + let* () = Lwt_list.iter_p unlink_safe to_delete in + (* Ignore temporaty files *) + let latest = + List.filter_map (function Latest f -> Some f | _ -> None) keep + in + Lwt.return latest + + (** Warning, may raise Unix.Unix_error *) + let read_from ~filename = + let flags = Unix.[O_RDONLY] in + let perm = 0o000 in + Lwt_io.with_file ~flags ~perm ~mode:Input filename Lwt_io.read + + let retry_push push (uuid, timestamp, key) contents = + let __FUN = __FUNCTION__ in + let push' () = push (uuid, timestamp, key) contents in + let rec retry k = + let on_error e = + D.info "%s: Error on push, attempt %i. Reason: %s" __FUN k + (Printexc.to_string e) ; + let* () = Lwt_unix.sleep 0.1 in + retry (k + 1) + in + Lwt.try_bind push' + (function Ok () -> Lwt.return_unit | Error e -> on_error e) + on_error + in + retry 1 + + let push_file push (key, path) = + let __FUN = __FUNCTION__ in + let on_error = function + | Unix.(Unix_error (ENOENT, _, _)) -> + Lwt.return_unit + | exn -> + D.info "%s: error when reading '%s': %s" __FUN path + Printexc.(to_string exn) ; + Lwt.return_unit + in + + Lwt.try_bind + (fun () -> read_from ~filename:path) + (fun contents -> + let* () = retry_push push key contents in + unlink_safe path + ) + on_error + + let push_files push files = Lwt_list.iter_s (push_file push) (List.rev files) + + let update_all queue push root = + let __FUN = __FUNCTION__ in + let* contents = get_latest_and_delete_rest root in + let* () = push_files push contents in + let@ () = with_lock queue.lock in + let* contents = get_latest_and_delete_rest root in + let* () = + match contents with + | [] -> + queue.state <- Direct ; + D.debug "%s: Cache clean; Going direct" __FUN ; + Lwt.return_unit + | _ -> + Lwt.return_unit + in + Lwt.return_unit + + let resolve queue push root = function + | File file -> ( + let* () = push_file push file in + let@ () = with_lock queue.lock in + match queue.state with + | Direct | Disengaged -> + Lwt.return_unit + | Engaged -> + let () = + if Lwt_bounded_stream.size queue.queue = 0 then + queue.state <- Direct + in + Lwt.return_unit + ) + | Update_all -> + update_all queue push root + | Wait -> + (* Do not busy loop when the system can cope with the requests *) + Lwt_unix.sleep 0.2 + + let watch ~direct typ queue = + let root = cache_of typ in + let __FUN = __FUNCTION__ in + let rec loop () = + (* When the pushing side is disengaged it doesn't push events to the + queue, this means that trying to drain it completely would leave the + pulling side locked waiting when the queue is empty. + - Read the number of elements in the queue while draining it and + then switch to read the contents from the cache; or + - Switch immediately to reading the contents from cache and ignore + the contents of the queue by calling an specialized method in the + queue module to drain it. + *) + let get_action () = + let@ () = with_lock queue.lock in + match queue.state with + | Disengaged when Lwt_bounded_stream.size queue.queue < 1 -> + let* () = Lwt.pause () in + Lwt.return Update_all + | Direct -> + let* () = Lwt.pause () in + Lwt.return Wait + | _ -> ( + let* elem = Lwt_bounded_stream.get queue.queue in + match elem with + | None -> + raise (Failure "Other side closed channel, cannot continue") + | Some elem -> + Lwt.return (File (elem, path_of_key root elem)) + ) + in + let* action = get_action () in + let* () = resolve queue direct root action in + loop () + in + loop +end + +let setup typ direct = + let queue, push = Lwt_bounded_stream.create 4098 in + let lock = Lwt_mutex.create () in + let q = {queue; push; lock; state= Disengaged} in + (Writer.with_cache ~direct typ q, Watcher.watch ~direct typ q) diff --git a/ocaml/xapi-guard/lib/disk_cache.mli b/ocaml/xapi-guard/lib/disk_cache.mli new file mode 100644 index 00000000000..0f387d4ebd1 --- /dev/null +++ b/ocaml/xapi-guard/lib/disk_cache.mli @@ -0,0 +1,25 @@ +(* Copyright (C) Cloud Software Group, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published + by the Free Software Foundation; version 2.1 only. with the special + exception on linking described in file LICENSE. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. +*) + +(** [t] t is the minimal type to recognise elements in a cache. This does not + contain the contents of the elements being contained, only the metadata *) +type t = Uuidm.t * Mtime.t * Types.Tpm.key + +val setup : + Types.Service.t + -> (t -> string -> (unit, exn) Lwt_result.t) + -> (((t -> string -> unit Lwt.t) -> 'a Lwt.t) -> 'a Lwt.t) + * (unit -> unit Lwt.t) +(** [setup service push_callback] Returns a local disk buffer for [service] + which will use [push_callback] to push the elements to their final + destination *) diff --git a/ocaml/xapi-guard/lib/dune b/ocaml/xapi-guard/lib/dune index 87d10e7766e..052810ead5f 100644 --- a/ocaml/xapi-guard/lib/dune +++ b/ocaml/xapi-guard/lib/dune @@ -20,10 +20,13 @@ ) (library (name xapi_guard) - (modules dorpc types) + (modules dorpc types disk_cache lwt_bounded_stream) (libraries rpclib.core + inotify + inotify.lwt lwt + lwt.unix uri xapi-backtrace xapi-consts diff --git a/ocaml/xapi-guard/lib/lwt_bounded_stream.ml b/ocaml/xapi-guard/lib/lwt_bounded_stream.ml new file mode 100644 index 00000000000..90efe83758b --- /dev/null +++ b/ocaml/xapi-guard/lib/lwt_bounded_stream.ml @@ -0,0 +1,48 @@ +(* + * Copyright (c) 2012 Citrix Systems + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + *) + +let ( let* ) = Lwt.bind + +type 'a t = {stream: 'a Lwt_stream.t; capacity: int; size: int ref} + +let create capacity = + let stream, stream_push = Lwt_stream.create () in + let t = {stream; capacity; size= ref 0} in + let push = function + | Some _ when !(t.size) > t.capacity -> + None + | None -> + stream_push None ; Some () + | elem -> + stream_push elem ; incr t.size ; Some () + in + (t, push) + +let size {size; _} = !size + +let get_available t = + let all = Lwt_stream.get_available t.stream in + t.size := !(t.size) - List.length all ; + all + +let get t = + let* elem = Lwt_stream.get t.stream in + decr t.size ; Lwt.return elem + +let nget n t = + let* all = Lwt_stream.nget n t.stream in + t.size := !(t.size) - List.length all ; + Lwt.return all diff --git a/ocaml/xapi-guard/lib/lwt_bounded_stream.mli b/ocaml/xapi-guard/lib/lwt_bounded_stream.mli new file mode 100644 index 00000000000..b2b310f77e0 --- /dev/null +++ b/ocaml/xapi-guard/lib/lwt_bounded_stream.mli @@ -0,0 +1,34 @@ +(* + * Copyright (c) 2012 Citrix Systems + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + *) + +(** Similar to Lwt_stream.bounded_push except threads never block in push() *) +type 'a t + +val create : int -> 'a t * ('a option -> unit option) +(** [create capacity] creates a stream which can contain at most + [capacity] elements *) + +val get_available : 'a t -> 'a list +(** [get_available t] returns all available elements from [t] without blocking *) + +val get : 'a t -> 'a option Lwt.t +(** [get t] returns an element from [t] *) + +val nget : int -> 'a t -> 'a list Lwt.t +(** [nget n t] returns [n] elements from [t] *) + +val size : 'a t -> int +(** [size t] return the number of enqueued elements *) diff --git a/ocaml/xapi-guard/src/dune b/ocaml/xapi-guard/src/dune index dfbf8e9a9e4..baac1d24101 100644 --- a/ocaml/xapi-guard/src/dune +++ b/ocaml/xapi-guard/src/dune @@ -1,5 +1,6 @@ (executable (name main) + (modules main) (libraries astring cmdliner diff --git a/ocaml/xapi-guard/test/cache_test.ml b/ocaml/xapi-guard/test/cache_test.ml new file mode 100644 index 00000000000..ce63c8cec8d --- /dev/null +++ b/ocaml/xapi-guard/test/cache_test.ml @@ -0,0 +1,156 @@ +let ( let@ ) f x = f x + +let ( let* ) = Lwt.bind + +module Tpm = Xapi_guard.Types.Tpm + +module TPMs = struct + let tpms_created = Atomic.make 1 + + let request_persist uuid write = + let __FUN = __FUNCTION__ in + + let key = Tpm.deserialize_key (Random.int 3) in + + let time = Mtime_clock.now () in + let serial_n = Atomic.fetch_and_add tpms_created 1 in + let contents = + Printf.sprintf "contents %s" (Mtime.to_uint64_ns time |> Int64.to_string) + in + let* () = + Logs_lwt.app (fun m -> + m "%s: Content № %i created: %a/%i/%a" __FUN serial_n Uuidm.pp uuid + Tpm.(serialize_key key) + Mtime.pp time + ) + in + write (uuid, time, key) contents +end + +let lwt_reporter () = + let buf_fmt ~like = + let b = Buffer.create 512 in + ( Fmt.with_buffer ~like b + , fun () -> + let m = Buffer.contents b in + Buffer.reset b ; m + ) + in + let app, app_flush = buf_fmt ~like:Fmt.stdout in + let dst, dst_flush = buf_fmt ~like:Fmt.stderr in + let reporter = Logs_fmt.reporter ~app ~dst () in + let report src level ~over k msgf = + let k () = + let write () = + match level with + | Logs.App -> + Lwt_io.write Lwt_io.stdout (app_flush ()) + | _ -> + Lwt_io.write Lwt_io.stderr (dst_flush ()) + in + let unblock () = over () ; Lwt.return_unit in + Lwt.finalize write unblock |> Lwt.ignore_result ; + k () + in + reporter.Logs.report src level ~over:(fun () -> ()) k msgf + in + {Logs.report} + +let setup_log level = + Logs.set_level level ; + Logs.set_reporter (lwt_reporter ()) ; + () + +let ok = Lwt_result.ok + +let retry_forever fname f = + let rec loop () = + let* () = + Lwt.catch f (function exn -> + let* () = + Logs_lwt.app (fun m -> + m "%s failed with %s, retrying..." fname (Printexc.to_string exn) + ) + in + Lwt_unix.sleep 0.5 + ) + in + loop () + [@@tailcall] + in + loop () + +let max_sent = 128 + +let received = ref 0 + +let throttled_reads = Mtime.Span.(200 * ms) + +let failing_writes_period = Mtime.Span.(500 * ms) + +let epoch = Mtime_clock.now () + +let should_fail () : bool = + let rec polarity elapsed = + if Mtime.Span.compare elapsed failing_writes_period < 0 then + true + else + not (polarity Mtime.Span.(abs_diff elapsed failing_writes_period)) + in + let elapsed = Mtime.span epoch (Mtime_clock.now ()) in + polarity elapsed + +let log (uuid, timestamp, key) content : (unit, exn) Result.t Lwt.t = + let __FUN = __FUNCTION__ in + let ( let* ) = Lwt_result.bind in + let maybe_fail () = + if should_fail () then + Lwt_result.fail + (failwith (Printf.sprintf {|oops, could not write '%s'|} content)) + else + Lwt_result.return () + in + let* () = maybe_fail () in + received := !received + 1 ; + Logs_lwt.app (fun m -> + m "%s Content № %i detected: %a/%i/%a" __FUN !received Uuidm.pp uuid + Tpm.(serialize_key key) + Mtime.pp timestamp + ) + |> ok + +let to_cache with_writer = + let __FUN = __FUNCTION__ in + let elapsed = Mtime_clock.counter () in + let rec loop_and_stop uuid sent () = + let sent = sent + 1 in + + let@ write_tpm = with_writer in + let* () = TPMs.request_persist uuid write_tpm in + if sent >= max_sent then + Logs_lwt.app (fun m -> + m "%s: Stopping requests after %i writes" __FUN sent + ) + else if Mtime.Span.compare (Mtime_clock.count elapsed) throttled_reads > 0 + then + let* () = Lwt_unix.sleep 0.1 in + loop_and_stop uuid sent () + else + let* () = Lwt.pause () in + loop_and_stop uuid sent () + in + List.init 4 (fun _ -> Uuidm.(v `V4)) + |> List.map (fun uuid -> loop_and_stop uuid 0 ()) + +let from_cache with_watcher = retry_forever "watcher" with_watcher + +let main () = + let with_writer, with_watcher = Xapi_guard.Disk_cache.(setup Swtpm log) in + let reader = from_cache with_watcher in + let writers = to_cache with_writer in + let* _ = Lwt.all (reader :: writers) in + Lwt.return_unit + +let () = + setup_log @@ Some Logs.Debug ; + Lwt_main.run (main ()) diff --git a/ocaml/xapi-guard/test/cache_test.mli b/ocaml/xapi-guard/test/cache_test.mli new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ocaml/xapi-guard/test/dune b/ocaml/xapi-guard/test/dune index 934256a9f7a..e082a47a690 100644 --- a/ocaml/xapi-guard/test/dune +++ b/ocaml/xapi-guard/test/dune @@ -1,6 +1,7 @@ (test (name xapi_guard_test) (modes exe) + (modules (:standard \ cache_test)) (libraries alcotest alcotest-lwt @@ -17,3 +18,18 @@ xen-api-client-lwt) (package varstored-guard) ) + +(executable + (name cache_test) + (modules cache_test) + (libraries + logs + logs.fmt + logs.lwt + lwt + lwt.unix + mtime + mtime.clock.os + uuidm + xapi_guard) + (preprocess (pps ppx_deriving_rpc))) From cfb7748dc2a36833a77970e2fafe5666159b13f5 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 16 Feb 2024 17:27:06 +0000 Subject: [PATCH 32/99] CA-383867: Delay conversion of VM's UUIDs to string This allows to pass the UUID directly to the on-disk cache that will be introduced Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/server_interface.ml | 11 +++++++---- ocaml/xapi-guard/src/main.ml | 2 +- ocaml/xapi-guard/test/xapi_guard_test.ml | 8 +++++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ocaml/xapi-guard/lib/server_interface.ml b/ocaml/xapi-guard/lib/server_interface.ml index 37e639f1192..aee1d242776 100644 --- a/ocaml/xapi-guard/lib/server_interface.ml +++ b/ocaml/xapi-guard/lib/server_interface.ml @@ -102,8 +102,9 @@ let serve_forever_lwt_callback rpc_fn path _ req body = let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = let get_vtpm_ref () = + let vm_uuid_str = Uuidm.to_string vm_uuid in let* vm = - with_xapi ~cache @@ Xen_api_lwt_unix.VM.get_by_uuid ~uuid:vm_uuid + with_xapi ~cache @@ Xen_api_lwt_unix.VM.get_by_uuid ~uuid:vm_uuid_str in let* vTPMs = with_xapi ~cache @@ Xen_api_lwt_unix.VM.get_VTPMs ~self:vm in match vTPMs with @@ -113,7 +114,8 @@ let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = ignoring request" __FUNCTION__ ; let msg = - Printf.sprintf "No VTPM associated with VM %s, nothing to do" vm_uuid + Printf.sprintf "No VTPM associated with VM %s, nothing to do" + vm_uuid_str in raise (Failure msg) | self :: _ -> @@ -163,9 +165,10 @@ let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = (* Create a restricted RPC function and socket for a specific VM *) let make_server_varstored ~cache path vm_uuid = + let vm_uuid_str = Uuidm.to_string vm_uuid in let module Server = Xapi_idl_guard_varstored.Interface.RPC_API (Rpc_lwt.GenServer ()) in - let get_vm_ref () = with_xapi ~cache @@ VM.get_by_uuid ~uuid:vm_uuid in + let get_vm_ref () = with_xapi ~cache @@ VM.get_by_uuid ~uuid:vm_uuid_str in let ret v = (* TODO: maybe map XAPI exceptions *) Lwt.bind v Lwt.return_ok |> Rpc_lwt.T.put @@ -187,7 +190,7 @@ let make_server_varstored ~cache path vm_uuid = (let* (_ : _ Ref.t) = with_xapi ~cache @@ Message.create ~name:"VM_SECURE_BOOT_FAILED" ~priority ~cls:`VM - ~obj_uuid:vm_uuid ~body + ~obj_uuid:vm_uuid_str ~body in Lwt.return_unit ) diff --git a/ocaml/xapi-guard/src/main.ml b/ocaml/xapi-guard/src/main.ml index b80bf354516..6979665a8a8 100644 --- a/ocaml/xapi-guard/src/main.ml +++ b/ocaml/xapi-guard/src/main.ml @@ -110,7 +110,7 @@ let listen_for_vm {Persistent.vm_uuid; path; gid; typ} = (Types.Service.to_string typ) path vm_uuid_str ; let* () = safe_unlink path in - let* stop_server = make_server ~cache path vm_uuid_str in + let* stop_server = make_server ~cache path vm_uuid in let* () = log_fds () in Hashtbl.add sockets path (stop_server, (vm_uuid, gid, typ)) ; let* () = Lwt_unix.chmod path 0o660 in diff --git a/ocaml/xapi-guard/test/xapi_guard_test.ml b/ocaml/xapi-guard/test/xapi_guard_test.ml index 41ce8f6e347..c4996bef0c7 100644 --- a/ocaml/xapi-guard/test/xapi_guard_test.ml +++ b/ocaml/xapi-guard/test/xapi_guard_test.ml @@ -60,7 +60,9 @@ let xapi_rpc call = | _ -> Fmt.failwith "XAPI RPC call %s not expected in test" call.Rpc.name -let vm_uuid = Uuidx.(to_string (make ())) +let vm_uuid = Uuidm.v `V4 + +let vm_uuid_str = Uuidm.to_string vm_uuid let () = let old_hook = !Lwt.async_exception_hook in @@ -101,7 +103,7 @@ let with_rpc f switch () = let dict = Alcotest.(list @@ pair string string) let test_change_nvram ~rpc ~session_id () = - let* self = VM.get_by_uuid ~rpc ~session_id ~uuid:vm_uuid in + let* self = VM.get_by_uuid ~rpc ~session_id ~uuid:vm_uuid_str in let* nvram0 = VM.get_NVRAM ~rpc ~session_id ~self in Alcotest.(check' dict) ~msg:"nvram initial" ~expected:[] ~actual:nvram0 ; let contents = "nvramnew" in @@ -131,7 +133,7 @@ let test_bad_set_nvram ~rpc ~session_id () = let* () = VM.set_NVRAM_EFI_variables ~rpc ~session_id ~self:vm_bad ~value:"bad" in - let* vm_ref = VM.get_by_uuid ~rpc ~session_id ~uuid:vm_uuid in + let* vm_ref = VM.get_by_uuid ~rpc ~session_id ~uuid:vm_uuid_str in let* nvram = VM.get_NVRAM ~rpc ~session_id ~self:vm_ref in Alcotest.(check' dict) ~msg:"only managed to change own nvram" ~actual:nvram From 8582a708f6e1579be52740bbbf28f1761d32aeaa Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 16 Feb 2024 17:37:35 +0000 Subject: [PATCH 33/99] CA-383867: Segregate vtpm persistance out of the callback This allows to use the persistence function from outside the callback, which will be useful to thread into the on-disk cache Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/server_interface.ml | 62 +++++++++++++----------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/ocaml/xapi-guard/lib/server_interface.ml b/ocaml/xapi-guard/lib/server_interface.ml index aee1d242776..7bd6e79a8b1 100644 --- a/ocaml/xapi-guard/lib/server_interface.ml +++ b/ocaml/xapi-guard/lib/server_interface.ml @@ -100,28 +100,37 @@ let serve_forever_lwt_callback rpc_fn path _ req body = in Cohttp_lwt_unix.Server.respond_string ~status:`Method_not_allowed ~body () -let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = - let get_vtpm_ref () = - let vm_uuid_str = Uuidm.to_string vm_uuid in - let* vm = - with_xapi ~cache @@ Xen_api_lwt_unix.VM.get_by_uuid ~uuid:vm_uuid_str - in - let* vTPMs = with_xapi ~cache @@ Xen_api_lwt_unix.VM.get_VTPMs ~self:vm in - match vTPMs with - | [] -> - D.warn - "%s: received a request from a VM that has no VTPM associated, \ - ignoring request" - __FUNCTION__ ; - let msg = - Printf.sprintf "No VTPM associated with VM %s, nothing to do" - vm_uuid_str - in - raise (Failure msg) - | self :: _ -> - let* uuid = with_xapi ~cache @@ Xen_api_lwt_unix.VTPM.get_uuid ~self in - with_xapi ~cache @@ VTPM.get_by_uuid ~uuid +let with_xapi_vtpm ~cache vm_uuid = + let vm_uuid_str = Uuidm.to_string vm_uuid in + let* vm = + with_xapi ~cache @@ Xen_api_lwt_unix.VM.get_by_uuid ~uuid:vm_uuid_str in + let* vTPMs = with_xapi ~cache @@ Xen_api_lwt_unix.VM.get_VTPMs ~self:vm in + match vTPMs with + | [] -> + D.warn + "%s: received a request from a VM that has no VTPM associated, \ + ignoring request" + __FUNCTION__ ; + let msg = + Printf.sprintf "No VTPM associated with VM %s, nothing to do" + vm_uuid_str + in + raise (Failure msg) + | self :: _ -> + let* uuid = with_xapi ~cache @@ Xen_api_lwt_unix.VTPM.get_uuid ~self in + with_xapi ~cache @@ VTPM.get_by_uuid ~uuid + +let push_vtpm ~cache vm_uuid path contents = + let* self = with_xapi_vtpm ~cache vm_uuid in + let* old_contents = with_xapi ~cache @@ VTPM.get_contents ~self in + let contents = + Tpm.(old_contents |> deserialize |> update key contents |> serialize) + in + let* () = with_xapi ~cache @@ VTPM.set_contents ~self ~contents in + Lwt.return_unit + +let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = let uri = Cohttp.Request.uri req in (* in case the connection is interrupted/etc. we may still have pending operations, so use a per vTPM mutex to ensure we really only have 1 pending operation at a time for a vTPM @@ -130,7 +139,7 @@ let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = (* TODO: some logging *) match (Cohttp.Request.meth req, Uri.path uri) with | `GET, path when path <> "/" -> - let* self = get_vtpm_ref () in + let* self = with_xapi_vtpm ~cache vm_uuid in let* contents = with_xapi ~cache @@ VTPM.get_contents ~self in let key = Tpm.key_of_swtpm path in let body = Tpm.(contents |> deserialize |> lookup ~key) in @@ -140,17 +149,12 @@ let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = Cohttp_lwt_unix.Server.respond_string ~headers ~status:`OK ~body () | `PUT, path when path <> "/" -> let* body = Cohttp_lwt.Body.to_string body in - let* self = get_vtpm_ref () in - let* contents = with_xapi ~cache @@ VTPM.get_contents ~self in let key = Tpm.key_of_swtpm path in - let contents = - Tpm.(contents |> deserialize |> update key body |> serialize) - in - let* () = with_xapi ~cache @@ VTPM.set_contents ~self ~contents in + let* () = push_vtpm ~cache vm_uuid key body in Cohttp_lwt_unix.Server.respond ~status:`No_content ~body:Cohttp_lwt.Body.empty () | `DELETE, path when path <> "/" -> - let* self = get_vtpm_ref () in + let* self = with_xapi_vtpm ~cache vm_uuid in let* contents = with_xapi ~cache @@ VTPM.get_contents ~self in let key = Tpm.key_of_swtpm path in let contents = From b16a018a29c6ae0ef7ecc71ff69250fdae17eac2 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 9 Feb 2024 11:51:18 +0000 Subject: [PATCH 34/99] CA-383867: Integrate the cache into xapi-guard's main loop Now the process creates a thread to read from disk and push vtpm events to xapi at its own pace, and integrates the disk-writing part into the callback of the deprivileged sockets. Special consideration was taken for the resume, when the deprivileged sockets and the write-to-cache function need to be integrated in a different way from the codepath that creates the sockets from the message-switch server. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/server_interface.ml | 19 ++-- ocaml/xapi-guard/src/main.ml | 115 ++++++++++++++++------- ocaml/xapi-guard/test/xapi_guard_test.ml | 3 +- 3 files changed, 92 insertions(+), 45 deletions(-) diff --git a/ocaml/xapi-guard/lib/server_interface.ml b/ocaml/xapi-guard/lib/server_interface.ml index 7bd6e79a8b1..1ca19a68820 100644 --- a/ocaml/xapi-guard/lib/server_interface.ml +++ b/ocaml/xapi-guard/lib/server_interface.ml @@ -51,8 +51,8 @@ let () = * this is only needed for syscalls that would otherwise block *) Lwt_unix.set_pool_size 16 -let with_xapi ~cache f = - Lwt_unix.with_timeout 120. (fun () -> SessionCache.with_session cache f) +let with_xapi ~cache ?(timeout = 120.) f = + Lwt_unix.with_timeout timeout (fun () -> SessionCache.with_session cache f) let serve_forever_lwt path callback = let conn_closed _ = () in @@ -121,17 +121,18 @@ let with_xapi_vtpm ~cache vm_uuid = let* uuid = with_xapi ~cache @@ Xen_api_lwt_unix.VTPM.get_uuid ~self in with_xapi ~cache @@ VTPM.get_by_uuid ~uuid -let push_vtpm ~cache vm_uuid path contents = +let push_vtpm ~cache (vm_uuid, _timestamp, key) contents = let* self = with_xapi_vtpm ~cache vm_uuid in let* old_contents = with_xapi ~cache @@ VTPM.get_contents ~self in let contents = Tpm.(old_contents |> deserialize |> update key contents |> serialize) in let* () = with_xapi ~cache @@ VTPM.set_contents ~self ~contents in - Lwt.return_unit + Lwt_result.return () -let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = +let serve_forever_lwt_callback_vtpm ~cache mutex persist vm_uuid _ req body = let uri = Cohttp.Request.uri req in + let timestamp = Mtime_clock.now () in (* in case the connection is interrupted/etc. we may still have pending operations, so use a per vTPM mutex to ensure we really only have 1 pending operation at a time for a vTPM *) @@ -150,7 +151,7 @@ let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = | `PUT, path when path <> "/" -> let* body = Cohttp_lwt.Body.to_string body in let key = Tpm.key_of_swtpm path in - let* () = push_vtpm ~cache vm_uuid key body in + let* () = persist (vm_uuid, timestamp, key) body in Cohttp_lwt_unix.Server.respond ~status:`No_content ~body:Cohttp_lwt.Body.empty () | `DELETE, path when path <> "/" -> @@ -168,7 +169,7 @@ let serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid _ req body = Cohttp_lwt_unix.Server.respond_string ~status:`Method_not_allowed ~body () (* Create a restricted RPC function and socket for a specific VM *) -let make_server_varstored ~cache path vm_uuid = +let make_server_varstored _persist ~cache path vm_uuid = let vm_uuid_str = Uuidm.to_string vm_uuid in let module Server = Xapi_idl_guard_varstored.Interface.RPC_API (Rpc_lwt.GenServer ()) in @@ -211,7 +212,7 @@ let make_server_varstored ~cache path vm_uuid = serve_forever_lwt_callback (Rpc_lwt.server Server.implementation) path |> serve_forever_lwt path -let make_server_vtpm_rest ~cache path vm_uuid = +let make_server_vtpm_rest persist ~cache path vm_uuid = let mutex = Lwt_mutex.create () in - let callback = serve_forever_lwt_callback_vtpm ~cache mutex vm_uuid in + let callback = serve_forever_lwt_callback_vtpm ~cache mutex persist vm_uuid in serve_forever_lwt path callback diff --git a/ocaml/xapi-guard/src/main.ml b/ocaml/xapi-guard/src/main.ml index 6979665a8a8..cfb3d3f258a 100644 --- a/ocaml/xapi-guard/src/main.ml +++ b/ocaml/xapi-guard/src/main.ml @@ -18,6 +18,8 @@ open Xapi_guard_server module Types = Xapi_guard.Types module SessionCache = Xen_api_lwt_unix.SessionCache +let ( let@ ) f x = f x + let daemon_name = "xapi-guard" module D = Debug.Make (struct let name = daemon_name end) @@ -97,7 +99,7 @@ let () = Xen_api_lwt_unix.SessionCache.destroy cache ) -let listen_for_vm {Persistent.vm_uuid; path; gid; typ} = +let listen_for_vm write_push {Persistent.vm_uuid; path; gid; typ} = let make_server = match typ with | Varstored -> @@ -110,19 +112,25 @@ let listen_for_vm {Persistent.vm_uuid; path; gid; typ} = (Types.Service.to_string typ) path vm_uuid_str ; let* () = safe_unlink path in - let* stop_server = make_server ~cache path vm_uuid in + let* stop_server = make_server write_push ~cache path vm_uuid in let* () = log_fds () in Hashtbl.add sockets path (stop_server, (vm_uuid, gid, typ)) ; let* () = Lwt_unix.chmod path 0o660 in Lwt_unix.chown path 0 gid -let resume () = +let resume ~vtpm_write_push ~uefi_write_push () = let* vms = Persistent.loadfrom recover_path in - let+ () = Lwt_list.iter_p listen_for_vm vms in + let listen_to_vm = function + | Persistent.{typ= Varstored; _} as vm -> + listen_for_vm uefi_write_push vm + | Persistent.{typ= Swtpm; _} as vm -> + listen_for_vm vtpm_write_push vm + in + let+ () = Lwt_list.iter_p listen_to_vm vms in D.debug "%s: completed" __FUNCTION__ (* caller here is trusted (xenopsd through message-switch) *) -let depriv_varstored_create dbg vm_uuid gid path = +let depriv_varstored_create write_push dbg vm_uuid gid path = if Hashtbl.mem sockets path then Lwt.return_error (Xapi_idl_guard_privileged.Interface.InternalError @@ -134,7 +142,9 @@ let depriv_varstored_create dbg vm_uuid gid path = @@ ( D.debug "[%s] creating deprivileged socket at %s, owned by group %d" dbg path gid ; - let* () = listen_for_vm {Persistent.path; vm_uuid; gid; typ= Varstored} in + let* () = + listen_for_vm write_push {Persistent.path; vm_uuid; gid; typ= Varstored} + in store_args sockets ) @@ -156,7 +166,7 @@ let depriv_varstored_destroy dbg gid path = D.debug "[%s] stopped server for gid %d and removed socket" dbg gid ; Lwt.return_unit -let depriv_swtpm_create dbg vm_uuid gid path = +let depriv_swtpm_create write_push dbg vm_uuid gid path = if Hashtbl.mem sockets path then Lwt.return_error (Xapi_idl_guard_privileged.Interface.InternalError @@ -168,7 +178,9 @@ let depriv_swtpm_create dbg vm_uuid gid path = @@ ( D.debug "[%s] creating deprivileged socket at %s, owned by group %d" dbg path gid ; - let* () = listen_for_vm {Persistent.path; vm_uuid; gid; typ= Swtpm} in + let* () = + listen_for_vm write_push {Persistent.path; vm_uuid; gid; typ= Swtpm} + in store_args sockets ) @@ -197,6 +209,9 @@ let depriv_swtpm_destroy dbg gid path = Lwt.return_unit (* TODO: these 2 APIs need to be updated to go through the generic interface *) +(* These 2 functions are only reachable from message-switch. They are part of + the control plane and be called when xapi controls the lifecycle of a VM, so + it's OK to assume it's available. *) let vtpm_set_contents dbg vtpm_uuid contents = let open Xen_api_lwt_unix in @@ -215,55 +230,85 @@ let vtpm_get_contents _dbg vtpm_uuid = @@ let* self = Server_interface.with_xapi ~cache @@ VTPM.get_by_uuid ~uuid in Server_interface.with_xapi ~cache @@ VTPM.get_contents ~self -let rpc_fn = +let rpc_fn ~vtpm_write_push ~uefi_write_push = let module Server = Xapi_idl_guard_privileged.Interface.RPC_API (Rpc_lwt.GenServer ()) in (* bind APIs *) - Server.varstore_create depriv_varstored_create ; + Server.varstore_create (depriv_varstored_create uefi_write_push) ; Server.varstore_destroy depriv_varstored_destroy ; - Server.vtpm_create depriv_swtpm_create ; + Server.vtpm_create (depriv_swtpm_create vtpm_write_push) ; Server.vtpm_destroy depriv_swtpm_destroy ; Server.vtpm_set_contents vtpm_set_contents ; Server.vtpm_get_contents vtpm_get_contents ; Rpc_lwt.server Server.implementation -let process body = +let process ~vtpm_write_push ~uefi_write_push body = let+ response = Xapi_guard.Dorpc.wrap_rpc Xapi_idl_guard_privileged.Interface.E.error (fun () -> let call = Jsonrpc.call_of_string body in D.debug "Received request from message-switch, method %s" call.Rpc.name ; - rpc_fn call + rpc_fn ~vtpm_write_push ~uefi_write_push call ) in Jsonrpc.string_of_response response +let retry_forever fname f = + let rec loop () = + let* () = + Lwt.catch f (function exn -> + D.info "%s failed with %s, retrying..." fname (Printexc.to_string exn) ; + Lwt_unix.sleep 0.5 + ) + in + (loop [@tailcall]) () + in + loop () + +let cache_reader with_watcher = retry_forever "cache watcher" with_watcher + let make_message_switch_server () = + let with_swtpm_push, with_watch = + Xapi_guard.Disk_cache.(setup Swtpm (Server_interface.push_vtpm ~cache)) + in let open Message_switch_lwt.Protocol_lwt in let wait_server, server_stopped = Lwt.task () in - let* result = - Server.listen ~process ~switch:!Xcp_client.switch_path - ~queue:Xapi_idl_guard_privileged.Interface.queue_name () + let@ vtpm_write_push = with_swtpm_push in + let uefi_write_push _ _ = + (* This is unused for the time being, added to be consistent with both + interfaces *) + Lwt.return_unit in - match Server.error_to_msg result with - | Ok t -> - Lwt_switch.add_hook (Some Server_interface.shutdown) (fun () -> - D.debug "Stopping message-switch queue server" ; - let+ () = Server.shutdown ~t () in - Lwt.wakeup server_stopped () - ) ; - (* best effort resume *) - let* () = - Lwt.catch resume (fun e -> - D.log_backtrace () ; - D.warn "Resume failed: %s" (Printexc.to_string e) ; - Lwt.return_unit - ) - in - wait_server - | Error (`Msg m) -> - Lwt.fail_with - (Printf.sprintf "Failed to listen on message-switch queue: %s" m) + let server = + let* result = + Server.listen + ~process:(process ~vtpm_write_push ~uefi_write_push) + ~switch:!Xcp_client.switch_path + ~queue:Xapi_idl_guard_privileged.Interface.queue_name () + in + match Server.error_to_msg result with + | Ok t -> + Lwt_switch.add_hook (Some Server_interface.shutdown) (fun () -> + D.debug "Stopping message-switch queue server" ; + let+ () = Server.shutdown ~t () in + Lwt.wakeup server_stopped () + ) ; + (* best effort resume *) + let* () = + Lwt.catch (resume ~vtpm_write_push ~uefi_write_push) (fun e -> + D.log_backtrace () ; + D.warn "Resume failed: %s" (Printexc.to_string e) ; + Lwt.return_unit + ) + in + wait_server + | Error (`Msg m) -> + Lwt.fail_with + (Printf.sprintf "Failed to listen on message-switch queue: %s" m) + in + let reader = cache_reader with_watch in + let* _ = Lwt.all [server; reader] in + Lwt.return_unit let main log_level = Debug.set_level log_level ; diff --git a/ocaml/xapi-guard/test/xapi_guard_test.ml b/ocaml/xapi-guard/test/xapi_guard_test.ml index c4996bef0c7..86efb713d29 100644 --- a/ocaml/xapi-guard/test/xapi_guard_test.ml +++ b/ocaml/xapi-guard/test/xapi_guard_test.ml @@ -80,9 +80,10 @@ let with_rpc f switch () = in (Lwt_switch.add_hook (Some switch) @@ fun () -> SessionCache.destroy cache) ; let path = Filename.concat tmp "socket" in + let push_nothing _ = Lwt_result.return () in (* Create an internal server on 'path', the socket that varstored would connect to *) let* stop_server = - Server_interface.make_server_varstored ~cache path vm_uuid + Server_interface.make_server_varstored push_nothing ~cache path vm_uuid in (* rpc simulates what varstored would do *) let uri = Uri.make ~scheme:"file" ~path () |> Uri.to_string in From 77d2b1dea6e5f5932b29e3d3e9ae4de294a873bc Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 20 Feb 2024 15:43:21 +0000 Subject: [PATCH 35/99] xapi-guard: reduce the number of calls to fetch the vTPM ref Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/server_interface.ml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ocaml/xapi-guard/lib/server_interface.ml b/ocaml/xapi-guard/lib/server_interface.ml index 1ca19a68820..ced743e501f 100644 --- a/ocaml/xapi-guard/lib/server_interface.ml +++ b/ocaml/xapi-guard/lib/server_interface.ml @@ -118,8 +118,7 @@ let with_xapi_vtpm ~cache vm_uuid = in raise (Failure msg) | self :: _ -> - let* uuid = with_xapi ~cache @@ Xen_api_lwt_unix.VTPM.get_uuid ~self in - with_xapi ~cache @@ VTPM.get_by_uuid ~uuid + Lwt.return self let push_vtpm ~cache (vm_uuid, _timestamp, key) contents = let* self = with_xapi_vtpm ~cache vm_uuid in From bf0e9c0bf38f4b3aab780c222d83a8865ca5d59b Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 23 Feb 2024 15:23:48 +0000 Subject: [PATCH 36/99] CA-383867: Add startup procedure to xapi-guard Because timestamps depend on a monotonic timestamp that depends on boot, files need to be renamed to ensure future writes have higher timestamps to be considered newer and be uploaded to xapi. On top of this, allows to report about remnant temporary files, delete invalid files and remove empty directories. Signed-off-by: Pau Ruiz Safont --- doc/content/xapi-guard/_index.md | 14 +++++ ocaml/xapi-guard/lib/disk_cache.ml | 85 +++++++++++++++++++++++++++-- ocaml/xapi-guard/lib/disk_cache.mli | 4 +- ocaml/xapi-guard/src/main.ml | 2 +- ocaml/xapi-guard/test/cache_test.ml | 5 +- 5 files changed, 101 insertions(+), 9 deletions(-) diff --git a/doc/content/xapi-guard/_index.md b/doc/content/xapi-guard/_index.md index 39d64a558e5..433f92f9d5e 100644 --- a/doc/content/xapi-guard/_index.md +++ b/doc/content/xapi-guard/_index.md @@ -35,6 +35,9 @@ This situation usually happens when xapi is being restarted as part of an update SWTPM, the vTPM daemon, reads the contents of the TPM from xapi-guard on startup, suspend, and resume. During normal operation SWTPM does not send read requests from xapi-guard. +Structure +--------- + The cache module consists of two Lwt threads, one that writes to disk, and another one that reads from disk. The writer is triggered when a VM writes to the vTPM. It never blocks if xapi is unreachable, but responds as soon as the data has been stored either by xapi or on the local disk, such that the VM receives a timely response to the write request. @@ -82,3 +85,14 @@ stateDiagram-v2 Engaged --> Engaged : Writer receives TPM, queue is not full Engaged --> Disengaged : Writer receives TPM, queue is full ``` + +Startup +------ + +At startup, there's a dedicated routine to transform the existing contents of the cache. +This is currently done because the timestamp reference change on each boot. +This means that the existing contents might have timestamps considered more recent than timestamps of writes coming from running events, leading to missing content updates. +This must be avoided and instead the updates with offending timestamps are renamed to a timestamp taken from the current timestamp, ensuring a consistent +ordering. +The routine is also used to keep a minimal file tree: unrecognised files are deleted, temporary files created to ensure atomic writes are left untouched, and empty directories are deleted. +This mechanism can be changed in the future to migrate to other formats. diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index b972654f085..5ededfbc81b 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -107,7 +107,7 @@ let get_all_contents root = let file = if found then Outdated valid_file else Latest valid_file in - loop (file :: acc, false) others + loop (file :: acc, true) others ) in let ordered = List.fast_sort (fun x y -> String.compare y x) contents in @@ -180,7 +180,6 @@ type channel = { IDEA: carryover: read contents of cache and "convert it" to the current run TODO: - - Add startup step to convert existing content to new time - Exponential backoff on xapi push error - Limit error logging on xapi push error: once per downtime is enough *) @@ -442,8 +441,86 @@ end = struct loop end +(** Module use to change the cache contents before the reader and writer start + running *) +module Setup : sig + val retime_cache_contents : Types.Service.t -> unit Lwt.t +end = struct + type file_action = + | Keep of file + | Delete of string + | Move of {from: string; into: string} + + let get_fs_action root now = function + | Latest ((uuid, timestamp, key), from) as latest -> + if Mtime.is_later ~than:now timestamp then + let timestamp = now in + let into = path_of_key root (uuid, timestamp, key) in + Move {from; into} + else + Keep latest + | Temporary _ as temp -> + Keep temp + | Invalid p | Outdated (_, p) -> + Delete p + + let commit __FUN = function + | Keep (Temporary p) -> + D.warn "%s: Found temporary file, ignoring '%s'" __FUN p ; + Lwt.return_unit + | Keep _ -> + Lwt.return_unit + | Delete p -> + D.info "%s: Deleting '%s'" __FUN p ; + Lwt_unix.unlink p + | Move {from; into} -> + D.info "%s: Moving '%s' to '%s'" __FUN from into ; + Lwt_unix.rename from into + + let rec delete_empty_dirs ~delete_root root = + (* Delete subdirectories, then *) + let* files = files_in root ~otherwise:(fun _ -> Lwt.return []) in + let* () = + Lwt_list.iter_p + (fun path -> + let* {st_kind; _} = Lwt_unix.stat path in + match st_kind with + | S_DIR -> + delete_empty_dirs ~delete_root:true path + | _ -> + Lwt.return_unit + ) + files + in + if not delete_root then + Lwt.return_unit + else + let* files = files_in root ~otherwise:(fun _ -> Lwt.return []) in + Lwt.catch + (fun () -> + if files = [] then + Lwt_unix.rmdir root + else + Lwt.return_unit + ) + (fun _ -> Lwt.return_unit) + + (* The code assumes it's the only with access to the disk cache while running *) + let retime_cache_contents typ = + let now = Mtime_clock.now () in + let root = cache_of typ in + let* contents = get_all_contents root in + let* () = + contents + |> List.map (get_fs_action root now) + |> Lwt_list.iter_p (commit __FUNCTION__) + in + delete_empty_dirs ~delete_root:false root +end + let setup typ direct = - let queue, push = Lwt_bounded_stream.create 4098 in + let* () = Setup.retime_cache_contents typ in + let queue, push = Lwt_bounded_stream.create 2 in let lock = Lwt_mutex.create () in let q = {queue; push; lock; state= Disengaged} in - (Writer.with_cache ~direct typ q, Watcher.watch ~direct typ q) + Lwt.return (Writer.with_cache ~direct typ q, Watcher.watch ~direct typ q) diff --git a/ocaml/xapi-guard/lib/disk_cache.mli b/ocaml/xapi-guard/lib/disk_cache.mli index 0f387d4ebd1..c8614bff31b 100644 --- a/ocaml/xapi-guard/lib/disk_cache.mli +++ b/ocaml/xapi-guard/lib/disk_cache.mli @@ -18,8 +18,10 @@ type t = Uuidm.t * Mtime.t * Types.Tpm.key val setup : Types.Service.t -> (t -> string -> (unit, exn) Lwt_result.t) - -> (((t -> string -> unit Lwt.t) -> 'a Lwt.t) -> 'a Lwt.t) + -> ( (((t -> string -> unit Lwt.t) -> 'a Lwt.t) -> 'a Lwt.t) * (unit -> unit Lwt.t) + ) + Lwt.t (** [setup service push_callback] Returns a local disk buffer for [service] which will use [push_callback] to push the elements to their final destination *) diff --git a/ocaml/xapi-guard/src/main.ml b/ocaml/xapi-guard/src/main.ml index cfb3d3f258a..f0e57121f07 100644 --- a/ocaml/xapi-guard/src/main.ml +++ b/ocaml/xapi-guard/src/main.ml @@ -268,7 +268,7 @@ let retry_forever fname f = let cache_reader with_watcher = retry_forever "cache watcher" with_watcher let make_message_switch_server () = - let with_swtpm_push, with_watch = + let* with_swtpm_push, with_watch = Xapi_guard.Disk_cache.(setup Swtpm (Server_interface.push_vtpm ~cache)) in let open Message_switch_lwt.Protocol_lwt in diff --git a/ocaml/xapi-guard/test/cache_test.ml b/ocaml/xapi-guard/test/cache_test.ml index ce63c8cec8d..e171de6b635 100644 --- a/ocaml/xapi-guard/test/cache_test.ml +++ b/ocaml/xapi-guard/test/cache_test.ml @@ -75,8 +75,7 @@ let retry_forever fname f = Lwt_unix.sleep 0.5 ) in - loop () - [@@tailcall] + (loop [@tailcall]) () in loop () @@ -145,7 +144,7 @@ let to_cache with_writer = let from_cache with_watcher = retry_forever "watcher" with_watcher let main () = - let with_writer, with_watcher = Xapi_guard.Disk_cache.(setup Swtpm log) in + let* with_writer, with_watcher = Xapi_guard.Disk_cache.(setup Swtpm log) in let reader = from_cache with_watcher in let writers = to_cache with_writer in let* _ = Lwt.all (reader :: writers) in From 4a67a207ad4b1601f52f09f909067972741183d2 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 27 Feb 2024 13:19:45 +0000 Subject: [PATCH 37/99] CA-383867: Prepare xapi-guard cache to change fallback on write error This is needed to a be able to disable the disk cache completely, maintaining previous behaviour if needed. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 55 +++++++++++++++++++----------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index 5ededfbc81b..ddc78a334be 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -241,8 +241,11 @@ end = struct let* outdated_contents = files_in_existing key_dir in let filename = key_dir // (Mtime.to_uint64_ns now |> Int64.to_string) in - (* 2. Write new timestamped content to cache, atomically, if needed; and - notify the other side, if needed *) + + (* 2. Try to push the changes, if possible. If it's not possible because of + the mode or a failure, write new timestamped content to cache, + atomically; and finally notify the other side if needed *) + (* Note that all queue operations must use while holding its mutex *) let persist () = persist_to ~filename ~contents in let persist_and_push () = let push () = @@ -260,27 +263,39 @@ end = struct let engage_and_persist exn = queue.state <- Engaged ; D.info "%s: Error on push. Reason: %s" __FUN (Printexc.to_string exn) ; - persist_and_push () + let* () = persist_and_push () in + Lwt_result.return () + in + let _fail exn = + Debug.log_backtrace exn (Backtrace.get exn) ; + Lwt_result.fail exn + in + let read_state_and_push on_exception () = + match queue.state with + | Direct -> + let* result = + Lwt.try_bind + (fun () -> direct (uuid, now, key) contents) + (function + | Ok () -> Lwt_result.return () | Error exn -> on_exception exn + ) + on_exception + in + Lwt.return result + | Engaged -> + let* () = persist_and_push () in + Lwt_result.return () + | Disengaged -> + let* () = persist () in + Lwt_result.return () in + let on_exception = engage_and_persist in + + let* result = with_lock queue.lock (read_state_and_push on_exception) in let* () = - with_lock queue.lock (fun () -> - match queue.state with - | Direct -> - Lwt.try_bind - (fun () -> direct (uuid, now, key) contents) - (function - | Ok () -> - Lwt.return_unit - | Error exn -> - engage_and_persist exn - ) - (function exn -> engage_and_persist exn) - | Engaged -> - persist_and_push () - | Disengaged -> - persist () - ) + match result with Ok () -> Lwt.return_unit | Error exn -> raise exn in + (* 4. Delete previous requests from filesystem *) let* _ = Lwt_list.map_p unlink_safe outdated_contents in Lwt.return_unit From b80357e5c4e4a7db23ff3576549ba9737c92007d Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 27 Feb 2024 15:14:32 +0000 Subject: [PATCH 38/99] CA-383867: Allow xapi-guard to disable the disk cache This is done through the fist point. Xapi_fist is not used directly because it needs to a new opam package, creating a lot of churn which is currently unwanted. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 14 ++++++++++++-- ocaml/xapi/xapi_fist.ml | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index ddc78a334be..60911410b8d 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -27,6 +27,15 @@ type t = Uuidm.t * Mtime.t * Types.Tpm.key let cache_of service = runtime_data // Types.Service.to_string service +let fistpoint () = + let name = "/tmp/fist_disable_xapi_guard_cache" in + Lwt.catch + (fun () -> + let* () = Lwt_unix.access name [Unix.F_OK] in + Lwt.return true + ) + (fun _ -> Lwt.return false) + let files_in dir ~otherwise = Lwt.catch (fun () -> @@ -266,7 +275,7 @@ end = struct let* () = persist_and_push () in Lwt_result.return () in - let _fail exn = + let fail exn = Debug.log_backtrace exn (Backtrace.get exn) ; Lwt_result.fail exn in @@ -289,7 +298,8 @@ end = struct let* () = persist () in Lwt_result.return () in - let on_exception = engage_and_persist in + let* cache_disabled = fistpoint () in + let on_exception = if cache_disabled then fail else engage_and_persist in let* result = with_lock queue.lock (read_state_and_push on_exception) in let* () = diff --git a/ocaml/xapi/xapi_fist.ml b/ocaml/xapi/xapi_fist.ml index 7798713e494..4f211185a92 100644 --- a/ocaml/xapi/xapi_fist.ml +++ b/ocaml/xapi/xapi_fist.ml @@ -160,3 +160,5 @@ let int_seed name : int option = let exchange_certificates_in_pool () : int option = let name = "exchange_certificates_in_pool" in int_seed name + +let disable_xapi_guard_cache () = fistpoint "disable_xapi_guard_cache" From 196c88c7701c6a34519be0c9e748d0cff3c9f7ba Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 28 Feb 2024 16:41:48 +0000 Subject: [PATCH 39/99] CA-383867: Make xapi-guard's cache aware of read requests Now all domains' vtpm read requests go through the cache. The read function is the same as before. There is no change in behaviour Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 50 +++++++++---- ocaml/xapi-guard/lib/disk_cache.mli | 12 ++-- ocaml/xapi-guard/lib/server_interface.ml | 19 +++-- ocaml/xapi-guard/src/main.ml | 42 ++++++----- ocaml/xapi-guard/test/cache_test.ml | 91 ++++++++++++++++++------ 5 files changed, 150 insertions(+), 64 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index 60911410b8d..60ac2f1d0fa 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -195,19 +195,21 @@ type channel = { module Writer : sig val with_cache : - direct:(t -> string -> (unit, exn) Lwt_result.t) + direct: + (t -> (string, exn) Lwt_result.t) + * (t -> string -> (unit, exn) Lwt_result.t) -> Types.Service.t -> channel - -> ((t -> string -> unit Lwt.t) -> 'a Lwt.t) + -> ((t -> string Lwt.t) * (t -> string -> unit Lwt.t) -> 'a Lwt.t) -> 'a Lwt.t (** [with_cache ~direct typ queue context] creates a cache for content of - type [typ]. The cache is writable through the function [context], which - is provided a writing function to persist to the cache. It uses [channel] - to push events to + type [typ]. The cache is readable and writable through the function + [context], which is provided a reading and writing functions [direct]. + It uses [channel] to push events to Example: - Xapi_guard.Disk_cache.(Writer.with_cache ~direct:(upload session_cache) Tpm channel) - @@ fun write_tpm -> write_tpm (uuid, time, key) contents + Xapi_guard.Disk_cache.(Writer.with_cache ~direct:(read, upload) Tpm channel) + @@ fun read_tpm, write_tpm -> write_tpm (uuid, time, key) contents *) end = struct let mkdir_p ?(perm = 0o755) path = @@ -241,9 +243,30 @@ end = struct in files_in dir ~otherwise:create_dir + let fail exn = + Debug.log_backtrace exn (Backtrace.get exn) ; + Lwt_result.fail exn + + let read_contents ~direct _root (uuid, now, key) = + let read, _ = direct in + let* result = + Lwt.try_bind + (fun () -> read (uuid, now, key)) + (function + | Ok contents -> Lwt_result.return contents | Error exn -> fail exn + ) + fail + in + match result with + | Ok contents -> + Lwt.return contents + | Error exn -> + raise exn + let write_contents ~direct root queue (uuid, now, key) contents = let __FUN = __FUNCTION__ in + let _, direct = direct in let key_str = Types.Tpm.(serialize_key key |> string_of_int) in let key_dir = root // Uuidm.(to_string uuid) // key_str in (* 1. Record existing requests in cache *) @@ -275,10 +298,6 @@ end = struct let* () = persist_and_push () in Lwt_result.return () in - let fail exn = - Debug.log_backtrace exn (Backtrace.get exn) ; - Lwt_result.fail exn - in let read_state_and_push on_exception () = match queue.state with | Direct -> @@ -313,7 +332,7 @@ end = struct let with_cache ~direct typ queue f = let root = cache_of typ in let* () = mkdir_p root ~perm:0o700 in - f (write_contents ~direct root queue) + f (read_contents ~direct root, write_contents ~direct root queue) end module Watcher : sig @@ -543,9 +562,12 @@ end = struct delete_empty_dirs ~delete_root:false root end -let setup typ direct = +let setup typ read write = let* () = Setup.retime_cache_contents typ in let queue, push = Lwt_bounded_stream.create 2 in let lock = Lwt_mutex.create () in let q = {queue; push; lock; state= Disengaged} in - Lwt.return (Writer.with_cache ~direct typ q, Watcher.watch ~direct typ q) + Lwt.return + ( Writer.with_cache ~direct:(read, write) typ q + , Watcher.watch ~direct:write typ q + ) diff --git a/ocaml/xapi-guard/lib/disk_cache.mli b/ocaml/xapi-guard/lib/disk_cache.mli index c8614bff31b..08c345615f5 100644 --- a/ocaml/xapi-guard/lib/disk_cache.mli +++ b/ocaml/xapi-guard/lib/disk_cache.mli @@ -17,11 +17,15 @@ type t = Uuidm.t * Mtime.t * Types.Tpm.key val setup : Types.Service.t + -> (t -> (string, exn) Lwt_result.t) -> (t -> string -> (unit, exn) Lwt_result.t) - -> ( (((t -> string -> unit Lwt.t) -> 'a Lwt.t) -> 'a Lwt.t) + -> ( ( ((t -> string Lwt.t) * (t -> string -> unit Lwt.t) -> 'a Lwt.t) + -> 'a Lwt.t + ) * (unit -> unit Lwt.t) ) Lwt.t -(** [setup service push_callback] Returns a local disk buffer for [service] - which will use [push_callback] to push the elements to their final - destination *) +(** [setup service read_callback push_callback] Returns a local disk buffer for + [service] which will use [push_callback] to push the elements to their + final destination and [read_callback] to read elements if they are not in + the buffer. *) diff --git a/ocaml/xapi-guard/lib/server_interface.ml b/ocaml/xapi-guard/lib/server_interface.ml index ced743e501f..0884c2bf1b2 100644 --- a/ocaml/xapi-guard/lib/server_interface.ml +++ b/ocaml/xapi-guard/lib/server_interface.ml @@ -129,7 +129,14 @@ let push_vtpm ~cache (vm_uuid, _timestamp, key) contents = let* () = with_xapi ~cache @@ VTPM.set_contents ~self ~contents in Lwt_result.return () -let serve_forever_lwt_callback_vtpm ~cache mutex persist vm_uuid _ req body = +let read_vtpm ~cache (vm_uuid, _timestamp, key) = + let* self = with_xapi_vtpm ~cache vm_uuid in + let* contents = with_xapi ~cache @@ VTPM.get_contents ~self in + let body = Tpm.(contents |> deserialize |> lookup ~key) in + Lwt_result.return body + +let serve_forever_lwt_callback_vtpm ~cache mutex (read, persist) vm_uuid _ req + body = let uri = Cohttp.Request.uri req in let timestamp = Mtime_clock.now () in (* in case the connection is interrupted/etc. we may still have pending operations, @@ -139,10 +146,8 @@ let serve_forever_lwt_callback_vtpm ~cache mutex persist vm_uuid _ req body = (* TODO: some logging *) match (Cohttp.Request.meth req, Uri.path uri) with | `GET, path when path <> "/" -> - let* self = with_xapi_vtpm ~cache vm_uuid in - let* contents = with_xapi ~cache @@ VTPM.get_contents ~self in let key = Tpm.key_of_swtpm path in - let body = Tpm.(contents |> deserialize |> lookup ~key) in + let* body = read (vm_uuid, timestamp, key) in let headers = Cohttp.Header.of_list [("Content-Type", "application/octet-stream")] in @@ -211,7 +216,9 @@ let make_server_varstored _persist ~cache path vm_uuid = serve_forever_lwt_callback (Rpc_lwt.server Server.implementation) path |> serve_forever_lwt path -let make_server_vtpm_rest persist ~cache path vm_uuid = +let make_server_vtpm_rest read_write ~cache path vm_uuid = let mutex = Lwt_mutex.create () in - let callback = serve_forever_lwt_callback_vtpm ~cache mutex persist vm_uuid in + let callback = + serve_forever_lwt_callback_vtpm ~cache mutex read_write vm_uuid + in serve_forever_lwt path callback diff --git a/ocaml/xapi-guard/src/main.ml b/ocaml/xapi-guard/src/main.ml index f0e57121f07..9fb40aa038b 100644 --- a/ocaml/xapi-guard/src/main.ml +++ b/ocaml/xapi-guard/src/main.ml @@ -99,7 +99,7 @@ let () = Xen_api_lwt_unix.SessionCache.destroy cache ) -let listen_for_vm write_push {Persistent.vm_uuid; path; gid; typ} = +let listen_for_vm read_write {Persistent.vm_uuid; path; gid; typ} = let make_server = match typ with | Varstored -> @@ -112,19 +112,19 @@ let listen_for_vm write_push {Persistent.vm_uuid; path; gid; typ} = (Types.Service.to_string typ) path vm_uuid_str ; let* () = safe_unlink path in - let* stop_server = make_server write_push ~cache path vm_uuid in + let* stop_server = make_server read_write ~cache path vm_uuid in let* () = log_fds () in Hashtbl.add sockets path (stop_server, (vm_uuid, gid, typ)) ; let* () = Lwt_unix.chmod path 0o660 in Lwt_unix.chown path 0 gid -let resume ~vtpm_write_push ~uefi_write_push () = +let resume ~vtpm_read_write ~uefi_read_write () = let* vms = Persistent.loadfrom recover_path in let listen_to_vm = function | Persistent.{typ= Varstored; _} as vm -> - listen_for_vm uefi_write_push vm + listen_for_vm uefi_read_write vm | Persistent.{typ= Swtpm; _} as vm -> - listen_for_vm vtpm_write_push vm + listen_for_vm vtpm_read_write vm in let+ () = Lwt_list.iter_p listen_to_vm vms in D.debug "%s: completed" __FUNCTION__ @@ -166,7 +166,7 @@ let depriv_varstored_destroy dbg gid path = D.debug "[%s] stopped server for gid %d and removed socket" dbg gid ; Lwt.return_unit -let depriv_swtpm_create write_push dbg vm_uuid gid path = +let depriv_swtpm_create read_write dbg vm_uuid gid path = if Hashtbl.mem sockets path then Lwt.return_error (Xapi_idl_guard_privileged.Interface.InternalError @@ -179,7 +179,7 @@ let depriv_swtpm_create write_push dbg vm_uuid gid path = ( D.debug "[%s] creating deprivileged socket at %s, owned by group %d" dbg path gid ; let* () = - listen_for_vm write_push {Persistent.path; vm_uuid; gid; typ= Swtpm} + listen_for_vm read_write {Persistent.path; vm_uuid; gid; typ= Swtpm} in store_args sockets ) @@ -230,25 +230,25 @@ let vtpm_get_contents _dbg vtpm_uuid = @@ let* self = Server_interface.with_xapi ~cache @@ VTPM.get_by_uuid ~uuid in Server_interface.with_xapi ~cache @@ VTPM.get_contents ~self -let rpc_fn ~vtpm_write_push ~uefi_write_push = +let rpc_fn ~vtpm_read_write ~uefi_read_write = let module Server = Xapi_idl_guard_privileged.Interface.RPC_API (Rpc_lwt.GenServer ()) in (* bind APIs *) - Server.varstore_create (depriv_varstored_create uefi_write_push) ; + Server.varstore_create (depriv_varstored_create uefi_read_write) ; Server.varstore_destroy depriv_varstored_destroy ; - Server.vtpm_create (depriv_swtpm_create vtpm_write_push) ; + Server.vtpm_create (depriv_swtpm_create vtpm_read_write) ; Server.vtpm_destroy depriv_swtpm_destroy ; Server.vtpm_set_contents vtpm_set_contents ; Server.vtpm_get_contents vtpm_get_contents ; Rpc_lwt.server Server.implementation -let process ~vtpm_write_push ~uefi_write_push body = +let process ~vtpm_read_write ~uefi_read_write body = let+ response = Xapi_guard.Dorpc.wrap_rpc Xapi_idl_guard_privileged.Interface.E.error (fun () -> let call = Jsonrpc.call_of_string body in D.debug "Received request from message-switch, method %s" call.Rpc.name ; - rpc_fn ~vtpm_write_push ~uefi_write_push call + rpc_fn ~vtpm_read_write ~uefi_read_write call ) in Jsonrpc.string_of_response response @@ -268,21 +268,25 @@ let retry_forever fname f = let cache_reader with_watcher = retry_forever "cache watcher" with_watcher let make_message_switch_server () = - let* with_swtpm_push, with_watch = - Xapi_guard.Disk_cache.(setup Swtpm (Server_interface.push_vtpm ~cache)) + let* with_swtpm_cache, with_watch = + Xapi_guard.Disk_cache.( + setup Swtpm + Server_interface.(read_vtpm ~cache) + Server_interface.(push_vtpm ~cache) + ) in let open Message_switch_lwt.Protocol_lwt in let wait_server, server_stopped = Lwt.task () in - let@ vtpm_write_push = with_swtpm_push in - let uefi_write_push _ _ = + let@ vtpm_read_write = with_swtpm_cache in + let uefi_read_write = (* This is unused for the time being, added to be consistent with both interfaces *) - Lwt.return_unit + ((fun _ -> Lwt.return ""), fun _ _ -> Lwt.return_unit) in let server = let* result = Server.listen - ~process:(process ~vtpm_write_push ~uefi_write_push) + ~process:(process ~vtpm_read_write ~uefi_read_write) ~switch:!Xcp_client.switch_path ~queue:Xapi_idl_guard_privileged.Interface.queue_name () in @@ -295,7 +299,7 @@ let make_message_switch_server () = ) ; (* best effort resume *) let* () = - Lwt.catch (resume ~vtpm_write_push ~uefi_write_push) (fun e -> + Lwt.catch (resume ~vtpm_read_write ~uefi_read_write) (fun e -> D.log_backtrace () ; D.warn "Resume failed: %s" (Printexc.to_string e) ; Lwt.return_unit diff --git a/ocaml/xapi-guard/test/cache_test.ml b/ocaml/xapi-guard/test/cache_test.ml index e171de6b635..97b144839a6 100644 --- a/ocaml/xapi-guard/test/cache_test.ml +++ b/ocaml/xapi-guard/test/cache_test.ml @@ -5,7 +5,9 @@ let ( let* ) = Lwt.bind module Tpm = Xapi_guard.Types.Tpm module TPMs = struct - let tpms_created = Atomic.make 1 + let writes_created = Atomic.make 1 + + let reads_created = Atomic.make 1 let request_persist uuid write = let __FUN = __FUNCTION__ in @@ -13,18 +15,35 @@ module TPMs = struct let key = Tpm.deserialize_key (Random.int 3) in let time = Mtime_clock.now () in - let serial_n = Atomic.fetch_and_add tpms_created 1 in + let serial_n = Atomic.fetch_and_add writes_created 1 in let contents = Printf.sprintf "contents %s" (Mtime.to_uint64_ns time |> Int64.to_string) in let* () = Logs_lwt.app (fun m -> - m "%s: Content № %i created: %a/%i/%a" __FUN serial_n Uuidm.pp uuid + m "%s: Write № %i requested: %a/%i/%a" __FUN serial_n Uuidm.pp uuid Tpm.(serialize_key key) Mtime.pp time ) in write (uuid, time, key) contents + + let request_read uuid read = + let __FUN = __FUNCTION__ in + + let key = Tpm.deserialize_key (Random.int 3) in + + let time = Mtime_clock.now () in + let serial_n = Atomic.fetch_and_add reads_created 1 in + let* () = + Logs_lwt.app (fun m -> + m "%s: Read № %i requested: %a/%i/%a" __FUN serial_n Uuidm.pp uuid + Tpm.(serialize_key key) + Mtime.pp time + ) + in + let* () = Lwt_unix.sleep 0.05 in + read (uuid, time, key) end let lwt_reporter () = @@ -79,9 +98,13 @@ let retry_forever fname f = in loop () -let max_sent = 128 +let max_writes = 128 + +let max_reads = 500_000 + +let received_writes = ref 0 -let received = ref 0 +let received_reads = ref 0 let throttled_reads = Mtime.Span.(200 * ms) @@ -99,7 +122,7 @@ let should_fail () : bool = let elapsed = Mtime.span epoch (Mtime_clock.now ()) in polarity elapsed -let log (uuid, timestamp, key) content : (unit, exn) Result.t Lwt.t = +let log_write (uuid, timestamp, key) content = let __FUN = __FUNCTION__ in let ( let* ) = Lwt_result.bind in let maybe_fail () = @@ -110,43 +133,69 @@ let log (uuid, timestamp, key) content : (unit, exn) Result.t Lwt.t = Lwt_result.return () in let* () = maybe_fail () in - received := !received + 1 ; + received_writes := !received_writes + 1 ; Logs_lwt.app (fun m -> - m "%s Content № %i detected: %a/%i/%a" __FUN !received Uuidm.pp uuid + m "%s Write № %i detected: %a/%i/%a" __FUN !received_writes Uuidm.pp uuid Tpm.(serialize_key key) Mtime.pp timestamp ) |> ok -let to_cache with_writer = +let log_read (uuid, timestamp, key) = + let __FUN = __FUNCTION__ in + let ( let* ) = Lwt_result.bind in + received_reads := !received_reads + 1 ; + let* () = + Logs_lwt.app (fun m -> + m "%s Read to source № %i detected: %a/%i/%a" __FUN !received_reads + Uuidm.pp uuid + Tpm.(serialize_key key) + Mtime.pp timestamp + ) + |> ok + in + Lwt_result.return "yes" + +let to_cache with_read_writes = let __FUN = __FUNCTION__ in let elapsed = Mtime_clock.counter () in - let rec loop_and_stop uuid sent () = + let persist uuid (_, write_tpm) = TPMs.request_persist uuid write_tpm in + let read uuid (read_tpm, _) = + let* contents = TPMs.request_read uuid read_tpm in + Logs_lwt.app (fun m -> m "%s Read received: '%s'" __FUN contents) + in + let rec loop_and_stop f name uuid max sent = let sent = sent + 1 in - - let@ write_tpm = with_writer in - let* () = TPMs.request_persist uuid write_tpm in - if sent >= max_sent then + let@ read_write = with_read_writes in + let* () = f uuid read_write in + if sent >= max then Logs_lwt.app (fun m -> - m "%s: Stopping requests after %i writes" __FUN sent + m "%s: Stopping requests after %i %ss" __FUN sent name ) else if Mtime.Span.compare (Mtime_clock.count elapsed) throttled_reads > 0 then let* () = Lwt_unix.sleep 0.1 in - loop_and_stop uuid sent () + loop_and_stop f name uuid max sent else let* () = Lwt.pause () in - loop_and_stop uuid sent () + loop_and_stop f name uuid max sent in - List.init 4 (fun _ -> Uuidm.(v `V4)) - |> List.map (fun uuid -> loop_and_stop uuid 0 ()) + let vms = List.init 4 (fun _ -> Uuidm.(v `V4)) in + + List.concat + [ + List.map (fun uuid -> loop_and_stop persist "write" uuid max_writes 0) vms + ; List.map (fun uuid -> loop_and_stop read "read" uuid max_reads 0) vms + ] let from_cache with_watcher = retry_forever "watcher" with_watcher let main () = - let* with_writer, with_watcher = Xapi_guard.Disk_cache.(setup Swtpm log) in + let* with_read_writes, with_watcher = + Xapi_guard.Disk_cache.(setup Swtpm log_read log_write) + in let reader = from_cache with_watcher in - let writers = to_cache with_writer in + let writers = to_cache with_read_writes in let* _ = Lwt.all (reader :: writers) in Lwt.return_unit From 0a0d55db0bec49c601de27eeed7bc5a52eb75af0 Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Fri, 23 Feb 2024 10:29:35 +0000 Subject: [PATCH 40/99] Create a new python3 directory for python3-only scripts This is intended as the start of a new directory structure. Splitting the python3-only scripts into a new directory gives a simple way to exclude them from python2 tests and coverage. Signed-off-by: Steven Woods --- .codecov.yml | 41 +++++++++++++++++++++-- .github/workflows/main.yml | 15 ++++++++- Makefile | 1 + pyproject.toml | 1 + python3/Makefile | 9 +++++ python3/README.md | 12 +++++++ {scripts => python3/packages}/observer.py | 2 -- scripts/Makefile | 1 - 8 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 python3/Makefile create mode 100644 python3/README.md rename {scripts => python3/packages}/observer.py (93%) diff --git a/.codecov.yml b/.codecov.yml index 8380434a2a5..9be7955160d 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -158,6 +158,31 @@ coverage: # threshold: 20% + python3: + + # + # The python3 limit applies to: + # ----------------------------- + # + # - python3/** + # - excluding: **/test_*.py + # + paths: ["python3/**", "!**/test_*.py"] + + # + # For python3/** (excluding tests): + # + # For python3, coverage should not be reduced compared to its base: + # + target: auto + + # + # Exception: the threshold value given is allowed + # + # Allows for not covering 20% if the changed lines of the PR: + # + threshold: 20% + # Checks each Python version separately: python-3.11: flags: ["python3.11"] @@ -175,18 +200,26 @@ coverage: # Python modules and scripts below scripts/ (excluding tests) # scripts: + paths: ["scripts/**", "!**/test_*.py"] target: 48% threshold: 2% - paths: ["scripts/**", "!**/test_*.py"] # - # Python modules and scripts below ocaml/ + # Python modules and scripts below ocaml/ (excluding tests) # ocaml: paths: ["ocaml/**", "!**/test_*.py"] target: 51% threshold: 3% + # + # Python modules and scripts below python3/ (excluding tests) + # + python3: + paths: ["python3/**", "!**/test_*.py"] + target: 48% + threshold: 2% + # # Test files # @@ -239,6 +272,10 @@ component_management: - "ocaml/xapi-storage-script/**" - "!**/test_*.py" + - component_id: python3 + name: python3 + paths: ["python3/**", "!**/test_*.py"] + - component_id: test_cases name: test_cases paths: ["**/test_*.py"] diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cadf84c35c4..7b660722b20 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,8 @@ jobs: - name: Install common dependencies for Python ${{matrix.python-version}} run: pip install future mock pytest-coverage pytest-mock - - name: Run Pytest and get code coverage for Codecov + - name: Run Pytest for python 2 and get code coverage for Codecov + if: ${{ matrix.python-version == '2.7' }} run: > pytest --cov=scripts --cov=ocaml/xcp-rrdd @@ -67,6 +68,18 @@ jobs: env: PYTHONDEVMODE: yes + - name: Run Pytest for python 3 and get code coverage for Codecov + if: ${{ matrix.python-version != '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ + scripts/ ocaml/xcp-rrdd python3/ -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + - name: Upload Python ${{matrix.python-version}} coverage report to Codecov uses: codecov/codecov-action@v3 with: diff --git a/Makefile b/Makefile index bcfc5b9eb78..921e65923de 100644 --- a/Makefile +++ b/Makefile @@ -147,6 +147,7 @@ install: build doc sdk doc-json mkdir -p $(DESTDIR)/etc/bash_completion.d # ocaml/xapi make -C scripts install + make -C python3 install cp -f _build/install/default/bin/xapi $(DESTDIR)$(OPTDIR)/bin/xapi scripts/install.sh 755 ocaml/quicktest/quicktest $(DESTDIR)$(OPTDIR)/debug cp -f _build/install/default/bin/quicktestbin $(DESTDIR)$(OPTDIR)/debug/quicktestbin diff --git a/pyproject.toml b/pyproject.toml index b65a36bb062..d05d028fbad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,7 @@ inputs = [ "scripts/examples/python", "scripts/yum-plugins", "scripts/*.py", + "python3/packages/*.py", # To be added later, # when converted to Python3-compatible syntax: diff --git a/python3/Makefile b/python3/Makefile new file mode 100644 index 00000000000..480aee5dfeb --- /dev/null +++ b/python3/Makefile @@ -0,0 +1,9 @@ +include ../config.mk + +SITE3_DIR=$(shell python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") + +IDATA=install -m 644 + +install: + mkdir -p $(DESTDIR)$(SITE3_DIR) + $(IDATA) packages/observer.py $(DESTDIR)$(SITE3_DIR)/ diff --git a/python3/README.md b/python3/README.md new file mode 100644 index 00000000000..e2bf8dcc464 --- /dev/null +++ b/python3/README.md @@ -0,0 +1,12 @@ +# Python3 Scripts + +This directory is intended for scripts that only run on python3. As more scripts are +ported to python3 this directory should start to fill up. The intended structure of +the directory is as follows: + +- bin: This contains files to be installed in bin and are meant to be run by users +- libexec: This contains files to be installed in libexec and are meant to only be +run by xapi and other daemons. +- packages: This contains files to be installed in python's site-packages and are meant +to be modules and packages to be imported by other scripts or executed via python3 -m +- plugins: This contains files that are meant to be xapi plugins diff --git a/scripts/observer.py b/python3/packages/observer.py similarity index 93% rename from scripts/observer.py rename to python3/packages/observer.py index 96013361866..a2538304e19 100644 --- a/scripts/observer.py +++ b/python3/packages/observer.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 - if __name__ == '__main__': # run a program passed as parameter, with its original arguments import runpy diff --git a/scripts/Makefile b/scripts/Makefile index 8f07e91efe7..17c8e383fac 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -177,7 +177,6 @@ endif sed -i 's/#!\/usr\/bin\/python/#!\/usr\/bin\/python3/' $(DESTDIR)$(SITE3_DIR)/XenAPIPlugin.py $(IDATA) examples/python/XenAPI/XenAPI.py $(DESTDIR)$(SITE3_DIR)/ $(IDATA) examples/python/inventory.py $(DESTDIR)$(SITE3_DIR)/ - $(IDATA) observer.py $(DESTDIR)$(SITE3_DIR)/ $(IPROG) examples/python/echo.py $(DESTDIR)$(PLUGINDIR)/echo $(IPROG) examples/python/shell.py $(DESTDIR)$(LIBEXECDIR)/shell.py # poweron From 7e4c476103a9f0da721c9025f0e4721fbc486a79 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 28 Feb 2024 17:34:22 +0000 Subject: [PATCH 41/99] CA-383867: xapi-guard's cache contents are used in read requests For domains requesting the TPM's contents, the xapi-guards returns the contents in the cache, if they are available from in-flight requests. It falls back to xapi if that couldn't be possible. The cache doesn't try to provide any availability for reads, like it does for writes. This means that if swtpm issues a read request while xapi is offline, the request will fail, as it happened before this change. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 113 +++++++++++++++++------------ 1 file changed, 68 insertions(+), 45 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index 60ac2f1d0fa..f3b83739f21 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -97,35 +97,33 @@ let path_is_temp path = let temp_of_path path = path ^ ".pre" -let get_all_contents root = - let classify contents = - let rec loop (acc, found) = function - | [] -> - List.rev acc - | latest :: others -> ( - match key_of_path latest with - | None -> - let file = - if path_is_temp latest then - Temporary latest - else - Invalid latest - in - loop (file :: acc, found) others - | Some valid_file -> - let file = - if found then Outdated valid_file else Latest valid_file - in - loop (file :: acc, true) others - ) - in - let ordered = List.fast_sort (fun x y -> String.compare y x) contents in - loop ([], false) ordered +let sort_updates contents = + let rec loop (acc, found) = function + | [] -> + List.rev acc + | latest :: others -> ( + match key_of_path latest with + | None -> + let file = + if path_is_temp latest then + Temporary latest + else + Invalid latest + in + loop (file :: acc, found) others + | Some valid_file -> + let file = if found then Outdated valid_file else Latest valid_file in + loop (file :: acc, true) others + ) in + let ordered = List.fast_sort (fun x y -> String.compare y x) contents in + loop ([], false) ordered + +let get_all_contents root = let empty = Fun.const (Lwt.return []) in let contents_of_key key = let* contents = files_in key ~otherwise:empty in - Lwt.return (classify contents) + Lwt.return (sort_updates contents) in let* tpms = files_in root ~otherwise:empty in let* files = @@ -138,6 +136,12 @@ let get_all_contents root = in Lwt.return List.(concat (concat files)) +(** Warning, may raise Unix.Unix_error *) +let read_from ~filename = + let flags = Unix.[O_RDONLY] in + let perm = 0o000 in + Lwt_io.with_file ~flags ~perm ~mode:Input filename Lwt_io.read + let persist_to ~filename:f_path ~contents = let atomic_write_to_file ~perm f = let tmp_path = temp_of_path f_path in @@ -247,21 +251,46 @@ end = struct Debug.log_backtrace exn (Backtrace.get exn) ; Lwt_result.fail exn - let read_contents ~direct _root (uuid, now, key) = - let read, _ = direct in - let* result = - Lwt.try_bind - (fun () -> read (uuid, now, key)) - (function - | Ok contents -> Lwt_result.return contents | Error exn -> fail exn - ) - fail + let read_contents ~direct root (uuid, now, key) = + let read_remote () = + let read, _ = direct in + let* result = + Lwt.try_bind + (fun () -> read (uuid, now, key)) + (function + | Ok contents -> Lwt_result.return contents | Error exn -> fail exn + ) + fail + in + match result with + | Ok contents -> + Lwt.return contents + | Error exn -> + raise exn + in + + let key_str = Types.Tpm.(serialize_key key |> string_of_int) in + let key_dir = root // Uuidm.(to_string uuid) // key_str in + + (* 1. Get updates *) + let* contents = files_in key_dir ~otherwise:(fun _ -> Lwt.return []) in + let updates = sort_updates contents in + + (* 2. Pick latest *) + let only_latest = function + | Latest (_, p) -> + Either.Left p + | Temporary p | Outdated (_, p) | Invalid p -> + Right p + in + let latest, _ = List.partition_map only_latest updates in + + (* 3. fall back to remote read if needed *) + let get_contents path = + Lwt.catch (fun () -> read_from ~filename:path) (fun _ -> read_remote ()) in - match result with - | Ok contents -> - Lwt.return contents - | Error exn -> - raise exn + + match latest with path :: _ -> get_contents path | [] -> read_remote () let write_contents ~direct root queue (uuid, now, key) contents = let __FUN = __FUNCTION__ in @@ -367,12 +396,6 @@ end = struct in Lwt.return latest - (** Warning, may raise Unix.Unix_error *) - let read_from ~filename = - let flags = Unix.[O_RDONLY] in - let perm = 0o000 in - Lwt_io.with_file ~flags ~perm ~mode:Input filename Lwt_io.read - let retry_push push (uuid, timestamp, key) contents = let __FUN = __FUNCTION__ in let push' () = push (uuid, timestamp, key) contents in From dcb8042cfd4b432e84643442fc53c04c5aaae926 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Thu, 29 Feb 2024 10:11:23 +0000 Subject: [PATCH 42/99] CA-383867: xapi-guard cache sort files by timestamp Previously, they were sorted by string order, which in rare cases might lead to erroneous ordering Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 44 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index f3b83739f21..0f0a6e2c248 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -98,26 +98,34 @@ let path_is_temp path = let temp_of_path path = path ^ ".pre" let sort_updates contents = - let rec loop (acc, found) = function + let classify elem = + match key_of_path elem with + | None -> + let file = + if path_is_temp elem then + Temporary elem + else + Invalid elem + in + Either.Right file + | Some valid_file -> + Either.Left valid_file + in + let valid_files, invalid = List.partition_map classify contents in + + let valid = + let ordered = + List.fast_sort + (fun ((_, x, _), _) ((_, y, _), _) -> Mtime.compare y x) + valid_files + in + match ordered with | [] -> - List.rev acc - | latest :: others -> ( - match key_of_path latest with - | None -> - let file = - if path_is_temp latest then - Temporary latest - else - Invalid latest - in - loop (file :: acc, found) others - | Some valid_file -> - let file = if found then Outdated valid_file else Latest valid_file in - loop (file :: acc, true) others - ) + [] + | latest :: outdated -> + Latest latest :: List.map (fun outdated -> Outdated outdated) outdated in - let ordered = List.fast_sort (fun x y -> String.compare y x) contents in - loop ([], false) ordered + List.concat [valid; invalid] let get_all_contents root = let empty = Fun.const (Lwt.return []) in From 130d7b9934792795044edc5d4d39ddae59d46a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 29 Feb 2024 10:36:33 +0000 Subject: [PATCH 43/99] fix(ci): Move differential shellcheck to its own workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is not supported in scheduled runs, and fails. Use unique key for shellcheck group Signed-off-by: Edwin Török --- .github/workflows/main.yml | 22 --------------------- .github/workflows/shellcheck.yaml | 32 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/shellcheck.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f52439cd69..cadf84c35c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,28 +12,6 @@ concurrency: # On new push, cancel old workflows from the same PR, branch or ta cancel-in-progress: true jobs: - # https://www.shellcheck.net/wiki/GitHub-Actions - # https://github.com/redhat-plumbers-in-action/differential-shellcheck?tab=readme-ov-file#usage - shell-test: - name: Differential ShellCheck - runs-on: ubuntu-latest - - permissions: - security-events: write - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - -# If needed severity levels can be controlled here -# severity: warning - - name: Differential ShellCheck - uses: redhat-plumbers-in-action/differential-shellcheck@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - python-test: name: Python tests runs-on: ubuntu-22.04 diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml new file mode 100644 index 00000000000..8725ae60710 --- /dev/null +++ b/.github/workflows/shellcheck.yaml @@ -0,0 +1,32 @@ +name: ShellCheck + +on: + push: + pull_request: + +concurrency: # On new push, cancel old workflows from the same PR, branch or tag: + group: sc-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + # https://www.shellcheck.net/wiki/GitHub-Actions + # https://github.com/redhat-plumbers-in-action/differential-shellcheck?tab=readme-ov-file#usage + shell-test: + name: Differential ShellCheck + runs-on: ubuntu-latest + + permissions: + security-events: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + +# If needed severity levels can be controlled here +# severity: warning + - name: Differential ShellCheck + uses: redhat-plumbers-in-action/differential-shellcheck@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} From 850183043437b232935cc998ec7601419f6dce7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B6r=C3=B6k=20Edwin?= Date: Thu, 29 Feb 2024 13:24:49 +0000 Subject: [PATCH 44/99] Update .github/workflows/shellcheck.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pau Ruiz Safont Signed-off-by: Török Edwin --- .github/workflows/shellcheck.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 8725ae60710..c17568d821c 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -1,7 +1,6 @@ name: ShellCheck on: - push: pull_request: concurrency: # On new push, cancel old workflows from the same PR, branch or tag: From faf2c76b3b3f124b0c76c88e0347d3f3059311e5 Mon Sep 17 00:00:00 2001 From: Luca Zhang Date: Wed, 6 Mar 2024 13:34:41 +0800 Subject: [PATCH 45/99] fix: typo in doc `xapi_xenpos.ml` -> `xapi_xenops.ml` Signed-off-by: Luca Zhang --- doc/content/xapi/walkthroughs/migration_overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/content/xapi/walkthroughs/migration_overview.md b/doc/content/xapi/walkthroughs/migration_overview.md index 17c108a2237..7af061574f7 100644 --- a/doc/content/xapi/walkthroughs/migration_overview.md +++ b/doc/content/xapi/walkthroughs/migration_overview.md @@ -63,7 +63,7 @@ flowchart TD like **xapi_vm_migrate.ml**`"] vm_management -- Call --> xapi_xenops["`Transform xenops - see (**xapi_xenpos.ml**)`"] + see (**xapi_xenops.ml**)`"] xapi_xenops <-- Post following IDL model (see xenops_interface.ml) --> msg_switch From 78d387d93f7764f6d17af27ee9b2ae98c5d86db1 Mon Sep 17 00:00:00 2001 From: Vincent Liu Date: Fri, 26 Jan 2024 14:46:46 +0000 Subject: [PATCH 46/99] Update xapi-idl unittest data for cluster interface Signed-off-by: Vincent Liu --- .../lib_test/test_data/cluster/requests/UPDATES.get.request.0 | 1 + .../lib_test/test_data/cluster/requests/create.request.0 | 2 +- .../lib_test/test_data/cluster/requests/create.request.1 | 2 +- .../lib_test/test_data/cluster/requests/create.request.10 | 1 + .../lib_test/test_data/cluster/requests/create.request.11 | 1 + .../lib_test/test_data/cluster/requests/create.request.12 | 1 + .../lib_test/test_data/cluster/requests/create.request.13 | 1 + .../lib_test/test_data/cluster/requests/create.request.14 | 1 + .../lib_test/test_data/cluster/requests/create.request.15 | 1 + .../lib_test/test_data/cluster/requests/create.request.16 | 1 + .../lib_test/test_data/cluster/requests/create.request.17 | 1 + .../lib_test/test_data/cluster/requests/create.request.18 | 1 + .../lib_test/test_data/cluster/requests/create.request.19 | 1 + .../lib_test/test_data/cluster/requests/create.request.2 | 2 +- .../lib_test/test_data/cluster/requests/create.request.20 | 1 + .../lib_test/test_data/cluster/requests/create.request.21 | 1 + .../lib_test/test_data/cluster/requests/create.request.22 | 1 + .../lib_test/test_data/cluster/requests/create.request.23 | 1 + .../lib_test/test_data/cluster/requests/create.request.3 | 2 +- .../lib_test/test_data/cluster/requests/create.request.4 | 2 +- .../lib_test/test_data/cluster/requests/create.request.5 | 2 +- .../lib_test/test_data/cluster/requests/create.request.6 | 2 +- .../lib_test/test_data/cluster/requests/create.request.7 | 2 +- .../lib_test/test_data/cluster/requests/create.request.8 | 1 + .../lib_test/test_data/cluster/requests/create.request.9 | 1 + .../test_data/cluster/requests/declare-changed-addrs.request.0 | 2 +- .../lib_test/test_data/cluster/requests/declare-dead.request.0 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.0 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.1 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.10 | 1 + .../lib_test/test_data/cluster/requests/enable.request.11 | 1 + .../lib_test/test_data/cluster/requests/enable.request.12 | 1 + .../lib_test/test_data/cluster/requests/enable.request.13 | 1 + .../lib_test/test_data/cluster/requests/enable.request.14 | 1 + .../lib_test/test_data/cluster/requests/enable.request.15 | 1 + .../lib_test/test_data/cluster/requests/enable.request.16 | 1 + .../lib_test/test_data/cluster/requests/enable.request.17 | 1 + .../lib_test/test_data/cluster/requests/enable.request.18 | 1 + .../lib_test/test_data/cluster/requests/enable.request.19 | 1 + .../lib_test/test_data/cluster/requests/enable.request.2 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.20 | 1 + .../lib_test/test_data/cluster/requests/enable.request.21 | 1 + .../lib_test/test_data/cluster/requests/enable.request.22 | 1 + .../lib_test/test_data/cluster/requests/enable.request.23 | 1 + .../lib_test/test_data/cluster/requests/enable.request.3 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.4 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.5 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.6 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.7 | 2 +- .../lib_test/test_data/cluster/requests/enable.request.8 | 1 + .../lib_test/test_data/cluster/requests/enable.request.9 | 1 + .../xapi-idl/lib_test/test_data/cluster/requests/join.request.0 | 2 +- .../xapi-idl/lib_test/test_data/cluster/requests/join.request.1 | 2 +- .../lib_test/test_data/cluster/requests/join.request.10 | 1 + .../lib_test/test_data/cluster/requests/join.request.11 | 1 + .../xapi-idl/lib_test/test_data/cluster/requests/join.request.2 | 2 +- .../xapi-idl/lib_test/test_data/cluster/requests/join.request.3 | 2 +- .../xapi-idl/lib_test/test_data/cluster/requests/join.request.4 | 1 + .../xapi-idl/lib_test/test_data/cluster/requests/join.request.5 | 1 + .../xapi-idl/lib_test/test_data/cluster/requests/join.request.6 | 1 + .../xapi-idl/lib_test/test_data/cluster/requests/join.request.7 | 1 + .../xapi-idl/lib_test/test_data/cluster/requests/join.request.8 | 1 + .../xapi-idl/lib_test/test_data/cluster/requests/join.request.9 | 1 + .../lib_test/test_data/cluster/responses/UPDATES.get.response.0 | 1 + .../lib_test/test_data/cluster/responses/UPDATES.get.response.1 | 1 + .../lib_test/test_data/cluster/responses/UPDATES.get.response.2 | 1 + .../lib_test/test_data/cluster/responses/UPDATES.get.response.3 | 1 + 67 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/UPDATES.get.request.0 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.10 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.11 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.12 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.13 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.14 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.15 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.16 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.17 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.18 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.19 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.20 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.21 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.22 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.23 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.8 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.9 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.10 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.11 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.12 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.13 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.14 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.15 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.16 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.17 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.18 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.19 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.20 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.21 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.22 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.23 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.8 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.9 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.10 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.11 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.4 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.5 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.6 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.7 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.8 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.9 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.0 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.1 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.2 create mode 100644 ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.3 diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/UPDATES.get.request.0 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/UPDATES.get.request.0 new file mode 100644 index 00000000000..5847d6e95a4 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/UPDATES.get.request.0 @@ -0,0 +1 @@ +UPDATES.gettimeout0dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.0 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.0 index 29aa7968eab..fb6ed49dd1c 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.0 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.0 @@ -1 +1 @@ -createinit_configlocal_ipIPv4local_ipdbgdbg \ No newline at end of file +createinit_configmemberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.1 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.1 index db07bf45361..38d48b437e8 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.1 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.1 @@ -1 +1 @@ -createinit_configtoken_timeout_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +createinit_configmemberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.10 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.10 new file mode 100644 index 00000000000..ef2070f1fda --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.10 @@ -0,0 +1 @@ +createinit_configtoken_coefficient_ms0token_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.11 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.11 new file mode 100644 index 00000000000..70554d971c1 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.11 @@ -0,0 +1 @@ +createinit_configtoken_coefficient_ms0token_timeout_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.12 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.12 new file mode 100644 index 00000000000..fb935fe1401 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.12 @@ -0,0 +1 @@ +createinit_confignamenamememberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.13 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.13 new file mode 100644 index 00000000000..020b72b36a7 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.13 @@ -0,0 +1 @@ +createinit_confignamenamememberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.14 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.14 new file mode 100644 index 00000000000..2a6d13da8fc --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.14 @@ -0,0 +1 @@ +createinit_confignamenamememberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.15 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.15 new file mode 100644 index 00000000000..10d3bbae90d --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.15 @@ -0,0 +1 @@ +createinit_confignamenametoken_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.16 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.16 new file mode 100644 index 00000000000..adb7d4d94b4 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.16 @@ -0,0 +1 @@ +createinit_confignamenametoken_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.17 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.17 new file mode 100644 index 00000000000..26300fb43ce --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.17 @@ -0,0 +1 @@ +createinit_confignamenametoken_timeout_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.18 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.18 new file mode 100644 index 00000000000..10306296dec --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.18 @@ -0,0 +1 @@ +createinit_confignamenametoken_coefficient_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.19 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.19 new file mode 100644 index 00000000000..08d3a076bc4 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.19 @@ -0,0 +1 @@ +createinit_confignamenametoken_coefficient_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.2 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.2 index e20a5cc9569..76c5e8f95ee 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.2 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.2 @@ -1 +1 @@ -createinit_configtoken_coefficient_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +createinit_configmemberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.20 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.20 new file mode 100644 index 00000000000..a616b4e4ab9 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.20 @@ -0,0 +1 @@ +createinit_confignamenametoken_coefficient_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.21 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.21 new file mode 100644 index 00000000000..243a88d1e12 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.21 @@ -0,0 +1 @@ +createinit_confignamenametoken_coefficient_ms0token_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.22 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.22 new file mode 100644 index 00000000000..181f86e37a1 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.22 @@ -0,0 +1 @@ +createinit_confignamenametoken_coefficient_ms0token_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.23 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.23 new file mode 100644 index 00000000000..a448fe4fe8f --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.23 @@ -0,0 +1 @@ +createinit_confignamenametoken_coefficient_ms0token_timeout_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.3 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.3 index 53686d5db8f..1f31e33f87d 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.3 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.3 @@ -1 +1 @@ -createinit_configtoken_coefficient_ms0token_timeout_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +createinit_configtoken_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.4 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.4 index 9ed87b078a9..c11fcb39de3 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.4 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.4 @@ -1 +1 @@ -createinit_confignamenamelocal_ipIPv4local_ipdbgdbg \ No newline at end of file +createinit_configtoken_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.5 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.5 index 019b690cdae..0748d980d28 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.5 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.5 @@ -1 +1 @@ -createinit_confignamenametoken_timeout_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +createinit_configtoken_timeout_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.6 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.6 index c433c588770..04683e3c7bb 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.6 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.6 @@ -1 +1 @@ -createinit_confignamenametoken_coefficient_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +createinit_configtoken_coefficient_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.7 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.7 index 5f1267237f7..aba3cb6edf2 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.7 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.7 @@ -1 +1 @@ -createinit_confignamenametoken_coefficient_ms0token_timeout_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +createinit_configtoken_coefficient_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.8 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.8 new file mode 100644 index 00000000000..8f855486b75 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.8 @@ -0,0 +1 @@ +createinit_configtoken_coefficient_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.9 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.9 new file mode 100644 index 00000000000..da42ebdac52 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/create.request.9 @@ -0,0 +1 @@ +createinit_configtoken_coefficient_ms0token_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/declare-changed-addrs.request.0 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/declare-changed-addrs.request.0 index 8b0e17d0d25..8186be62618 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/declare-changed-addrs.request.0 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/declare-changed-addrs.request.0 @@ -1 +1 @@ -declare-changed-addrschanged_membersIPv4changed_membersdbgdbg \ No newline at end of file +declare-changed-addrschanged_membersIPv4changed_membersExtendedhostnamehostnamehostuuidhostuuidip::1Extendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/declare-dead.request.0 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/declare-dead.request.0 index 6287e3c2837..088815b5b74 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/declare-dead.request.0 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/declare-dead.request.0 @@ -1 +1 @@ -declare-deaddead_membersIPv4dead_membersdbgdbg \ No newline at end of file +declare-deaddead_membersIPv4dead_membersExtendedhostnamehostnamehostuuidhostuuidip::1Extendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.0 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.0 index b8f993c2834..11f2c6e5533 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.0 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.0 @@ -1 +1 @@ -enableinit_configlocal_ipIPv4local_ipdbgdbg \ No newline at end of file +enableinit_configmemberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.1 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.1 index 3ba0183c9f5..bb7c41e1d0d 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.1 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.1 @@ -1 +1 @@ -enableinit_configtoken_timeout_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +enableinit_configmemberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.10 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.10 new file mode 100644 index 00000000000..e09a7148b9d --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.10 @@ -0,0 +1 @@ +enableinit_configtoken_coefficient_ms0token_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.11 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.11 new file mode 100644 index 00000000000..49b8fb930d6 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.11 @@ -0,0 +1 @@ +enableinit_configtoken_coefficient_ms0token_timeout_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.12 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.12 new file mode 100644 index 00000000000..abc5b8867f9 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.12 @@ -0,0 +1 @@ +enableinit_confignamenamememberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.13 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.13 new file mode 100644 index 00000000000..5eda169e2e2 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.13 @@ -0,0 +1 @@ +enableinit_confignamenamememberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.14 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.14 new file mode 100644 index 00000000000..6fbfd4a9455 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.14 @@ -0,0 +1 @@ +enableinit_confignamenamememberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.15 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.15 new file mode 100644 index 00000000000..32de015ec27 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.15 @@ -0,0 +1 @@ +enableinit_confignamenametoken_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.16 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.16 new file mode 100644 index 00000000000..d7d7806d9b1 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.16 @@ -0,0 +1 @@ +enableinit_confignamenametoken_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.17 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.17 new file mode 100644 index 00000000000..f9a6817d674 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.17 @@ -0,0 +1 @@ +enableinit_confignamenametoken_timeout_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.18 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.18 new file mode 100644 index 00000000000..89e72a50788 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.18 @@ -0,0 +1 @@ +enableinit_confignamenametoken_coefficient_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.19 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.19 new file mode 100644 index 00000000000..dbdb1142cca --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.19 @@ -0,0 +1 @@ +enableinit_confignamenametoken_coefficient_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.2 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.2 index 51f4ecec4ab..321c330979b 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.2 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.2 @@ -1 +1 @@ -enableinit_configtoken_coefficient_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +enableinit_configmemberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.20 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.20 new file mode 100644 index 00000000000..a31cfebbdc5 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.20 @@ -0,0 +1 @@ +enableinit_confignamenametoken_coefficient_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.21 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.21 new file mode 100644 index 00000000000..a20ccd296de --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.21 @@ -0,0 +1 @@ +enableinit_confignamenametoken_coefficient_ms0token_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.22 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.22 new file mode 100644 index 00000000000..8ad7120e72e --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.22 @@ -0,0 +1 @@ +enableinit_confignamenametoken_coefficient_ms0token_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.23 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.23 new file mode 100644 index 00000000000..8d7e1a3556c --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.23 @@ -0,0 +1 @@ +enableinit_confignamenametoken_coefficient_ms0token_timeout_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.3 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.3 index bf87a95cd29..56863f1cf9c 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.3 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.3 @@ -1 +1 @@ -enableinit_configtoken_coefficient_ms0token_timeout_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +enableinit_configtoken_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.4 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.4 index a671202d8d5..32fee9d1a54 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.4 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.4 @@ -1 +1 @@ -enableinit_confignamenamelocal_ipIPv4local_ipdbgdbg \ No newline at end of file +enableinit_configtoken_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.5 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.5 index da60afd5455..02cf626a9af 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.5 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.5 @@ -1 +1 @@ -enableinit_confignamenametoken_timeout_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +enableinit_configtoken_timeout_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.6 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.6 index 9d1a2a1d55b..ff5e7ffb9c7 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.6 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.6 @@ -1 +1 @@ -enableinit_confignamenametoken_coefficient_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +enableinit_configtoken_coefficient_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.7 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.7 index a9a1f87824c..ee5567269fc 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.7 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.7 @@ -1 +1 @@ -enableinit_confignamenametoken_coefficient_ms0token_timeout_ms0local_ipIPv4local_ipdbgdbg \ No newline at end of file +enableinit_configtoken_coefficient_ms0memberExtendedhostnamehostnamehostuuidhostuuidip::1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.8 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.8 new file mode 100644 index 00000000000..d0ad3232b39 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.8 @@ -0,0 +1 @@ +enableinit_configtoken_coefficient_ms0memberIPv4memberdbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.9 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.9 new file mode 100644 index 00000000000..24748b08e27 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/enable.request.9 @@ -0,0 +1 @@ +enableinit_configtoken_coefficient_ms0token_timeout_ms0memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1dbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.0 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.0 index a512fb470ec..a64d926749d 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.0 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.0 @@ -1 +1 @@ -joinexisting_membersIPv4existing_memberstls_configcncnserver_pem_pathserver_pem_pathnew_memberIPv4new_membertokentokendbgdbg \ No newline at end of file +joinexisting_membersIPv4existing_membersExtendedhostnamehostnamehostuuidhostuuidip::1Extendedhostnamehostnamehostuuidhostuuidip127.0.0.1tls_configcncnserver_pem_pathserver_pem_pathnew_memberIPv4new_membertokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.1 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.1 index ff7959f5f7f..922726c04e5 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.1 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.1 @@ -1 +1 @@ -joinexisting_membersIPv4existing_memberstls_configtrusted_bundle_pathtrusted_bundle_pathcncnserver_pem_pathserver_pem_pathnew_memberIPv4new_membertokentokendbgdbg \ No newline at end of file +joinexisting_membersIPv4existing_membersExtendedhostnamehostnamehostuuidhostuuidip::1Extendedhostnamehostnamehostuuidhostuuidip127.0.0.1tls_configcncnserver_pem_pathserver_pem_pathnew_memberExtendedhostnamehostnamehostuuidhostuuidip::1tokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.10 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.10 new file mode 100644 index 00000000000..0f7caf358b2 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.10 @@ -0,0 +1 @@ +joinexisting_memberstls_configtrusted_bundle_pathtrusted_bundle_pathcncnserver_pem_pathserver_pem_pathnew_memberExtendedhostnamehostnamehostuuidhostuuidip::1tokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.11 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.11 new file mode 100644 index 00000000000..af324d536e1 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.11 @@ -0,0 +1 @@ +joinexisting_memberstls_configtrusted_bundle_pathtrusted_bundle_pathcncnserver_pem_pathserver_pem_pathnew_memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1tokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.2 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.2 index e56c0fcea49..c792f81d190 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.2 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.2 @@ -1 +1 @@ -joinexisting_memberstls_configcncnserver_pem_pathserver_pem_pathnew_memberIPv4new_membertokentokendbgdbg \ No newline at end of file +joinexisting_membersIPv4existing_membersExtendedhostnamehostnamehostuuidhostuuidip::1Extendedhostnamehostnamehostuuidhostuuidip127.0.0.1tls_configcncnserver_pem_pathserver_pem_pathnew_memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1tokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.3 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.3 index 060a7f50800..2d00fe1665d 100644 --- a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.3 +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.3 @@ -1 +1 @@ -joinexisting_memberstls_configtrusted_bundle_pathtrusted_bundle_pathcncnserver_pem_pathserver_pem_pathnew_memberIPv4new_membertokentokendbgdbg \ No newline at end of file +joinexisting_membersIPv4existing_membersExtendedhostnamehostnamehostuuidhostuuidip::1Extendedhostnamehostnamehostuuidhostuuidip127.0.0.1tls_configtrusted_bundle_pathtrusted_bundle_pathcncnserver_pem_pathserver_pem_pathnew_memberIPv4new_membertokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.4 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.4 new file mode 100644 index 00000000000..fa37ecbff41 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.4 @@ -0,0 +1 @@ +joinexisting_membersIPv4existing_membersExtendedhostnamehostnamehostuuidhostuuidip::1Extendedhostnamehostnamehostuuidhostuuidip127.0.0.1tls_configtrusted_bundle_pathtrusted_bundle_pathcncnserver_pem_pathserver_pem_pathnew_memberExtendedhostnamehostnamehostuuidhostuuidip::1tokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.5 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.5 new file mode 100644 index 00000000000..f76d9c9d55c --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.5 @@ -0,0 +1 @@ +joinexisting_membersIPv4existing_membersExtendedhostnamehostnamehostuuidhostuuidip::1Extendedhostnamehostnamehostuuidhostuuidip127.0.0.1tls_configtrusted_bundle_pathtrusted_bundle_pathcncnserver_pem_pathserver_pem_pathnew_memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1tokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.6 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.6 new file mode 100644 index 00000000000..e56c0fcea49 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.6 @@ -0,0 +1 @@ +joinexisting_memberstls_configcncnserver_pem_pathserver_pem_pathnew_memberIPv4new_membertokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.7 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.7 new file mode 100644 index 00000000000..0b111904a4a --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.7 @@ -0,0 +1 @@ +joinexisting_memberstls_configcncnserver_pem_pathserver_pem_pathnew_memberExtendedhostnamehostnamehostuuidhostuuidip::1tokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.8 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.8 new file mode 100644 index 00000000000..975efbf4f0d --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.8 @@ -0,0 +1 @@ +joinexisting_memberstls_configcncnserver_pem_pathserver_pem_pathnew_memberExtendedhostnamehostnamehostuuidhostuuidip127.0.0.1tokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.9 b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.9 new file mode 100644 index 00000000000..060a7f50800 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/requests/join.request.9 @@ -0,0 +1 @@ +joinexisting_memberstls_configtrusted_bundle_pathtrusted_bundle_pathcncnserver_pem_pathserver_pem_pathnew_memberIPv4new_membertokentokendbgdbg \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.0 b/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.0 new file mode 100644 index 00000000000..b685d66de05 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.0 @@ -0,0 +1 @@ +StatusSuccessValueupdates \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.1 b/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.1 new file mode 100644 index 00000000000..86fa7244e6f --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.1 @@ -0,0 +1 @@ +StatusSuccessValue \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.2 b/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.2 new file mode 100644 index 00000000000..53dd412b045 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.2 @@ -0,0 +1 @@ +StatusFailureErrorDescriptionInternalErrorerror \ No newline at end of file diff --git a/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.3 b/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.3 new file mode 100644 index 00000000000..e8c5d8d5181 --- /dev/null +++ b/ocaml/xapi-idl/lib_test/test_data/cluster/responses/UPDATES.get.response.3 @@ -0,0 +1 @@ +StatusFailureErrorDescriptionUnix_errorerror \ No newline at end of file From 30373433e04f4388bc9c00b06bc911f1c0a33e2e Mon Sep 17 00:00:00 2001 From: Vincent Liu Date: Thu, 18 Jan 2024 17:20:39 +0000 Subject: [PATCH 47/99] CP-45496: Xapi writes host name/uuid to corosync.conf Add support for clusterd to write host name and uuid to corosync.conf on each host. This is done by augmenting the address type with uuid info rather than adding a new parameter to the cluster api calls which can cause compatibility issues during pool upgrade. Signed-off-by: Vincent Liu --- ocaml/xapi-idl/cluster/cluster_interface.ml | 44 ++++++++++++++++++--- ocaml/xapi-idl/cluster/dune | 1 + ocaml/xapi-idl/cluster/ipaddr_rpc_type.ml | 25 ++++++++++++ ocaml/xapi-idl/cluster/ipaddr_rpc_type.mli | 5 +++ ocaml/xapi/xapi_cluster.ml | 17 +++++++- ocaml/xapi/xapi_cluster_host.ml | 43 +++++++++++++++----- ocaml/xapi/xapi_clustering.ml | 12 +++--- 7 files changed, 126 insertions(+), 21 deletions(-) create mode 100644 ocaml/xapi-idl/cluster/ipaddr_rpc_type.ml create mode 100644 ocaml/xapi-idl/cluster/ipaddr_rpc_type.mli diff --git a/ocaml/xapi-idl/cluster/cluster_interface.ml b/ocaml/xapi-idl/cluster/cluster_interface.ml index b457ebbf195..effda821999 100644 --- a/ocaml/xapi-idl/cluster/cluster_interface.ml +++ b/ocaml/xapi-idl/cluster/cluster_interface.ml @@ -2,6 +2,7 @@ open Rpc open Idl +open Ipaddr_rpc_type let service_name = "cluster" @@ -15,12 +16,45 @@ type debug_info = string [@@deriving rpcty] (** Name of the cluster *) type cluster_name = string [@@deriving rpcty] -(** An IPv4 address (a.b.c.d) *) -type address = IPv4 of string [@@deriving rpcty] +type ip = IPv4 of string | IPv6 of string [@@deriving rpcty] -let printaddr () = function IPv4 s -> Printf.sprintf "IPv4(%s)" s +let string_of_ip = Ipaddr.to_string -let str_of_address address = match address with IPv4 a -> a +(** this address includes the hostname and hostuuid along with an ip address + this is done to maintain backwards compatability, and should be combined with + the other variant in the future. + *) +type extended_addr = {ip: Ipaddr.t; hostuuid: string; hostname: string} +[@@deriving rpcty] + +type address = IPv4 of string | Extended of extended_addr [@@deriving rpcty] + +let ipstr_of_address = function + | IPv4 a -> + a + | Extended {ip; _} -> + string_of_ip ip + +let ipaddr_of_address = function + | IPv4 _ as ip -> + ip + | Extended {ip; _} -> + (* FIXME: introduce IPv6 variant when IPv6 support is added *) + IPv4 (string_of_ip ip) + +(** The complete address potentially including uuid and hostname *) +let fullstr_of_address = function + | IPv4 a -> + a + | Extended {ip; hostuuid; hostname} -> + Printf.sprintf "(%s, %s, %s)" (string_of_ip ip) hostuuid hostname + +let printaddr () = function + | IPv4 s -> + Printf.sprintf "IPv4(%s)" s + | Extended {ip; hostuuid; hostname} -> + Printf.sprintf "IP(%s) hostuuid (%s) hostname(%s)" (string_of_ip ip) + hostuuid hostname type addresslist = address list [@@deriving rpcty] @@ -40,7 +74,7 @@ type all_members = node list [@@deriving rpcty] (** This type contains all of the information required to initialise the cluster. All optional params will have the recommended defaults if None. *) type init_config = { - local_ip: address + member: address ; token_timeout_ms: int64 option ; token_coefficient_ms: int64 option ; name: string option diff --git a/ocaml/xapi-idl/cluster/dune b/ocaml/xapi-idl/cluster/dune index 5216ef3fca8..50777aeb2b3 100644 --- a/ocaml/xapi-idl/cluster/dune +++ b/ocaml/xapi-idl/cluster/dune @@ -10,6 +10,7 @@ rresult xapi-idl threads + ipaddr ) (wrapped false) (preprocess (pps ppx_deriving_rpc))) diff --git a/ocaml/xapi-idl/cluster/ipaddr_rpc_type.ml b/ocaml/xapi-idl/cluster/ipaddr_rpc_type.ml new file mode 100644 index 00000000000..6d8c1823896 --- /dev/null +++ b/ocaml/xapi-idl/cluster/ipaddr_rpc_type.ml @@ -0,0 +1,25 @@ +module Ipaddr = struct + include Ipaddr + + let typ_of = + Rpc.Types.Abstract + { + aname= "ipaddr" + ; test_data= + [Ipaddr.V4 Ipaddr.V4.localhost; Ipaddr.V6 Ipaddr.V6.localhost] + ; rpc_of= (fun t -> Rpc.String (Ipaddr.to_string t)) + ; of_rpc= + (function + | Rpc.String s -> + Ipaddr.of_string s + | r -> + Error + (`Msg + (Printf.sprintf + "typ_of_vm_uuid: expected rpc string but got %s" + (Rpc.to_string r) + ) + ) + ) + } +end diff --git a/ocaml/xapi-idl/cluster/ipaddr_rpc_type.mli b/ocaml/xapi-idl/cluster/ipaddr_rpc_type.mli new file mode 100644 index 00000000000..d4ebb00c2f7 --- /dev/null +++ b/ocaml/xapi-idl/cluster/ipaddr_rpc_type.mli @@ -0,0 +1,5 @@ +module Ipaddr : sig + include module type of Ipaddr + + val typ_of : t Rpc.Types.typ +end diff --git a/ocaml/xapi/xapi_cluster.ml b/ocaml/xapi/xapi_cluster.ml index c16a1b4410a..0ec8c1d3a8e 100644 --- a/ocaml/xapi/xapi_cluster.ml +++ b/ocaml/xapi/xapi_cluster.ml @@ -13,6 +13,7 @@ *) open Xapi_clustering +open Ipaddr_rpc_type module D = Debug.Make (struct let name = "xapi_cluster" end) @@ -49,14 +50,26 @@ let create ~__context ~pIF ~cluster_stack ~pool_auto_join ~token_timeout let host = Helpers.get_master ~__context in let pifrec = Db.PIF.get_record ~__context ~self:pIF in assert_pif_prerequisites (pIF, pifrec) ; - let ip = ip_of_pif (pIF, pifrec) in + let ip_addr = ip_of_pif (pIF, pifrec) in + let hostuuid = Inventory.lookup Inventory._installation_uuid in + let hostname = Db.Host.get_hostname ~__context ~self:host in + let member = + Cluster_interface.( + Extended + { + ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) + ; hostuuid + ; hostname + } + ) + in let token_timeout_ms = Int64.of_float (token_timeout *. 1000.0) in let token_timeout_coefficient_ms = Int64.of_float (token_timeout_coefficient *. 1000.0) in let init_config = { - Cluster_interface.local_ip= ip + Cluster_interface.member ; token_timeout_ms= Some token_timeout_ms ; token_coefficient_ms= Some token_timeout_coefficient_ms ; name= None diff --git a/ocaml/xapi/xapi_cluster_host.ml b/ocaml/xapi/xapi_cluster_host.ml index 05cc439619f..38b33677c4a 100644 --- a/ocaml/xapi/xapi_cluster_host.ml +++ b/ocaml/xapi/xapi_cluster_host.ml @@ -14,6 +14,7 @@ open Xapi_clustering open Xapi_cluster_helpers +open Ipaddr_rpc_type module D = Debug.Make (struct let name = "xapi_cluster_host" end) @@ -129,7 +130,20 @@ let join_internal ~__context ~self = let cluster_token = Db.Cluster.get_cluster_token ~__context ~self:cluster in - let ip = ip_of_pif (pIF, Db.PIF.get_record ~__context ~self:pIF) in + let ip_addr = ip_of_pif (pIF, Db.PIF.get_record ~__context ~self:pIF) in + let hostuuid = Inventory.lookup Inventory._installation_uuid in + let host = Db.Cluster_host.get_host ~__context ~self in + let hostname = Db.Host.get_hostname ~__context ~self:host in + let member = + Cluster_interface.( + Extended + { + ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) + ; hostuuid + ; hostname + } + ) + in let ip_list = List.filter_map (fun self -> @@ -137,9 +151,9 @@ let join_internal ~__context ~self = let p_rec = Db.PIF.get_record ~__context ~self:p_ref in (* parallel join: some hosts may not have an IP yet *) try - let other_ip = ip_of_pif (p_ref, p_rec) in - if other_ip <> ip then - Some other_ip + let other_ip_addr = ip_of_pif (p_ref, p_rec) in + if other_ip_addr <> ip_addr then + Some other_ip_addr else None with _ -> None @@ -156,8 +170,8 @@ let join_internal ~__context ~self = let verify = Stunnel_client.get_verify_by_default () in let tls_config = build_tls_config ~__context ~verify in let result = - Cluster_client.LocalClient.join (rpc ~__context) dbg cluster_token ip - tls_config ip_list + Cluster_client.LocalClient.join (rpc ~__context) dbg cluster_token + member tls_config ip_list in match Idl.IdM.run @@ Cluster_client.IDL.T.get result with | Ok () -> @@ -316,8 +330,19 @@ let enable ~__context ~self = let pifref = Db.Cluster_host.get_PIF ~__context ~self in let pifrec = Db.PIF.get_record ~__context ~self:pifref in assert_pif_prerequisites (pifref, pifrec) ; - let ip = ip_of_pif (pifref, pifrec) in - + let ip_addr = ip_of_pif (pifref, pifrec) in + let hostuuid = Inventory.lookup Inventory._installation_uuid in + let hostname = Db.Host.get_hostname ~__context ~self:host in + let member = + Cluster_interface.( + Extended + { + ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) + ; hostuuid + ; hostname + } + ) + in (* TODO: Pass these through from CLI *) if not !Xapi_clustering.Daemon.enabled then ( D.debug @@ -331,7 +356,7 @@ let enable ~__context ~self = set_tls_config ~__context ~self ~verify ; let init_config = { - Cluster_interface.local_ip= ip + Cluster_interface.member ; token_timeout_ms= None ; token_coefficient_ms= None ; name= None diff --git a/ocaml/xapi/xapi_clustering.ml b/ocaml/xapi/xapi_clustering.ml index 699aa93420a..f9a78fef05f 100644 --- a/ocaml/xapi/xapi_clustering.ml +++ b/ocaml/xapi/xapi_clustering.ml @@ -420,10 +420,11 @@ let on_corosync_update ~__context ~cluster updates = List.map (fun ch -> let pIF = Db.Cluster_host.get_PIF ~__context ~self:ch in - let (Cluster_interface.IPv4 ip) = + let ipstr = ip_of_pif (pIF, Db.PIF.get_record ~__context ~self:pIF) + |> ipstr_of_address in - (ip, ch) + (ipstr, ch) ) all_cluster_hosts in @@ -440,13 +441,14 @@ let on_corosync_update ~__context ~cluster updates = | Some nodel -> let quorum_hosts = List.filter_map - (fun {addr= IPv4 addr; _} -> - match List.assoc_opt addr ip_ch with + (fun {addr; _} -> + let ipstr = ipstr_of_address addr in + match List.assoc_opt ipstr ip_ch with | None -> error "%s: cannot find cluster host with network address %s, \ ignoring this host" - __FUNCTION__ addr ; + __FUNCTION__ ipstr ; None | Some ch -> Some ch From a5b544e1bc2f2aeaec208ca0d397cc74ce9696fc Mon Sep 17 00:00:00 2001 From: Vincent Liu Date: Wed, 6 Mar 2024 10:52:54 +0000 Subject: [PATCH 48/99] Add feature flag Signed-off-by: Vincent Liu --- ocaml/xapi/xapi_cluster.ml | 19 ++++++++++------- ocaml/xapi/xapi_cluster_host.ml | 38 +++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/ocaml/xapi/xapi_cluster.ml b/ocaml/xapi/xapi_cluster.ml index 0ec8c1d3a8e..6fd6f66ea99 100644 --- a/ocaml/xapi/xapi_cluster.ml +++ b/ocaml/xapi/xapi_cluster.ml @@ -54,14 +54,17 @@ let create ~__context ~pIF ~cluster_stack ~pool_auto_join ~token_timeout let hostuuid = Inventory.lookup Inventory._installation_uuid in let hostname = Db.Host.get_hostname ~__context ~self:host in let member = - Cluster_interface.( - Extended - { - ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) - ; hostuuid - ; hostname - } - ) + if Xapi_cluster_helpers.cluster_health_enabled ~__context then + Cluster_interface.( + Extended + { + ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) + ; hostuuid + ; hostname + } + ) + else + Cluster_interface.(IPv4 (ipstr_of_address ip_addr)) in let token_timeout_ms = Int64.of_float (token_timeout *. 1000.0) in let token_timeout_coefficient_ms = diff --git a/ocaml/xapi/xapi_cluster_host.ml b/ocaml/xapi/xapi_cluster_host.ml index 38b33677c4a..f026e782f2e 100644 --- a/ocaml/xapi/xapi_cluster_host.ml +++ b/ocaml/xapi/xapi_cluster_host.ml @@ -135,14 +135,17 @@ let join_internal ~__context ~self = let host = Db.Cluster_host.get_host ~__context ~self in let hostname = Db.Host.get_hostname ~__context ~self:host in let member = - Cluster_interface.( - Extended - { - ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) - ; hostuuid - ; hostname - } - ) + if Xapi_cluster_helpers.cluster_health_enabled ~__context then + Cluster_interface.( + Extended + { + ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) + ; hostuuid + ; hostname + } + ) + else + Cluster_interface.(IPv4 (ipstr_of_address ip_addr)) in let ip_list = List.filter_map @@ -334,14 +337,17 @@ let enable ~__context ~self = let hostuuid = Inventory.lookup Inventory._installation_uuid in let hostname = Db.Host.get_hostname ~__context ~self:host in let member = - Cluster_interface.( - Extended - { - ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) - ; hostuuid - ; hostname - } - ) + if Xapi_cluster_helpers.cluster_health_enabled ~__context then + Cluster_interface.( + Extended + { + ip= Ipaddr.of_string_exn (ipstr_of_address ip_addr) + ; hostuuid + ; hostname + } + ) + else + Cluster_interface.(IPv4 (ipstr_of_address ip_addr)) in (* TODO: Pass these through from CLI *) if not !Xapi_clustering.Daemon.enabled then ( From 72e96146d6ca6c3ed8f3016f5ed6570d03cf2021 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 7 Feb 2024 15:02:36 +0000 Subject: [PATCH 49/99] Replace use of `sdksanity` with reusable workflow for testing SDKs Signed-off-by: Danilo Del Busso --- .github/workflows/main.yml | 74 ++++++++++---------------------------- Makefile | 5 --- 2 files changed, 18 insertions(+), 61 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cadf84c35c4..9f1ee1d29f8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,9 +5,9 @@ on: pull_request: schedule: # run daily, this refreshes the cache - - cron: '13 2 * * *' + - cron: "13 2 * * *" -concurrency: # On new push, cancel old workflows from the same PR, branch or tag: +concurrency: # On new push, cancel old workflows from the same PR, branch or tag: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -18,12 +18,12 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['2.7', '3.11'] + python-version: ["2.7", "3.11"] steps: - name: Checkout code uses: actions/checkout@v3 with: - fetch-depth: 0 # To check which files changed: origin/master..HEAD + fetch-depth: 0 # To check which files changed: origin/master..HEAD - uses: LizardByte/setup-python-action@master with: python-version: ${{matrix.python-version}} @@ -31,8 +31,8 @@ jobs: - uses: actions/cache@v3 name: Setup cache for running pre-commit fast with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" @@ -103,55 +103,21 @@ jobs: # Fails for user workflows without permissions(fork-based pull requests): continue-on-error: true with: - message-path: .git/pytype-summary.md # Add the content of it as comment + message-path: .git/pytype-summary.md # Add the content of it as comment - ocaml-test: - name: Ocaml tests + ocaml-tests: + name: Run OCaml tests runs-on: ubuntu-20.04 env: XAPI_VERSION: "v0.0.0" - steps: - name: Checkout code uses: actions/checkout@v3 - - name: Free space - run: sudo rm -rf /usr/local/lib/android - - - name: Pull configuration from xs-opam - run: | - curl --fail --silent https://raw.githubusercontent.com/xapi-project/xs-opam/master/tools/xs-opam-ci.env | cut -f2 -d " " > .env - - - name: Load environment file - id: dotenv - uses: falti/dotenv-action@v1.0.4 - - - name: Update Ubuntu repositories - run: sudo apt-get update - - - name: Use disk with more space for TMPDIR and XDG_CACHE_HOME - run: | - df -h || true - export TMPDIR="/mnt/build/tmp" - export XDG_CACHE_HOME="/mnt/build/cache" - sudo mkdir -p "${TMPDIR}" "${XDG_CACHE_HOME}" - sudo chown "$(id -u):$(id -g)" "${TMPDIR}" "${XDG_CACHE_HOME}" - echo "TMPDIR=${TMPDIR}" >>"$GITHUB_ENV" - echo "XDG_CACHE_HOME=${XDG_CACHE_HOME}" >>"$GITHUB_ENV" - - - name: Use ocaml - uses: ocaml/setup-ocaml@v2 + - name: Setup XenAPI environment + uses: ./.github/workflows/setup-xapi-environment with: - ocaml-compiler: ${{ steps.dotenv.outputs.ocaml_version_full }} - opam-repositories: | - xs-opam: ${{ steps.dotenv.outputs.repository }} - dune-cache: true - - - name: Install dependencies - run: opam install . --deps-only --with-test -v - - - name: Configure - run: opam exec -- ./configure --xapi_version="$XAPI_VERSION" + xapi_version: ${{ env.XAPI_VERSION }} - name: Build run: opam exec -- make @@ -166,21 +132,11 @@ jobs: run: opam exec -- make stresstest if: ${{ github.event_name == 'schedule' }} - - name: Build SDK - run: | - mkdir -p /opt/xensource/sm - wget -O /opt/xensource/sm/XE_SR_ERRORCODES.xml https://raw.githubusercontent.com/xapi-project/sm/master/drivers/XE_SR_ERRORCODES.xml - opam exec -- make sdk - - name: Make install smoketest run: | opam exec -- make install DESTDIR=$(mktemp -d) opam exec -- make install DESTDIR=$(mktemp -d) BUILD_PY2=NO - - name: Sanity test SDK - run: | - opam exec -- make sdksanity - - name: Check disk space run: df -h || true @@ -211,3 +167,9 @@ jobs: github_token: ${{ secrets.github_token }} reporter: github-pr-review level: error + + test-sdk-builds: + name: Test SDK builds + uses: ./.github/workflows/generate-and-build-sdks.yml + with: + xapi_version: ${{ env.XAPI_VERSION }} diff --git a/Makefile b/Makefile index bcfc5b9eb78..7b5d6cb8703 100644 --- a/Makefile +++ b/Makefile @@ -112,11 +112,6 @@ sdk: sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/csharp sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/powershell -# workaround for no .resx generation, just for compilation testing -sdksanity: sdk - sed -i 's/FriendlyErrorNames.ResourceManager/null/g' ./_build/install/default/xapi/sdk/csharp/src/Failure.cs - cd _build/install/default/xapi/sdk/csharp/src && dotnet add package Newtonsoft.Json && dotnet build -f netstandard2.0 - .PHONY: sdk-build-java sdk-build-java: sdk From 89e4f79dc43c665380a7693bdad6f03af526431c Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 7 Feb 2024 15:02:36 +0000 Subject: [PATCH 50/99] Build and package C# and PowerShell SDKs when creating a release Signed-off-by: Danilo Del Busso damnit --- .github/workflows/main.yml | 8 +++++- .github/workflows/release.yml | 50 ++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9f1ee1d29f8..6400d6cc97d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,6 +109,9 @@ jobs: name: Run OCaml tests runs-on: ubuntu-20.04 env: + # Ensure you also update test-sdk-builds + # when changing this value, to keep builds + # consistent XAPI_VERSION: "v0.0.0" steps: - name: Checkout code @@ -172,4 +175,7 @@ jobs: name: Test SDK builds uses: ./.github/workflows/generate-and-build-sdks.yml with: - xapi_version: ${{ env.XAPI_VERSION }} + # Ensure you also update ocaml-tests + # when changing this value, to keep builds + # consistent + xapi_version: "v0.0.0" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f55374239b..48e7f81a839 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,10 +3,11 @@ name: Create release from tag on: push: tags: - - 'v*' + - "v*" jobs: build-python: + name: Build and upload Python artifacts runs-on: ubuntu-latest steps: @@ -16,7 +17,7 @@ jobs: - name: Use python uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: "3.x" - name: Install build dependencies run: | @@ -34,23 +35,60 @@ jobs: name: XenAPI path: scripts/examples/python/dist/ + build-sdks: + name: Build and upload SDK artifacts + uses: ./.github/workflows/generate-and-build-sdks.yml + with: + xapi_version: ${{ github.ref_name }} release: + name: "Create and package release" runs-on: ubuntu-latest - needs: build-python + needs: [build-python, build-sdks] steps: - - name: Retrieve python distribution artifacts + - name: Retrieve Python SDK distribution artifacts uses: actions/download-artifact@v3 with: name: XenAPI path: dist/ + - name: Retrieve C# SDK distribution artifacts + uses: actions/download-artifact@v3 + with: + name: SDK_Binaries_CSharp + path: dist/ + + - name: Retrieve PowerShell 5.x SDK distribution artifacts + uses: actions/download-artifact@v3 + with: + name: XenServerPowerShell_NET45 + path: sdk_powershell_5x/ + + - name: Retrieve PowerShell 7.x SDK distribution artifacts + uses: actions/download-artifact@v3 + with: + name: XenServerPowerShell_NET8 + path: sdk_powershell_7x/ + + - name: Zip PowerShell 5.x SDK artifacts for deployment + shell: bash + run: zip PowerShell-SDK-5.x-prerelease-unsigned.zip ./sdk_powershell_5x -r + + - name: Zip PowerShell 7.x SDK artifacts for deployment + shell: bash + run: zip PowerShell-SDK-7.x-prerelease-unsigned.zip ./sdk_powershell_7x -r + - name: Create release ${{ github.ref_name }} - run: gh release create ${{ github.ref_name }} --repo ${{ github.repository }} --generate-notes dist/* + shell: bash + run: | + gh release create ${{ github.ref_name }} --repo ${{ github.repository }} --generate-notes dist/* + gh release upload ${{ github.ref_name }} --repo ${{ github.repository }} PowerShell-SDK-5.x-prerelease-unsigned.zip + gh release upload ${{ github.ref_name }} --repo ${{ github.repository }} PowerShell-SDK-7.x-prerelease-unsigned.zip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} publish-pypi: + name: Publish Python release to PyPI runs-on: ubuntu-latest needs: release environment: pypi @@ -63,5 +101,5 @@ jobs: name: XenAPI path: dist/ - - name: Publish the Python release to PyPI + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 From 3d2738e3295c2a915764351700db6ac3cb3e43f3 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 14 Feb 2024 08:39:34 -0500 Subject: [PATCH 51/99] Add and use `cleanup-xapi-environment` composite action Signed-off-by: Danilo Del Busso --- .../workflows/cleanup-xapi-environment/action.yml | 13 +++++++++++++ .github/workflows/generate-and-build-sdks.yml | 11 +++++++---- .github/workflows/main.yml | 10 +++------- .github/workflows/setup-xapi-environment/action.yml | 2 +- 4 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/cleanup-xapi-environment/action.yml diff --git a/.github/workflows/cleanup-xapi-environment/action.yml b/.github/workflows/cleanup-xapi-environment/action.yml new file mode 100644 index 00000000000..96323007e4e --- /dev/null +++ b/.github/workflows/cleanup-xapi-environment/action.yml @@ -0,0 +1,13 @@ +name: Cleanup XenAPI environment +description: Cleanup XenAPI environment created using the setup-xapi-environment composite action + +runs: + using: "composite" + steps: + - name: Uninstall unversioned packages and remove pins + shell: bash + # This should purge them from the cache, unversioned package have + # 'master' as its version + run: | + opam list | awk -F " " '$2 == "master" { print $1 }' | xargs opam uninstall + opam pin list | cut -f1 -d "." | xargs opam unpin diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index c5bff2cad77..6a58aa6ce83 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -36,13 +36,16 @@ jobs: name: SDK_Source_PowerShell path: _build/install/default/xapi/sdk/powershell/* + - name: Cleanup XenAPI environment + uses: ./.github/workflows/cleanup-xapi-environment + build-csharp-sdk: name: Build C# SDK runs-on: windows-2022 needs: generate-sdk-sources - steps: + steps: - name: Strip 'v' prefix from xapi version - shell : pwsh + shell: pwsh run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve C# SDK source @@ -74,7 +77,7 @@ jobs: runs-on: windows-2019 steps: - name: Strip 'v' prefix from xapi version - shell : pwsh + shell: pwsh run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve PowerShell SDK source @@ -148,7 +151,7 @@ jobs: steps: - name: Strip 'v' prefix from xapi version - shell : pwsh + shell: pwsh run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve PowerShell SDK source diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6400d6cc97d..3f077bbf3b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -143,12 +143,8 @@ jobs: - name: Check disk space run: df -h || true - - name: Uninstall unversioned packages and remove pins - # This should purge them from the cache, unversioned package have - # 'master' as its version - run: | - opam list | awk -F " " '$2 == "master" { print $1 }' | xargs opam uninstall - opam pin list | cut -f1 -d "." | xargs opam unpin + - name: Cleanup XenAPI environment + uses: ./.github/workflows/cleanup-xapi-environment deprecation-test: name: Deprecation tests @@ -172,7 +168,7 @@ jobs: level: error test-sdk-builds: - name: Test SDK builds + name: Test SDK builds uses: ./.github/workflows/generate-and-build-sdks.yml with: # Ensure you also update ocaml-tests diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index ca0863dfe32..702162d42f3 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -18,7 +18,7 @@ runs: curl --fail --silent https://raw.githubusercontent.com/xapi-project/xs-opam/master/tools/xs-opam-ci.env | cut -f2 -d " " > .env - name: Download XE_SR_ERRORCODES.xml - shell : bash + shell: bash run: | mkdir -p /opt/xensource/sm wget -O /opt/xensource/sm/XE_SR_ERRORCODES.xml https://raw.githubusercontent.com/xapi-project/sm/master/drivers/XE_SR_ERRORCODES.xml From b95cbccaefbdb520d26634ac021cfe8410bc31e7 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 14 Feb 2024 10:32:22 -0500 Subject: [PATCH 52/99] Misc changes to SDK actions - Upload PS DLLs as part of SDK artefacts - Use one call to upload all artefacts in release.yml Signed-off-by: Danilo Del Busso --- .github/workflows/generate-and-build-sdks.yml | 4 ++-- .github/workflows/release.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 6a58aa6ce83..a0bee3a7fd0 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -129,7 +129,7 @@ jobs: New-Item -Path "." -Name "output" -ItemType "directory" Copy-Item -Verbose "source\README_51.md" -Destination "output" -Force Copy-Item -Verbose "source\LICENSE" -Destination "output" -Force - Copy-Item -Path "source\src\bin\Release\net45\*" -Include ".dll" "output\" + Copy-Item -Path "source\src\bin\Release\net45\*" -Include "*.dll" "output\" Get-ChildItem -Path "source" |` Where-Object { $_.Extension -eq ".ps1" -or $_.Extension -eq ".ps1xml" -or $_.Extension -eq ".psd1" -or $_.Extension -eq ".txt" } |` ForEach-Object -Process { Copy-Item -Verbose $_.FullName -Destination "output" } @@ -199,7 +199,7 @@ jobs: New-Item -Path "." -Name "output" -ItemType "directory" Copy-Item -Verbose "source\README.md" -Destination "output" -Force Copy-Item -Verbose "source\LICENSE" -Destination "output" -Force - Copy-Item -Path "source\src\bin\Release\net${{ matrix.dotnet }}.0\*" -Include ".dll" "output\" + Copy-Item -Path "source\src\bin\Release\net${{ matrix.dotnet }}.0\*" -Include "*.dll" "output\" Get-ChildItem -Path "source" |` Where-Object { $_.Extension -eq ".ps1" -or $_.Extension -eq ".ps1xml" -or $_.Extension -eq ".psd1" -or $_.Extension -eq ".txt" } |` ForEach-Object -Process { Copy-Item -Verbose $_.FullName -Destination "output" } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 48e7f81a839..cf7ebebd2a1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,9 +81,9 @@ jobs: - name: Create release ${{ github.ref_name }} shell: bash run: | - gh release create ${{ github.ref_name }} --repo ${{ github.repository }} --generate-notes dist/* - gh release upload ${{ github.ref_name }} --repo ${{ github.repository }} PowerShell-SDK-5.x-prerelease-unsigned.zip - gh release upload ${{ github.ref_name }} --repo ${{ github.repository }} PowerShell-SDK-7.x-prerelease-unsigned.zip + gh release create ${{ github.ref_name }} --repo ${{ github.repository }} --generate-notes dist/* \ + PowerShell-SDK-5.x-prerelease-unsigned.zip \ + PowerShell-SDK-7.x-prerelease-unsigned.zip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 227a1d70c91af3f394389435ada79e40018d6835 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 19 Feb 2024 10:32:59 -0500 Subject: [PATCH 53/99] Use consistent artefact naming for SDK binaries Other binaries are of the form `SDK_Binaries_XYZ` Signed-off-by: Danilo Del Busso --- .github/workflows/generate-and-build-sdks.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index a0bee3a7fd0..92f0f52b854 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -137,7 +137,7 @@ jobs: - name: Store PowerShell SDK (.NET Framework 4.5) uses: actions/upload-artifact@v3 with: - name: XenServerPowerShell_NET45 + name: SDK_Binaries_XenServerPowerShell_NET45 path: output/**/* build-powershell-7x-sdk: @@ -207,5 +207,5 @@ jobs: - name: Store PowerShell SDK (.NET ${{ matrix.dotnet }}) uses: actions/upload-artifact@v3 with: - name: XenServerPowerShell_NET${{ matrix.dotnet }} + name: SDK_Binaries_XenServerPowerShell_NET${{ matrix.dotnet }} path: output/**/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf7ebebd2a1..9045949aea2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,13 +61,13 @@ jobs: - name: Retrieve PowerShell 5.x SDK distribution artifacts uses: actions/download-artifact@v3 with: - name: XenServerPowerShell_NET45 + name: SDK_Binaries_XenServerPowerShell_NET45 path: sdk_powershell_5x/ - name: Retrieve PowerShell 7.x SDK distribution artifacts uses: actions/download-artifact@v3 with: - name: XenServerPowerShell_NET8 + name: SDK_Binaries_XenServerPowerShell_NET6 path: sdk_powershell_7x/ - name: Zip PowerShell 5.x SDK artifacts for deployment From 8596e9852a5b51c02784d1d1a87bf2056a13e93d Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Tue, 20 Feb 2024 15:59:31 +0000 Subject: [PATCH 54/99] CP-46151: Productise the observer.py. Instead of being a noop, it will now automatically instrument the python code given to it and create spans accordingly. The trace created will link to the xapi code that called it. It will export to all endpoints that it is given, which it reads from the observer.conf files created by xapi. Signed-off-by: Steven Woods Co-authored-by: Bernhard Kaindl Co-authored-by: Marcus Granado <590521+mg12@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- pyproject.toml | 2 +- python3/packages/__init__.py | 0 python3/packages/observer.py | 425 ++++++++++++++++++++++++++++++++- python3/tests/__init__.py | 0 python3/tests/test_observer.py | 133 +++++++++++ 6 files changed, 550 insertions(+), 12 deletions(-) create mode 100644 python3/packages/__init__.py create mode 100644 python3/tests/__init__.py create mode 100644 python3/tests/test_observer.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b660722b20..6662dc218ea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,7 +51,7 @@ jobs: - name: Install dependencies only needed for python 3 if: ${{ matrix.python-version != '2.7' }} - run: pip install pandas pytype toml + run: pip install pandas pytype toml wrapt - name: Install common dependencies for Python ${{matrix.python-version}} run: pip install future mock pytest-coverage pytest-mock diff --git a/pyproject.toml b/pyproject.toml index d05d028fbad..505f16e430c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,8 +39,8 @@ ensure_newline_before_comments = false # Note mypy has no config setting for PYTHONPATH, so you need to call it with: # PYTHONPATH="scripts/examples/python:.:scripts:scripts/plugins:scripts/examples" files = [ + "python3", "scripts/usb_reset.py", - "scripts/unit_tests", ] pretty = true error_summary = true diff --git a/python3/packages/__init__.py b/python3/packages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python3/packages/observer.py b/python3/packages/observer.py index a2538304e19..1eee0f2a7d6 100644 --- a/python3/packages/observer.py +++ b/python3/packages/observer.py @@ -1,13 +1,418 @@ -if __name__ == '__main__': - # run a program passed as parameter, with its original arguments - import runpy - import sys +# Copyright (C) Cloud Software Group. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; version 2.1 only. with the special +# exception on linking described in file LICENSE. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# - # shift all the argvs one left - sys.argv = sys.argv[1:] - argv0=sys.argv[0] +""" +Calls the passed script with its original arguments, instrumenting it to make a +trace of all function calls if at least one *observer.conf file exists in the +OBSERVER_CONFIG_DIR directory. - def run(file): - runpy.run_path(file, run_name='__main__') +If there are no *observer.conf files or something fails, this script runs the +passed script without any instrumentation. +""" - run(argv0) +import configparser +import functools +import inspect +import logging +import os +import runpy +import sys +import traceback +from datetime import datetime, timezone +from logging.handlers import SysLogHandler +from typing import List, Sequence + +# The opentelemetry library may generate exceptions we aren't expecting, this code +# must not fail or it will cause the pass-through script to fail when at worst +# this script should be a noop. As such, we sometimes need to catch broad exceptions: +# pylint: disable=broad-exception-caught, too-many-locals, too-many-statements +# wrapt.decorator adds the extra parameters so we shouldn't provide them: +# pylint: disable=no-value-for-parameter +# We only want to import opentelemetry libraries if instrumentation is enabled +# pylint: disable=import-outside-toplevel + +DEBUG_ENABLED = False +DEFAULT_MODULES = "LVHDSR,XenAPI,SR,SRCommand,util" +FORMAT = "observer.py: %(message)s" +handler = SysLogHandler(facility="local5", address="/dev/log") +logging.basicConfig(format=FORMAT, handlers=[handler]) +syslog = logging.getLogger(__name__) +if DEBUG_ENABLED: + syslog.setLevel(logging.DEBUG) +else: + syslog.setLevel(logging.INFO) +debug = syslog.debug + + +def _get_configs_list(config_dir): + try: + # There can be many observer config files in the configuration directory + return [ + f"{config_dir}/{f}" + for f in os.listdir(config_dir) + if os.path.isfile(os.path.join(config_dir, f)) + and f.endswith("observer.conf") + ] + except FileNotFoundError as err: + debug("configs exception: %s", err) + return [] + + +def read_config(config_path, header): + """Read a config file and return a dictionary of key-value pairs.""" + + parser = configparser.ConfigParser() + with open(config_path, encoding="utf-8") as config_file: + try: + parser.read_string(f"[{header}]\n{config_file.read()}") + except configparser.ParsingError as e: + debug("read_config(): invalid config file %s: %s", config_path, e) + return {} + + config = {k: v.strip("'") for k, v in dict(parser[header]).items()} + debug("%s: %s", config_path, config) + return config + + +def _span_noop(wrapped=None, span_name_prefix=""): + """Noop decorator. Overridden by _init_tracing() if there are configs.""" + if wrapped is None: + return functools.partial(_span_noop, span_name_prefix=span_name_prefix) + + return wrapped + + +def _patch_module_noop(_): + """Noop patch_module. Overridden by _init_tracing() if there are configs.""" + + +def _init_tracing(configs: List[str], config_dir: str): + """ + Initialise tracing with the given configuration files. + + If configs is empty, return the noop span and patch_module functions. + If configs are passed: + - Import the opentelemetry packages + - Read the configuration file + - Create a tracer + - Trace the script + - Return the span and patch_module functions for tracing the program + """ + if not configs: + return _span_noop, _patch_module_noop + + try: + from warnings import simplefilter + + # On 3.10-3.12, the import of wrapt might trigger warnings, filter them: + simplefilter(action="ignore", category=DeprecationWarning) + import wrapt # type: ignore[import-untyped] + from opentelemetry import context, trace + from opentelemetry.baggage.propagation import W3CBaggagePropagator + from opentelemetry.exporter.zipkin.json import ZipkinExporter + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + except ImportError as err: + syslog.error("missing opentelemetry dependencies: %s", err) + return _span_noop, _patch_module_noop + + try: + config_dict = read_config(f"{config_dir}/all.conf", header="default") + except FileNotFoundError: + config_dict = {} + module_names = config_dict.get("module_names", DEFAULT_MODULES).split(",") + debug("module_names: %s", module_names) + + # pylint: disable=too-few-public-methods + class FileZipkinExporter(ZipkinExporter): + """Class to export spans to a file in Zipkin format.""" + + def __init__(self, *args, **kwargs): + self.bugtool_filename_callback = kwargs.pop("filename_callback") + self.bugtool_filename = self.bugtool_filename_callback() + self.trace_log_dir = kwargs.pop("trace_log_dir") + debug("bugtool filename=%s", self.bugtool_filename) + self.bytes_written = 0 + super().__init__(*args, **kwargs) + + def export(self, spans: Sequence[trace.Span]) -> SpanExportResult: + """Export the given spans to the file endpoint.""" + + data = self.encoder.serialize(spans, self.local_node) + datastr = str(data) + debug("data.type=%s,data.len=%s", type(data), len(datastr)) + debug("data=%s", datastr) + os.makedirs(name=self.trace_log_dir, exist_ok=True) + + with open(self.bugtool_filename, "a", encoding="utf-8") as bugtool_file: + bugtool_file.write(f"{datastr}\n") # ndjson + self.bytes_written += len(data) + + # Create new file if it gets > 1MB + if self.bytes_written > 1024 * 1024: + self.bugtool_filename = self.bugtool_filename_callback() + self.bytes_written = 0 + + return SpanExportResult.SUCCESS + + def create_tracer_from_config(path): + """Create a tracer from a config file.""" + + otelvars = "opentelemetry-python.readthedocs.io/en/latest/sdk/environment_variables.html" + config = read_config(path, header=otelvars) + config_otel_resource_attrs = config.get("otel_resource_attributes", "") + + if config_otel_resource_attrs: + # OTEL requires some attributes e.g. service.name + # to be in the environment variable + os.environ["OTEL_RESOURCE_ATTRIBUTES"] = config_otel_resource_attrs + + trace_log_dir = config.get("xs_exporter_bugtool_endpoint", "") + + zipkin_endpoints = config.get("xs_exporter_zipkin_endpoints") + otel_exporter_zipkin_endpoints = ( + zipkin_endpoints.split(",") if zipkin_endpoints else [] + ) + otel_resource_attrs = dict( + item.split("=") + for item in config.get("otel_resource_attributes", "").split(",") + if "=" in item + ) + + service_name = config.get( + "otel_service_name", otel_resource_attrs.get("service.name", "unknown") + ) + host_uuid = otel_resource_attrs.get("xs.host.uuid", "unknown") + # Remove . to prevent users changing directories in the bugtool_filenamer + tracestate = os.getenv("TRACESTATE", "unknown").strip("'").replace(".", "") + + # rfc3339 + def bugtool_filenamer(): + """Return an rfc3339-compliant ndjson file name.""" + now = datetime.now(timezone.utc).isoformat() + return ( + f"{trace_log_dir}/{service_name}-{host_uuid}-{tracestate}-{now}.ndjson" + ) + + traceparent = os.getenv("TRACEPARENT", None) + propagator = TraceContextTextMapPropagator() + context_with_traceparent = propagator.extract({"traceparent": traceparent}) + + context.attach(context_with_traceparent) + + # Create a tracer provider with the given resource attributes + provider = TracerProvider( + resource=Resource.create( + W3CBaggagePropagator().extract({}, otel_resource_attrs) + ) + ) + + # Add a span processor for each endpoint defined in the config + if trace_log_dir: + processor_file_zipkin = BatchSpanProcessor( + FileZipkinExporter( + filename_callback=bugtool_filenamer, + trace_log_dir=trace_log_dir + ) + ) + provider.add_span_processor(processor_file_zipkin) + for zipkin_endpoint in otel_exporter_zipkin_endpoints: + processor_zipkin = BatchSpanProcessor( + ZipkinExporter(endpoint=zipkin_endpoint) + ) + provider.add_span_processor(processor_zipkin) + + trace.set_tracer_provider(provider) + return trace.get_tracer(__name__) + + tracers = list(map(create_tracer_from_config, configs)) + debug("tracers=%s", tracers) + + def span_of_tracers(wrapped=None, span_name_prefix=""): + """ + Public decorator that creates a trace around a function. + + If there are no tracers, the function is called without any tracing. + If there are tracers, the function is called with a trace around it. + + It creates a span with the given span name prefix and then clones + the returned span for each of the existing traces to produce a nested + trace for each of them. + + Args: + wrapped: The function to be traced. + span_name_prefix: The prefix to be added to the span name. + + Returns: + The decorated function or a partial function if wrapped is None. + + If wrapped is None, the decorator is being used with parameters and + a partial function is returned instead of the decorated function so + that the function is decorated properly on the second pass. + """ + if wrapped is None: # handle decorators with parameters + return functools.partial(span_of_tracers, span_name_prefix=span_name_prefix) + + @wrapt.decorator + def instrument_function(wrapped, _, args, kwargs): + """Decorator that creates a trace around a function.""" + if not tracers: + return wrapped(*args, **kwargs) + + module_name = wrapped.__module__ if hasattr(wrapped, "__module__") else "" + qual_name = wrapped.__qualname__ if hasattr(wrapped, "__qualname__") else "" + + if not module_name and not qual_name: + span_name = str(wrapped) + else: + prefix = f"{span_name_prefix}:" if span_name_prefix else "" + span_name = f"{prefix}{module_name}:{qual_name}" + + tracer = tracers[0] + with tracer.start_as_current_span(span_name) as aspan: + if inspect.isclass(wrapped): + # class or classmethod + aspan.set_attribute("xs.span.args.str", str(args)) + aspan.set_attribute("xs.span.kwargs.str", str(kwargs)) + else: + # function, staticmethod or instancemethod + bound_args = inspect.signature(wrapped).bind(*args, **kwargs) + bound_args.apply_defaults() + for k, v in bound_args.arguments.items(): + aspan.set_attribute(f"xs.span.arg.{k}", str(v)) + + # must be inside "aspan" to produce nested trace + result = wrapped(*args, **kwargs) + return result + + def autoinstrument_class(aclass): + """Auto-instrument a class.""" + + t = tracers[0] + module_name = f"{aclass.__module__}:{aclass.__qualname__}" + + with t.start_as_current_span(f"auto_instrumentation.add: {module_name}"): + for method_name, method in aclass.__dict__.items(): + if not callable(getattr(aclass, method_name)): + continue + + with t.start_as_current_span( + f"class.instrument:{module_name}.{method_name}={method}" + ): + # Avoid RecursionError: + # 'maximum recursion depth exceeded in comparison' + # in the XenAPI module (triggered by XMLRPC calls in it): + if method_name in ["__getattr__", "__call__", "__init__"]: + continue + try: + setattr(aclass, method_name, instrument_function(method)) + except Exception: + debug( + "setattr.instrument_function: Exception %s", + traceback.format_exc(), + ) + + + def autoinstrument_module(amodule): + """Autoinstrument the classes and functions in a module.""" + + # Instrument the methods of the classes in the module + for _, aclass in inspect.getmembers(amodule, inspect.isclass): + try: + autoinstrument_class(aclass) + except Exception: + debug("instrument_function: Exception %s", traceback.format_exc()) + + # Instrument the module-level functions of the module + for fname, afunction in inspect.getmembers(amodule, inspect.isfunction): + setattr(amodule, fname, instrument_function(afunction)) + + if inspect.ismodule(wrapped): + autoinstrument_module(wrapped) + + return instrument_function(wrapped) + + def _patch_module(module_name): + wrapt.importer.discover_post_import_hooks(module_name) + wrapt.importer.when_imported(module_name)( + lambda hook: span_of_tracers(wrapped=hook) + ) + + for m in module_names: + _patch_module(m) + + return span_of_tracers, _patch_module + + +observer_config_dir = os.getenv("OBSERVER_CONFIG_DIR", default=".").strip("'") +observer_configs = _get_configs_list(observer_config_dir) +debug("configs = %s", observer_configs) + +try: + # If there are configs, span and patch_module are now operational + # and can be used to trace the program. + # If there are no configs, or an exception is raised, span and patch_module + # are not overridden and will be the defined no-op functions. + span, patch_module = _init_tracing(observer_configs, observer_config_dir) +except Exception as exc: + syslog.error("Exception while setting up tracing, running script untraced: %s", exc) + span, patch_module = _span_noop, _patch_module_noop + + +def main(): + """ + Run the passed python script using the runpy module, passing the given arguments. + + The program will be automatically instrumented when the corresponding module + in the program is imported. + """ + + # When sys.argv has only argv[0], but no command to call, exit with an error message + if len(sys.argv) < 2: + print(__file__ + ": usage: command argument list", file=sys.stderr) + return 31 # EINVAL + + # Shift the arguments by one so that the program to run is first in sys.argv + sys.argv = sys.argv[1:] + argv0 = sys.argv[0] + + @span(span_name_prefix=argv0) + def run(file): + """Run the given python file calling its __main__ function.""" + + # Defensive error handling should hopefully only be needed in exceptional cases + # but in case things go wrong, this may be a starting point for logging it: + try: + runpy.run_path(file, run_name="__main__") + return 0 + except FileNotFoundError as e: + print( + f"{__file__}: {' '.join(sys.argv)}\n{e.filename}: No such file", + file=sys.stderr, + ) + return 2 + except Exception: + print( + f"{__file__}: {' '.join(sys.argv)}\n{traceback.format_exc()}", + file=sys.stderr) + return 139 # This is what the default SIGSEGV handler on Linux returns + + return run(argv0) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python3/tests/__init__.py b/python3/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python3/tests/test_observer.py b/python3/tests/test_observer.py new file mode 100644 index 00000000000..53944d97ca9 --- /dev/null +++ b/python3/tests/test_observer.py @@ -0,0 +1,133 @@ +"""Test python3/packages/observer.py""" + +import os +import sys +import unittest + +from mock import MagicMock, mock_open, patch + +# Ensure observer is initialised as noop +with patch("os.listdir") as mock_listdir: + # Prevent it finding an observer.conf + mock_listdir.return_value = [] + from packages import observer + +# mock modules to avoid dependencies +sys.modules["opentelemetry"] = MagicMock() +sys.modules["opentelemetry.sdk.resources"] = MagicMock() +sys.modules["opentelemetry.sdk.trace"] = MagicMock() +sys.modules["opentelemetry.sdk.trace.export"] = MagicMock() +sys.modules["opentelemetry.exporter.zipkin.json"] = MagicMock() +sys.modules["opentelemetry.baggage.propagation"] = MagicMock() +sys.modules["opentelemetry.trace.propagation.tracecontext"] = MagicMock() +sys.modules["opentelemetry.context"] = MagicMock() +sys.modules["opentelemetry.trace"] = MagicMock() + +TEST_CONFIG = """ + XS_EXPORTER_BUGTOOL_ENDPOINT='/var/log/dt/test' + OTEL_SERVICE_NAME='test-observer' + OTEL_RESOURCE_ATTRIBUTES='service.name=sm' + """ +TEST_OBSERVER_CONF = "test-observer.conf" +OBSERVER_OPEN = "packages.observer.open" + + +# pylint: disable=missing-function-docstring,protected-access +class TestObserver(unittest.TestCase): + """Test python3/packages/observer.py""" + + def simple_method(self): + """A simple helper method for tests to wrap using observer.span""" + return 5 + + def init_tracing_and_run_simple_method(self, read_data): + """Run the init_tracing method with the given read_data""" + + with patch(OBSERVER_OPEN, mock_open(read_data=read_data)): + span, _ = observer._init_tracing([TEST_OBSERVER_CONF], ".") + + simple_method = span(self.simple_method) + self.assertEqual(simple_method(), 5) + + def test_span_with_parameters(self): + span, _ = observer._init_tracing([], ".") + + # This needs to use the decorator sugar so that wrapped is initially None + @span(span_name_prefix="test") + def simple_method(): + return 5 + + self.assertEqual(simple_method(), 5) + + def test_span_of_tracers_with_parameters(self): + with patch(OBSERVER_OPEN, mock_open(read_data="empty")): + span, _ = observer._init_tracing([TEST_OBSERVER_CONF], ".") + + # This needs to use the decorator sugar so that wrapped is initially None + @span(span_name_prefix="test") + def simple_method(): + return 5 + + self.assertEqual(simple_method(), 5) + + # Use the span method defined on import with no configs + def test_span_after_import(self): + simple_method = observer.span(self.simple_method) + + self.assertEqual(simple_method(), 5) + + # Check the span is a noop when configs is empty + def test_configs_empty(self): + span, _ = observer._init_tracing([], ".") + + simple_method = span(self.simple_method) + + self.assertEqual(simple_method(), 5) + + @patch("os.path.isfile") + @patch("os.listdir") + def test_get_configs_list(self, mock_ls, mock_isfile): + mock_ls.return_value = [ + "first-observer.conf", + "ignore.conf", + "second-observer.conf", + ] + mock_isfile.return_value = True + configs = observer._get_configs_list("test-dir") + + self.assertEqual( + configs, ["test-dir/first-observer.conf", "test-dir/second-observer.conf"] + ) + + def test_configs_directory_not_found(self): + configs = observer._get_configs_list("non-existent") + + self.assertEqual(configs, []) + + def test_configs_exist(self): + self.init_tracing_and_run_simple_method(read_data=TEST_CONFIG) + self.assertEqual(os.environ["OTEL_RESOURCE_ATTRIBUTES"], "service.name=sm") + + def test_all_conf_missing(self): + with patch(OBSERVER_OPEN) as mock_file: + mock_file.return_value.__enter__.side_effect = [ + FileNotFoundError, + mock_open(read_data="empty").return_value, + ] + span, _ = observer._init_tracing([TEST_OBSERVER_CONF], ".") + + simple_method = span(self.simple_method) + + self.assertEqual(simple_method(), 5) + + # A method to decorate with observer.span + def simple_method_with_args(self, a, b=3): + return a + b + + def test_tracing_with_arguments(self): + with patch(OBSERVER_OPEN, mock_open(read_data="empty")): + span, _ = observer._init_tracing([TEST_OBSERVER_CONF], ".") + + simple_method_with_args = span(self.simple_method_with_args) + + self.assertEqual(simple_method_with_args(5), 8) From 711a82c5fd156a4f9c095b2ce6a670d21e8d5191 Mon Sep 17 00:00:00 2001 From: Gabriel Buica Date: Tue, 6 Feb 2024 13:24:57 +0000 Subject: [PATCH 55/99] CP-46157: Add unit test for `observed_components_of` Adds a unit test for `observed_components_of`. Tests over a list of inputs fot `observed_components_of_inputs` while changing the value of `Xapi_globs.observer_experimental_components` with: `no_exp_comp` -- empty `StringSet`; `smapi_exp_comp` -- singleton `StringSet` that represents smapi. Signed-off-by: Gabriel Buica --- ocaml/tests/test_observer.ml | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/ocaml/tests/test_observer.ml b/ocaml/tests/test_observer.ml index 69c2f754c93..165bc4afd1b 100644 --- a/ocaml/tests/test_observer.ml +++ b/ocaml/tests/test_observer.ml @@ -16,6 +16,19 @@ open Tracing module D = Debug.Make (struct let name = "test_observer" end) open D +module ComponentSet = Set.Make (Xapi_observer_components) + +let component_set_to_string cs = + cs + |> ComponentSet.to_seq + |> Seq.map Xapi_observer_components.to_string + |> List.of_seq + |> String.concat "," + +let component_set_testable = + let pp = Fmt.of_to_string component_set_to_string in + + Alcotest.testable pp ComponentSet.equal let () = Printexc.record_backtrace true @@ -515,6 +528,47 @@ let test_attribute_validation () = List.iter test_good_attribute good_attributes ; List.iter test_bad_attribute bad_attributes +let test_observed_components_of () = + let open Xapi_globs in + let open Xapi_observer_components in + let original_value = !observer_experimental_components in + let remove comp = List.filter (fun c -> comp <> c) in + let inputs = [[]; [SMApi]; [Xapi; SMApi]; all] in + + let expected_components_given_config_value = + [ + ( "No experimental component is expected" + , StringSet.empty + , [remove Xapi_clusterd all; [SMApi]; [Xapi; SMApi]; all] + ) + ; ( "SMapi is experimental component" + , StringSet.singleton Constants.observer_component_smapi + , [ + all |> remove Xapi_clusterd |> remove SMApi + ; [] + ; [Xapi] + ; remove SMApi all + ] + ) + ] + in + + let test_exp_comp (msg, v, expected_list) = + Xapi_globs.observer_experimental_components := v ; + let observed_components = List.map observed_components_of inputs in + List.iter2 + (fun expected received -> + Alcotest.(check component_set_testable) + msg + (ComponentSet.of_list expected) + (ComponentSet.of_list received) + ) + expected_list observed_components + in + + List.iter test_exp_comp expected_components_given_config_value ; + observer_experimental_components := original_value + let test = [ ( "test_observer_create_and_destroy" @@ -528,6 +582,7 @@ let test = ; ("test_hashtbl_leaks", `Quick, test_hashtbl_leaks) ; ("test_tracing_exn_backtraces", `Quick, test_tracing_exn_backtraces) ; ("test_attribute_validation", `Quick, test_attribute_validation) + ; ("test_observed_components_of", `Quick, test_observed_components_of) ] let () = From dc7bcfb240d97fb0f506c39befff840fa1f8c650 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Thu, 7 Mar 2024 15:01:30 +0000 Subject: [PATCH 56/99] opam: add hex to xapi dependencies This direct dependency was missing Signed-off-by: Pau Ruiz Safont --- xapi.opam | 1 + xapi.opam.template | 1 + 2 files changed, 2 insertions(+) diff --git a/xapi.opam b/xapi.opam index 020358cf4d8..e414d694b2c 100644 --- a/xapi.opam +++ b/xapi.opam @@ -24,6 +24,7 @@ depends: [ "domain-name" "ezxenstore" "fmt" {with-test} + "hex" "http-lib" {with-test} # the public library is only used for testing "ipaddr" "mirage-crypto" {with-test} diff --git a/xapi.opam.template b/xapi.opam.template index 8f3e119ec99..dc48554787e 100644 --- a/xapi.opam.template +++ b/xapi.opam.template @@ -22,6 +22,7 @@ depends: [ "domain-name" "ezxenstore" "fmt" {with-test} + "hex" "http-lib" {with-test} # the public library is only used for testing "ipaddr" "mirage-crypto" {with-test} From 316dc9793e2763fb0517d0e96c1a231a696fa30b Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 18 Oct 2023 14:43:10 +0000 Subject: [PATCH 57/99] CP-45888: Add `JsonRpcClient` classes Also update `pom.xml` to remove `org.apache.xmlrpc` packages and add `jackson-databind` and `org.apache.httpcomponents.client5` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/autogen/xen-api/pom.xml | 27 +-- .../com/xensource/xenapi/JsonRpcClient.java | 200 ++++++++++++++++++ .../{Marshalling.java => JsonRpcRequest.java} | 76 +++---- .../com/xensource/xenapi/JsonRpcResponse.java | 59 ++++++ .../xenapi/JsonRpcResponseError.java | 54 +++++ 5 files changed, 352 insertions(+), 64 deletions(-) create mode 100644 ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java rename ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/{Marshalling.java => JsonRpcRequest.java} (51%) create mode 100644 ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponse.java create mode 100644 ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponseError.java diff --git a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml index 471568f1179..c78ec926540 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml +++ b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml @@ -53,29 +53,14 @@ - org.apache.xmlrpc - xmlrpc-client - 3.1.3 + com.fasterxml.jackson.core + jackson-databind + 2.15.1 - org.apache.xmlrpc - xmlrpc-common - 3.1.3 - - - org.apache.ws.commons.util - ws-commons-util - 1.0.2 - - - junit - junit - - - xml-apis - xml-apis - - + org.apache.httpcomponents.client5 + httpclient5 + 5.2.1 diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java new file mode 100644 index 00000000000..8747822672c --- /dev/null +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -0,0 +1,200 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.xensource.xenapi; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.cookie.StandardCookieSpec; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.io.entity.StringEntity; + +import java.io.IOException; +import java.net.URL; +import java.net.http.HttpClient; +import java.text.SimpleDateFormat; +import java.util.concurrent.TimeUnit; + +/** + * Provides a JSON-RPC v2.0 client for making remote procedure calls to xapi's backend URL. + *
+ * This class enables the communication to the JSON-RPC backend. The client utilizes the HttpClient class for + * sending HTTP POST requests with JSON payloads and the ObjectMapper class from the Jackson library for + * serialization and deserialization of JSON data. + *
+ * The client can be customised by passing it as a parameter to corresponding constructor, enabling custom + * handling of requests. + *
+ * By default, the timeout for requests is set to 10 minutes (600 seconds). The default timeout for connecting to the + * JSON-RPC backend is set to 5 seconds. + * + * @see HttpClient HttpClient is used to make requests and connect to the backend + * @see ObjectMapper ObjectMapper is used to marshall requests and responses + */ +public class JsonRpcClient { + private static final int DEFAULT_REQUEST_TIMEOUT = 600; + private static final int DEFAULT_CONNECTION_TIMEOUT = 5; + + protected static final int MAX_CONCURRENT_CONNECTIONS = 10; + + protected static final String JSON_BACKEND_PATH = "/jsonrpc"; + + protected final CloseableHttpClient httpClient; + protected final String jsonRpcBackendUrl; + protected final ObjectMapper objectMapper; + protected final int requestTimeout; + + private final RequestConfig defaultRequestConfig = RequestConfig.custom() + .setCookieSpec(StandardCookieSpec.IGNORE) + .build(); + + /** + * Create a JsonRpcClient with default settings. + * + * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. + * @see JsonRpcClient JsonRpcClient for more info on using this class + */ + public JsonRpcClient(URL jsonRpcBackendUrl) { + this(jsonRpcBackendUrl, DEFAULT_REQUEST_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT); + } + + /** + * Create a JsonRpcClient with the option to define the request and connection timeout values. + * + * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. + * @param requestTimeout the timeout value for requests. + * @param connectionTimeout the timeout value for the initial connection to the host. + * @see JsonRpcClient JsonRpcClient for more info on using this class + */ + public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTimeout) { + var connectionConfig = ConnectionConfig + .custom() + .setConnectTimeout(connectionTimeout, TimeUnit.SECONDS) + .build(); + + var connectionManager = new PoolingHttpClientConnectionManager(); + connectionManager.setDefaultConnectionConfig(connectionConfig); + connectionManager.setMaxTotal(MAX_CONCURRENT_CONNECTIONS); + + this.httpClient = HttpClients + .custom() + .setConnectionManager(connectionManager) + .build(); + this.jsonRpcBackendUrl = formatBackendUrl(jsonRpcBackendUrl); + this.requestTimeout = requestTimeout; + this.objectMapper = new ObjectMapper(); + initializeObjectMapperConfiguration(); + } + + /** + * Initialize a JsonRpcClient using a custom HttpClient instance. + * + * @param client the custom HttpClient to use for all requests + * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. + * @param requestTimeout the timeout value for requests. + * @see HttpClient + * @see JsonRpcClient JsonRpcClient for more info on using this class + */ + public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requestTimeout) { + httpClient = client; + + this.requestTimeout = requestTimeout; + this.jsonRpcBackendUrl = formatBackendUrl(jsonRpcBackendUrl); + + this.objectMapper = new ObjectMapper(); + initializeObjectMapperConfiguration(); + } + + /** + * Send a method call to xapi's backend. You need to provide the type of the data returned by a successful response. + * + * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password + * @param methodParameters the parameters of the method call + * @param responseTypeReference the type of the response, wrapped with a TypeReference + * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records + * @return a JsonRpcResponse object. If its error field is empty, the response was successful. + * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON + * @throws IOException if an I/O error occurs when sending or receiving + */ + public JsonRpcResponse sendRequest(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws IOException { + var requestBody = objectMapper + .writeValueAsString(new JsonRpcRequest(methodCall, methodParameters)); + + var requestEntity = new StringEntity(requestBody, ContentType.APPLICATION_JSON); + + var requestConfig = RequestConfig.copy(defaultRequestConfig) + .setConnectionRequestTimeout(this.requestTimeout, TimeUnit.SECONDS) + .build(); + + var request = new HttpPost(this.jsonRpcBackendUrl); + request.setConfig(requestConfig); + request.setEntity(requestEntity); + + return httpClient.execute(request, response -> { + try (response) { + var typeFactory = objectMapper.getTypeFactory(); + var responseObjectType = typeFactory.constructType(responseTypeReference.getType()); + var type = typeFactory.constructParametricType(JsonRpcResponse.class, responseObjectType); + + var responseContent = response.getEntity().getContent(); + return objectMapper.readValue(responseContent, type); + } + }); + } + + /** + * Helper method to initialize jackson's ObjectMapper. + */ + private void initializeObjectMapperConfiguration() { + this.objectMapper.setDateFormat(new SimpleDateFormat("yyyyMMdd'T'HH:mm:ss'Z'")); + } + + /** + * Format input URL to the form protocol://host/jsonrpc + * + * @param url the input URL to format + * @return a string version of a valid xen-api backend URL + */ + private String formatBackendUrl(URL url) { + // We only replace it when it's empty. + // If the user purposely set the path + // we use the given value even if incorrect + if (!url.getPath().isEmpty()) { + return url.getProtocol() + "://" + url.getHost() + JSON_BACKEND_PATH; + } + return url.toString(); + } +} diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Marshalling.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcRequest.java similarity index 51% rename from ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Marshalling.java rename to ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcRequest.java index cc7177a92ea..fa49528c205 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Marshalling.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcRequest.java @@ -1,18 +1,18 @@ /* * Copyright (c) Cloud Software Group, Inc. - * + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: - * + * * 1) Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. - * + * * 2) Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS @@ -29,48 +29,38 @@ package com.xensource.xenapi; -import java.util.*; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; /** - * Marshalls Java types onto the wire. - * Does not cope with records. Use individual record.toMap() + * Represents the payload of a request sent to a + * JSON-RPC v2.0 backend. */ -public final class Marshalling { - /** - * Converts Integers to Strings - * and Sets to Lists recursively. - */ - public static Object toXMLRPC(Object o) { - if (o instanceof String || - o instanceof Boolean || - o instanceof Double || - o instanceof Date) { - return o; - } else if (o instanceof Long) { - return o.toString(); - } else if (o instanceof Map) { - Map result = new HashMap(); - Map m = (Map)o; - for (Object k : m.keySet()) - { - result.put(toXMLRPC(k), toXMLRPC(m.get(k))); - } - return result; - } else if (o instanceof Set) { - List result = new ArrayList(); - for (Object e : ((Set)o)) - { - result.add(toXMLRPC(e)); - } - return result; - } else if (o instanceof XenAPIObject) { - return ((XenAPIObject) o).toWireString(); - } else if (o instanceof Enum) { - return o.toString(); - }else if (o == null){ - return ""; - } else { - throw new RuntimeException ("=============don't know how to marshall:({[" + o + "]})"); +public class JsonRpcRequest { + @JsonProperty("jsonrpc") + public String jsonRpc; + public int id; + public String method; + @JsonProperty("params") + public Object[] parameters; + + public JsonRpcRequest(String method, Object[] parameters) { + this.method = method; + this.parameters = parameters; + this.jsonRpc = "2.0"; + this.id = 1; + } + + @Override + public String toString() { + try { + var mapper = new ObjectMapper(); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + return mapper.writeValueAsString(this); + } catch (JsonProcessingException ex) { + return "Error while processing object. Could not serialize as JSON."; } } } diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponse.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponse.java new file mode 100644 index 00000000000..11e2c152474 --- /dev/null +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponse.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.xensource.xenapi; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +/** + * Represents the payload of responses returned from a + * JSON-RPC v2.0 backend. + * @param The type of the response's result + */ +public class JsonRpcResponse { + @JsonProperty("jsonrpc") + public String jsonRpc; + public int id; + public T result; + public JsonRpcResponseError error; + + @Override + public String toString() { + try { + var mapper = new ObjectMapper(); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + return mapper.writeValueAsString(this); + } catch (JsonProcessingException ex) { + return "Error while processing object. Could not serialize as JSON"; + } + } +} diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponseError.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponseError.java new file mode 100644 index 00000000000..80cbabcb582 --- /dev/null +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponseError.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.xensource.xenapi; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +/** + * Represents the structure of an error returned by a + * JSON-RPC v2.0 backend. Does not apply to JSON-RPC v1.0. + */ +public class JsonRpcResponseError { + public int code; + public String message; + public String[] data; + + public String toString() { + try { + var mapper = new ObjectMapper(); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + return mapper.writeValueAsString(this); + } catch (JsonProcessingException ex) { + return "Error while processing object. Could not serialize as JSON."; + } + } +} From 2ec9542c0fd97a763daf5358eeeee625bbda8b9a Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 18 Oct 2023 14:43:10 +0000 Subject: [PATCH 58/99] CP-45888: Update `Connection.java` to use new `JsonRpcClient` Signed-off-by: Danilo Del Busso --- .../java/com/xensource/xenapi/Connection.java | 274 +++++++++--------- 1 file changed, 130 insertions(+), 144 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index ef5b137b315..f568c452894 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -29,228 +29,214 @@ package com.xensource.xenapi; -import java.net.URL; -import java.util.Map; -import java.util.TimeZone; - -import org.apache.xmlrpc.XmlRpcException; -import org.apache.xmlrpc.client.XmlRpcClient; -import org.apache.xmlrpc.client.XmlRpcClientConfigImpl; -import org.apache.xmlrpc.client.XmlRpcHttpClientConfig; - +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.xensource.xenapi.Types.BadServerResponse; import com.xensource.xenapi.Types.XenAPIException; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; + +import java.io.IOException; +import java.net.URL; /** - * Represents a connection to a XenServer. Creating a new instance of this class initialises a new XmlRpcClient that is - * then used by all method calls: each method call in xenapi takes a Connection as a parameter, composes an XMLRPC + * Represents a connection to a XenServer. Creating a new instance of this class initialises a new JsonRpcClient that is + * then used by all method calls: each method call in xen-api takes a Connection as a parameter, composes a JSON-RPC * method call, and dispatches it on the Connection's client via the dispatch method. */ -public class Connection -{ +public class Connection { + private final JsonRpcClient client; private APIVersion apiVersion; - - /** - * Default reply timeout for xml-rpc calls in seconds - */ - protected static final int DEFAULT_REPLY_TIMEOUT = 600; - /** - * Default connection timeout for xml-rpc calls in seconds + * The opaque reference to the session used by this connection */ - protected static final int DEFAULT_CONNECTION_TIMEOUT = 5; + private String sessionReference; /** - * Updated when Session.login_with_password() is called. + * Creates a connection to a particular server using a custom implementation of the JsonRpcClient. + *

+ * Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, + * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. + *

+ * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually + * logging out the Session. + * + * @param jsonRpcClient The JsonRpcClient used to connect to the JSON-RPC backed. */ - public APIVersion getAPIVersion() - { - return apiVersion; + public Connection(JsonRpcClient jsonRpcClient) { + this.client = jsonRpcClient; } /** - * The opaque reference to the session used by this connection - */ - private String sessionReference; - - /** - * As seen by the xmlrpc library. From our point of view it's a server. + * Creates a connection to a particular server using a given url. This object can then be passed + * in to any other API calls. + *

+ * Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, + * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. + *

+ * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually + * logging out the Session. + * + * @param httpClient The HttpClient used to make calls, this will be used by JsonRpcClient for handling requests + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param requestTimeout The reply timeout for JSON-RPC calls in seconds */ - private final XmlRpcClient client; + public Connection(CloseableHttpClient httpClient, URL url, int requestTimeout) { + this.client = new JsonRpcClient(httpClient, url, requestTimeout); + } /** * Creates a connection to a particular server using a given url. This object can then be passed * in to any other API calls. - * + *

* Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. - * + *

* When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. - * - * This constructor uses the default values of the reply and connection timeouts for the xmlrpc calls + *

+ * This constructor uses the default values of the reply and connection timeouts for the JSON-RPC calls * (600 seconds and 5 seconds respectively). * - * @param url The URL of the server to connect to + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. */ - public Connection(URL url) - { - this.client = getClientFromURL(url, DEFAULT_REPLY_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT); + public Connection(URL url) { + this.client = new JsonRpcClient(url); } /** * Creates a connection to a particular server using a given url. This object can then be passed * in to any other API calls. - * + *

* Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. - * + *

* When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. * - * @param url The URL of the server to connect to - * @param replyTimeout The reply timeout for xml-rpc calls in seconds - * @param connTimeout The connection timeout for xml-rpc calls in seconds + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param requestTimeout The reply timeout for JSON-RPC calls in seconds + * @param connectionTimeout The connection timeout for JSON-RPC calls in seconds */ - public Connection(URL url, int replyTimeout, int connTimeout) - { - this.client = getClientFromURL(url, replyTimeout, connTimeout); + public Connection(URL url, int requestTimeout, int connectionTimeout) { + this.client = new JsonRpcClient(url, requestTimeout, connectionTimeout); } - /** * Creates a connection to a particular server using a given url. This object can then be passed * in to any other API calls. - * - * This constructor uses the default values of the reply and connection timeouts for the xmlrpc calls + *

+ * Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, + * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. + *

+ * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually + * logging out the Session. + *

+ * This constructor uses the default values of the reply and connection timeouts for the JSON-RPC calls * (600 seconds and 5 seconds respectively). * - * @param url The URL of the server to connect to + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. * @param sessionReference A reference to a logged-in Session. Any method calls on this - * Connection will use it. This constructor does not call Session.loginWithPassword, and dispose() on the resulting - * Connection object does not call Session.logout. The programmer is responsible for ensuring the Session is logged - * in and out correctly. + * Connection will use it. This constructor does not call Session.loginWithPassword, and dispose() on the resulting + * Connection object does not call Session.logout. The programmer is responsible for ensuring the Session is logged + * in and out correctly. */ - public Connection(URL url, String sessionReference) - { - this.client = getClientFromURL(url, DEFAULT_REPLY_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT); + public Connection(URL url, String sessionReference) { + this.client = new JsonRpcClient(url); this.sessionReference = sessionReference; } /** * Creates a connection to a particular server using a given url. This object can then be passed * in to any other API calls. + *

+ * Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, + * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. + *

+ * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually + * logging out the Session. * - * @param url The URL of the server to connect to - * @param sessionReference A reference to a logged-in Session. Any method calls on this Connection will use it. - * This constructor does not call Session.loginWithPassword, and dispose() on the resulting - * Connection object does not call Session.logout. The programmer is responsible for - * ensuring the Session is logged in and out correctly. - * @param replyTimeout The reply timeout for xml-rpc calls in seconds - * @param connTimeout The connection timeout for xml-rpc calls in seconds + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param sessionReference A reference to a logged-in Session. Any method calls on this Connection will use it. + * This constructor does not call Session.loginWithPassword, and dispose() on the resulting + * Connection object does not call Session.logout. The programmer is responsible for + * ensuring the Session is logged in and out correctly. + * @param requestTimeout The reply timeout for JSON-RPC calls in seconds + * @param connectionTimeout The connection timeout for JSON-RPC calls in seconds */ - public Connection(URL url, String sessionReference, int replyTimeout, int connTimeout) - { - this.client = getClientFromURL(url, replyTimeout, connTimeout); + public Connection(URL url, String sessionReference, int requestTimeout, int connectionTimeout) { + this.client = new JsonRpcClient(url, requestTimeout, connectionTimeout); this.sessionReference = sessionReference; } - private XmlRpcClientConfigImpl config = new XmlRpcClientConfigImpl(); - - public XmlRpcClientConfigImpl getConfig() - { - return config; + /** + * Updated when Session.login_with_password() is called. + */ + public APIVersion getAPIVersion() { + return apiVersion; } - private XmlRpcClient getClientFromURL(URL url, int replyWait, int connWait) - { - config.setTimeZone(TimeZone.getTimeZone("UTC")); - config.setServerURL(url); - config.setReplyTimeout(replyWait * 1000); - config.setConnectionTimeout(connWait * 1000); - XmlRpcClient client = new XmlRpcClient(); - client.setConfig(config); - return client; + private void setAPIVersion(Session session) throws IOException { + try { + long major = session.getThisHost(this).getAPIVersionMajor(this); + long minor = session.getThisHost(this).getAPIVersionMajor(this); + apiVersion = APIVersion.fromMajorMinor(major, minor); + } catch (BadServerResponse exn) { + apiVersion = APIVersion.UNKNOWN; + } } /* * Because the binding calls are constructing their own parameter lists, they need to be able to get to * the session reference directly. This is all rather ugly and needs redone - * Changed to public to allow easier integration with HTTP-level streaming interface, - * see CA-15447 + * CA-15447: Changed to public in order to allow easier integration with HTTP-level streaming interface, */ - public String getSessionReference() - { + public String getSessionReference() { return this.sessionReference; } /** - * The (auto-generated parts of) the bindings dispatch XMLRPC calls on this Connection's client through this method. + * Send a method call to xapi's backend. You need to provide the type of the data returned by a successful response. + * + * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password + * @param methodParameters the methodParameters of the method call + * @param responseTypeReference the type of the response, wrapped with a TypeReference + * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records + * @return The result of the call with the type specified under T. + * @throws XenAPIException if the call failed. + * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON + * @throws IOException if an I/O error occurs when sending or receiving */ - protected Map dispatch(String method_call, Object[] method_params) throws XmlRpcException, XenAPIException - { - Map response = (Map) client.execute(method_call, method_params); + public T dispatch(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws XenAPIException, JsonProcessingException, IOException { + var result = client.sendRequest(methodCall, methodParameters, responseTypeReference); + if (result.error != null) { + throw new XenAPIException(String.valueOf(result.error)); + } - if (method_call.equals("session.login_with_password") && - response.get("Status").equals("Success")) - { - Session session = Types.toSession(response.get("Value")); + if (methodCall.equals("session.login_with_password")) { + var session = ((Session) result.result); sessionReference = session.ref; setAPIVersion(session); - } - else if (method_call.equals("session.slave_local_login_with_password") && - response.get("Status").equals("Success")) - { - sessionReference = Types.toSession(response.get("Value")).ref; + } else if (methodCall.equals("session.slave_local_login_with_password")) { + var session = ((Session) result.result); + sessionReference = session.ref; apiVersion = APIVersion.latest(); } - else if (method_call.equals("session.logout")) - { - // Work around a bug in XenServer 5.0 and below. - // session.login_with_password should have rejected us with - // HOST_IS_SLAVE, but instead we don't find out until later. - // We don't want to leak the session, so we need to log out - // this session from the master instead. - if (response.get("Status").equals("Failure")) - { - Object[] error = (Object[]) response.get("ErrorDescription"); - if (error.length == 2 && error[0].equals("HOST_IS_SLAVE")) - { - try - { - XmlRpcHttpClientConfig clientConfig = (XmlRpcHttpClientConfig)client.getClientConfig(); - URL client_url = clientConfig.getServerURL(); - URL masterUrl = new URL(client_url.getProtocol(), (String)error[1], client_url.getPort(), client_url.getFile()); - - Connection tmp_conn = new Connection(masterUrl, sessionReference, clientConfig.getReplyTimeout(), clientConfig.getConnectionTimeout()); - - Session.logout(tmp_conn); - } - catch (Exception ex) - { - // Ignore - } - } - } - - this.sessionReference = null; - } - return Types.checkResponse(response); + return result.result; } - - private void setAPIVersion(Session session) throws XenAPIException, XmlRpcException - { - try - { - long major = session.getThisHost(this).getAPIVersionMajor(this); - long minor = session.getThisHost(this).getAPIVersionMinor(this); - apiVersion = APIVersion.fromMajorMinor(major, minor); - } - catch (BadServerResponse exn) - { - apiVersion = APIVersion.UNKNOWN; - } + /** + * Send a method call to xapi's backend. To be used with methods without a return type + * + * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password + * @param methodParameters the methodParameters of the method call + * @throws XenAPIException if the call failed. + * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON + * @throws IOException if an I/O error occurs when sending or receiving + */ + public void dispatch(String methodCall, Object[] methodParameters) throws XenAPIException, JsonProcessingException, IOException { + var typeReference = new TypeReference() { + }; + this.dispatch(methodCall, methodParameters, typeReference); } } From 31062352aebd0a7e6480bfdde9aae7947a2749bf Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 18 Oct 2023 14:43:10 +0000 Subject: [PATCH 59/99] CP-45888: Update Java SDK generation to use `JsonRpcClient` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 195 +++++++++++++------------------------ 1 file changed, 70 insertions(+), 125 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 8efaafed4f3..b30873670bf 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -142,7 +142,7 @@ let rec get_java_type ty = Hashtbl.replace enums name ls ; sprintf "Types.%s" (class_case name) | Set t1 -> - sprintf "Set<%s>" (get_java_type t1) + sprintf "HashSet<%s>" (get_java_type t1) | Map (t1, t2) -> sprintf "Map<%s, %s>" (get_java_type t1) (get_java_type t2) | Ref x -> @@ -278,10 +278,13 @@ let gen_method file cls message params async_version = ( "BadServerResponse" , "Thrown if the response from the server contains an invalid status." ) - ; ("XenAPIException", "Thrown if the call failed.") - ; ( "XmlRpcException" - , "Thrown if the result of an asynchronous call could not be parsed." + ; ("XenAPIException", "if the call failed.") + ; ( "JsonProcessingException" + , "if the request's payload or the response's payload cannot be written \ + or read as valid JSON." ) + ; ("IOException", "if an I/O error occurs when sending or receiving.") + ; ("InterruptedException", " if the operation is interrupted.") ] in let publishInfo = get_published_info_message message cls in @@ -346,14 +349,14 @@ let gen_method file cls message params async_version = fprintf file " %s {\n" (String.concat ",\n " all_errors) ; if async_version then - fprintf file " String method_call = \"Async.%s.%s\";\n" + fprintf file " String methodCall = \"Async.%s.%s\";\n" message.msg_obj_name message.msg_name else - fprintf file " String method_call = \"%s.%s\";\n" - message.msg_obj_name message.msg_name ; + fprintf file " String methodCall = \"%s.%s\";\n" message.msg_obj_name + message.msg_name ; if message.msg_session then - fprintf file " String session = c.getSessionReference();\n" + fprintf file " String sessionReference = c.getSessionReference();\n" else () ; @@ -371,38 +374,32 @@ let gen_method file cls message params async_version = ) record_params ; - fprintf file " Object[] method_params = {" ; + fprintf file " Object[] methodParameters = {" ; let methodParamsList = if message.msg_session then - "session" :: get_method_params_for_xml message params + "sessionReference" :: get_method_params_for_xml message params else get_method_params_for_xml message params in output_string file - (String.concat ", " - (List.map - (fun s -> sprintf "Marshalling.toXMLRPC(%s)" s) - methodParamsList - ) - ) ; + (String.concat ", " (List.map (fun s -> sprintf "%s" s) methodParamsList)) ; fprintf file "};\n" ; - fprintf file - " Map response = c.dispatch(method_call, method_params);\n" ; - ( if async_version then ( - fprintf file " Object result = response.get(\"Value\");\n" ; - fprintf file " return Types.toTask(result);\n" - ) else - match message.msg_result with - | None -> - fprintf file "" - | Some _ -> - fprintf file " Object result = response.get(\"Value\");\n" ; - gen_method_return file cls message - ) ; + if message.msg_result != None || async_version then + fprintf file " var typeReference = new TypeReference<%s>(){};\n" + (if async_version then "Task" else return_type) ; + + let last_statement = + match message.msg_result with + | None when not async_version -> + " c.dispatch(methodCall, methodParameters);\n" + | _ -> + " return c.dispatch(methodCall, methodParameters, typeReference);\n" + in + fprintf file "%s" last_statement ; fprintf file " }\n\n" @@ -430,14 +427,14 @@ let gen_method_and_asynchronous_counterpart file cls message = let gen_record_field file prefix field cls = let ty = get_java_type field.ty in - let name = - camel_case (String.concat "_" (List.rev (field.field_name :: prefix))) - in + let full_name = String.concat "_" (List.rev (field.field_name :: prefix)) in + let name = camel_case full_name in let publishInfo = get_published_info_field field cls in fprintf file " /**\n" ; fprintf file " * %s\n" (escape_xml field.field_description) ; if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; fprintf file " */\n" ; + fprintf file " @JsonProperty(\"%s\")\n" full_name ; fprintf file " public %s %s;\n" ty name let rec gen_record_namespace file prefix (name, contents) cls = @@ -571,16 +568,15 @@ let gen_class cls folder = print_license file ; fprintf file "package com.xensource.xenapi;\n\n\ + import com.fasterxml.jackson.annotation.JsonProperty;\n\ + import com.fasterxml.jackson.core.JsonProcessingException;\n\ + import com.fasterxml.jackson.core.type.TypeReference;\n\ import com.xensource.xenapi.Types.BadServerResponse;\n\ import com.xensource.xenapi.Types.XenAPIException;\n\n\ import java.io.PrintWriter;\n\ import java.io.StringWriter;\n\ - import java.util.Date;\n\ - import java.util.HashMap;\n\ - import java.util.LinkedHashSet;\n\ - import java.util.Map;\n\ - import java.util.Set;\n\n\ - import org.apache.xmlrpc.XmlRpcException;\n\n" ; + import java.util.*;\n\ + import java.io.IOException;\n\n" ; fprintf file "/**\n" ; fprintf file " * %s\n" cls.description ; if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; @@ -753,7 +749,8 @@ let rec gen_marshall_body file = function let ty_name = get_java_type ty in let marshall_fn = get_marshall_function ty in fprintf file " Object[] items = (Object[]) object;\n" ; - fprintf file " Set<%s> result = new LinkedHashSet<>();\n" ty_name ; + fprintf file " HashSet<%s> result = new LinkedHashSet<>();\n" + ty_name ; fprintf file " for(Object item: items) {\n" ; fprintf file " %s typed = %s(item);\n" ty_name marshall_fn ; fprintf file " result.add(typed);\n" ; @@ -767,7 +764,7 @@ let rec gen_marshall_body file = function fprintf file " Map map = (Map)object;\n" ; fprintf file " Map<%s,%s> result = new HashMap<>();\n" ty_name ty_name' ; - fprintf file " Set entries = map.entrySet();\n" ; + fprintf file " HashSet entries = map.entrySet();\n" ; fprintf file " for(Map.Entry entry: entries) {\n" ; fprintf file " %s key = %s(entry.getKey());\n" ty_name marshall_fn ; @@ -821,13 +818,20 @@ let gen_enum file name ls = let final_description = global_replace (regexp_string "\n") "\n * " escaped_description in - " /**\n" - ^ " * " - ^ final_description - ^ "\n" - ^ " */\n" - ^ " " - ^ enum_of_wire name + let comment = + " /**\n" + ^ " * " + ^ final_description + ^ "\n" + ^ " */\n" + in + let json_property = + if name != "UNRECOGNIZED" then + "@JsonProperty(\"" ^ name ^ "\")" + else + "@JsonEnumDefaultValue" + in + comment ^ " " ^ json_property ^ "\n" ^ " " ^ enum_of_wire name in fprintf file "%s" (String.concat ",\n" (List.map to_member_declaration ls)) ; fprintf file ";\n" ; @@ -888,15 +892,15 @@ let gen_method_error_throw file name error = ) in - fprintf file " if (ErrorDescription[0].equals(\"%s\"))\n" name ; + fprintf file " if (errorName.equals(\"%s\"))\n" name ; fprintf file " {\n" ; (* Prepare the parameters to the Exception constructor *) List.iter (fun i -> fprintf file - " String p%i = ErrorDescription.length > %i ? \ - ErrorDescription[%i] : \"\";\n" + " String p%i = errorData.length > %i ? \ + errorData[%i] : \"\";\n" i i i ) (range (List.length error.err_params)) ; @@ -910,19 +914,12 @@ let gen_types_class folder = print_license file ; fprintf file "package com.xensource.xenapi;\n\n\ - import java.util.Date;\n\ - import java.util.Map;\n\ - import java.util.HashMap;\n\ - import java.util.Set;\n\ - import java.util.LinkedHashSet;\n\ - import java.io.IOException;\n\n\ - import java.util.regex.Pattern;\n\ - import java.util.regex.Matcher;\n\n\ - import org.apache.xmlrpc.XmlRpcException;\n\n\ + import java.util.Map;\n\n\ + import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;\n\ + import com.fasterxml.jackson.annotation.JsonProperty;\n\ + import java.io.IOException;\n\n\ /**\n\ - \ * This class holds vital marshalling functions, enum types and exceptions.\n\ - \ *\n\ - \ * @author Cloud Software Group, Inc.\n\ + \ * This class holds enum types and exceptions.\n\ \ */\n\ public class Types\n\ {\n\ @@ -937,18 +934,6 @@ let gen_types_class folder = \ Map toMap();\n\ \ }\n\n\ \ /**\n\ - \ * Helper method.\n\ - \ */\n\ - \ private static String[] ObjectArrayToStringArray(Object[] objArray)\n\ - \ {\n\ - \ String[] result = new String[objArray.length];\n\ - \ for (int i = 0; i < objArray.length; i++)\n\ - \ {\n\ - \ result[i] = (String) objArray[i];\n\ - \ }\n\ - \ return result;\n\ - \ }\n\n\ - \ /**\n\ \ * Base class for all XenAPI Exceptions\n\ \ */\n\ \ public static class XenAPIException extends IOException {\n\ @@ -993,30 +978,11 @@ let gen_types_class folder = \ */\n\ \ public static class BadServerResponse extends XenAPIException\n\ \ {\n\ - \ public BadServerResponse(Map response)\n\ - \ {\n\ - \ super(ObjectArrayToStringArray((Object[]) \ - response.get(\"ErrorDescription\")));\n\ - \ }\n\ - \ }\n\n\ - \ public static class BadAsyncResult extends XenAPIException\n\ - \ {\n\ - \ public final String result;\n\n\ - \ public BadAsyncResult(String result)\n\ + \ public BadServerResponse(JsonRpcResponseError responseError)\n\ \ {\n\ - \ super(result);\n\ - \ this.result = result;\n\ + \ super(String.valueOf(responseError));\n\ \ }\n\ \ }\n\n\ - \ private static String parseResult(String result) throws BadAsyncResult\n\ - \ {\n\ - \ Pattern pattern = Pattern.compile(\"(.*)\");\n\ - \ Matcher matcher = pattern.matcher(result);\n\ - \ if (!matcher.find() || matcher.groupCount() != 1) {\n\ - \ throw new Types.BadAsyncResult(\"Can't interpret: \" + result);\n\ - \ }\n\n\ - \ return matcher.group(1);\n\ - \ }\n\ \ " ; fprintf file @@ -1026,48 +992,27 @@ let gen_types_class folder = \ * returned an invalid response, throws a BadServerResponse. \ Otherwise, returns the server response as passed in.\n\ \ */\n\ - \ static Map checkResponse(Map response) throws XenAPIException, \ + \ public static void checkError(JsonRpcResponseError response) throws XenAPIException, \ BadServerResponse\n\ \ {\n\ - \ if (response.get(\"Status\").equals(\"Success\"))\n\ - \ {\n\ - \ return response;\n\ - \ }\n\n\ - \ if (response.get(\"Status\").equals(\"Failure\"))\n\ - \ {\n\ - \ String[] ErrorDescription = \ - ObjectArrayToStringArray((Object[]) response.get(\"ErrorDescription\"));\n\n" ; + \ var errorData = response.data; + \ if(errorData.length == 0){ + \ throw new BadServerResponse(response); + \ } + \ var errorName = errorData[0];\n\n" ; Hashtbl.iter (gen_method_error_throw file) Datamodel.errors ; fprintf file "\n\ - \ // An unknown error occurred\n\ - \ throw new Types.XenAPIException(ErrorDescription);\n\ - \ }\n\n\ - \ throw new BadServerResponse(response);\n\ - \ }\n\n" ; + \ // An unknown error occurred\n\ + \ throw new Types.XenAPIException(errorData);\n\ + \ }\n\n" ; gen_enums file ; fprintf file "\n" ; Hashtbl.iter (gen_error file) Datamodel.errors ; fprintf file "\n" ; - TypeSet.iter (gen_marshall_func file) !types ; - fprintf file "\n" ; - TypeSet.iter (gen_task_result_func file) !types ; - fprintf file - "\n\ - \ public static EventBatch toEventBatch(Object object) {\n\ - \ if (object == null) {\n\ - \ return null;\n\ - \ }\n\n\ - \ Map map = (Map) object;\n\ - \ EventBatch batch = new EventBatch();\n\ - \ batch.token = toString(map.get(\"token\"));\n\ - \ batch.validRefCounts = map.get(\"valid_ref_counts\");\n\ - \ batch.events = toSetOfEventRecord(map.get(\"events\"));\n\ - \ return batch;\n\ - \ }" ; fprintf file "}\n" (* Now run it *) From 93b9839c755d6b8db454c3550067931062c2106d Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 1 Nov 2023 15:23:38 +0000 Subject: [PATCH 60/99] CP-45888: Remove unsed functions from Java's `main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 208 ------------------------------------- 1 file changed, 208 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index b30873670bf..6a77a157b97 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -159,36 +159,6 @@ let switch_enum = let _ = get_java_type switch_enum -(*Helper function for get_marshall_function*) -let rec get_marshall_function_rec = function - | SecretString | String -> - "String" - | Int -> - "Long" - | Float -> - "Double" - | Bool -> - "Boolean" - | DateTime -> - "Date" - | Enum (name, _) -> - class_case name - | Set t1 -> - sprintf "SetOf%s" (get_marshall_function_rec t1) - | Map (t1, t2) -> - sprintf "MapOf%s%s" - (get_marshall_function_rec t1) - (get_marshall_function_rec t2) - | Ref ty -> - class_case ty (* We want to hide all refs *) - | Record ty -> - sprintf "%sRecord" (class_case ty) - | Option ty -> - get_marshall_function_rec ty - -(*get_marshall_function (Set(Map(Float,Bool)));; -> "toSetOfMapOfDoubleBoolean"*) -let get_marshall_function ty = "to" ^ get_marshall_function_rec ty - (* Generate the methods *) let get_java_type_or_void = function @@ -240,22 +210,6 @@ let get_method_params_for_xml message params = else "this.ref" :: List.map f params -let gen_method_return_cast message = - match message.msg_result with - | None -> - sprintf "" - | Some (ty, _) -> - sprintf " Types.%s(result)" (get_marshall_function ty) - -let gen_method_return file cls message = - if - String.lowercase_ascii cls.name = "event" - && String.lowercase_ascii message.msg_name = "from" - then - fprintf file " return Types.toEventBatch(result);\n" - else - fprintf file " return%s;\n" (gen_method_return_cast message) - let rec range = function 0 -> [] | i -> range (i - 1) @ [i] (* Here is the main method generating function.*) @@ -284,7 +238,6 @@ let gen_method file cls message params async_version = or read as valid JSON." ) ; ("IOException", "if an I/O error occurs when sending or receiving.") - ; ("InterruptedException", " if the operation is interrupted.") ] in let publishInfo = get_published_info_message message cls in @@ -644,167 +597,6 @@ let gen_class cls folder = fprintf file "}" ; close_out file -(* Generate Marshalling Class *) - -(*This generates the special case code for marshalling the snapshot field in an Event.Record*) -let generate_snapshot_hack file = - fprintf file "\n" ; - fprintf file "\n" ; - fprintf file " Object a,b;\n" ; - fprintf file " a=map.get(\"snapshot\");\n" ; - fprintf file " switch(%s(record.clazz))\n" - (get_marshall_function switch_enum) ; - fprintf file " {\n" ; - List.iter - (fun x -> - fprintf file " case %17s: b = %25s(a); break;\n" - (String.uppercase_ascii x) - (get_marshall_function (Record x)) - ) - (List.map - (fun x -> x.name) - (List.filter (fun x -> not (class_is_empty x)) classes) - ) ; - fprintf file - " default: throw new RuntimeException(\"Internal error in \ - auto-generated code whilst unmarshalling event snapshot\");\n" ; - fprintf file " }\n" ; - fprintf file " record.snapshot = b;\n" - -let gen_marshall_record_field file prefix field = - let ty = get_marshall_function field.ty in - let name = String.concat "_" (List.rev (field.field_name :: prefix)) in - let name' = camel_case name in - fprintf file " record.%s = %s(map.get(\"%s\"));\n" name' ty name - -let rec gen_marshall_record_namespace file prefix (name, contents) = - List.iter (gen_marshall_record_contents file (name :: prefix)) contents - -and gen_marshall_record_contents file prefix = function - | Field f -> - gen_marshall_record_field file prefix f - | Namespace (n, cs) -> - gen_marshall_record_namespace file prefix (n, cs) ; - () - -(*Every type which may be returned by a function may also be the result of the*) -(* corresponding asynchronous task. We therefore need to generate corresponding*) -(* marshalling functions which can take the raw xml of the tasks result field*) -(* and turn it into the corresponding type. Luckily, the only things returned by*) -(* asynchronous tasks are object references and strings, so rather than implementing*) -(* the general recursive structure we'll just make one for each of the classes*) -(* that's been registered as a marshall-needing type*) - -let generate_reference_task_result_func file clstr = - fprintf file - " public static %s to%s(Task task, Connection connection) throws \ - XenAPIException, BadServerResponse, XmlRpcException, BadAsyncResult{\n" - clstr clstr ; - fprintf file - " return Types.to%s(parseResult(task.getResult(connection)));\n" - clstr ; - fprintf file " }\n" ; - fprintf file "\n" - -let gen_task_result_func file = function - | Ref ty -> - generate_reference_task_result_func file (class_case ty) - | _ -> - () - -(*don't generate for complicated types. They're not needed.*) - -let rec gen_marshall_body file = function - | SecretString | String -> - fprintf file " return (String) object;\n" - | Int -> - fprintf file " return Long.valueOf((String) object);\n" - | Float -> - fprintf file " return (Double) object;\n" - | Bool -> - fprintf file " return (Boolean) object;\n" - | DateTime -> - fprintf file - " try {\n\ - \ return (Date) object;\n\ - \ } catch (ClassCastException e){\n\ - \ //Occasionally the date comes back as an ocaml float \ - rather than\n\ - \ //in the xmlrpc format! Catch this and convert.\n\ - \ return (new Date((long) (1000*Double.parseDouble((String) \ - object))));\n\ - \ }\n" - | Ref ty -> - fprintf file " return new %s((String) object);\n" (class_case ty) - | Enum (name, _) -> - fprintf file " try {\n" ; - fprintf file - " return %s.valueOf(((String) \ - object).toUpperCase().replace('-','_'));\n" - (class_case name) ; - fprintf file " } catch (IllegalArgumentException ex) {\n" ; - fprintf file " return %s.UNRECOGNIZED;\n" (class_case name) ; - fprintf file " }\n" - | Set ty -> - let ty_name = get_java_type ty in - let marshall_fn = get_marshall_function ty in - fprintf file " Object[] items = (Object[]) object;\n" ; - fprintf file " HashSet<%s> result = new LinkedHashSet<>();\n" - ty_name ; - fprintf file " for(Object item: items) {\n" ; - fprintf file " %s typed = %s(item);\n" ty_name marshall_fn ; - fprintf file " result.add(typed);\n" ; - fprintf file " }\n" ; - fprintf file " return result;\n" - | Map (ty, ty') -> - let ty_name = get_java_type ty in - let ty_name' = get_java_type ty' in - let marshall_fn = get_marshall_function ty in - let marshall_fn' = get_marshall_function ty' in - fprintf file " Map map = (Map)object;\n" ; - fprintf file " Map<%s,%s> result = new HashMap<>();\n" ty_name - ty_name' ; - fprintf file " HashSet entries = map.entrySet();\n" ; - fprintf file " for(Map.Entry entry: entries) {\n" ; - fprintf file " %s key = %s(entry.getKey());\n" ty_name - marshall_fn ; - fprintf file " %s value = %s(entry.getValue());\n" ty_name' - marshall_fn' ; - fprintf file " result.put(key, value);\n" ; - fprintf file " }\n" ; - fprintf file " return result;\n" - | Record ty -> - let contents = Hashtbl.find records ty in - let cls_name = class_case ty in - fprintf file - " Map map = (Map) object;\n" ; - fprintf file " %s.Record record = new %s.Record();\n" cls_name - cls_name ; - List.iter (gen_marshall_record_contents file []) contents ; - (*Event.Record needs a special case to handle snapshots*) - if ty = "event" then generate_snapshot_hack file ; - fprintf file " return record;\n" - | Option ty -> - gen_marshall_body file ty - -let rec gen_marshall_func file ty = - match ty with - | Option x -> - if TypeSet.mem x !types then - () - else - gen_marshall_func file ty - | _ -> - let type_string = get_java_type ty in - let fn_name = get_marshall_function ty in - fprintf file " public static %s %s(Object object) {\n" type_string - fn_name ; - fprintf file " if (object == null) {\n" ; - fprintf file " return null;\n" ; - fprintf file " }\n" ; - gen_marshall_body file ty ; - fprintf file " }\n\n" - let gen_enum file name ls = let name = class_case name in let ls = From 9bed681e684cde4af9d4be3ab7c779e312b46639 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 6 Nov 2023 11:29:56 +0000 Subject: [PATCH 61/99] CP-45888: Update dependencies list Java SDK README Signed-off-by: Danilo Del Busso --- .../java/autogen/xen-api/src/main/resources/README.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/resources/README.txt b/ocaml/sdk-gen/java/autogen/xen-api/src/main/resources/README.txt index ea1bf8c490e..632b8af5728 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/resources/README.txt +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/resources/README.txt @@ -39,5 +39,6 @@ https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver/ Dependencies ------------ -XenServerJava is dependent upon Apache XML-RPC by the Apache Software Foundation, -licensed under the Apache Software License 2.0. +XenServerJava is dependent upon: +- The jackson-databind (https://github.com/FasterXML/jackson-databind) package by the Jackson Project (https://github.com/FasterXML/jackson), licensed under the Apache Software License 2.0. +- The Apache HttpClient (https://hc.apache.org/httpcomponents-client/) package by the Apache Software Foundation (https://www.apache.org/), licensed under the Apache Software License 2.0. \ No newline at end of file From 166fa826426b279aad60ca58d10f0d8683cf96c9 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 16 Nov 2023 10:14:31 +0000 Subject: [PATCH 62/99] CP-45888: Fix formatting in Java's `main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 6a77a157b97..b0f3538978c 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -691,8 +691,8 @@ let gen_method_error_throw file name error = List.iter (fun i -> fprintf file - " String p%i = errorData.length > %i ? \ - errorData[%i] : \"\";\n" + " String p%i = errorData.length > %i ? errorData[%i] : \ + \"\";\n" i i i ) (range (List.length error.err_params)) ; @@ -706,10 +706,10 @@ let gen_types_class folder = print_license file ; fprintf file "package com.xensource.xenapi;\n\n\ - import java.util.Map;\n\n\ - import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;\n\ - import com.fasterxml.jackson.annotation.JsonProperty;\n\ - import java.io.IOException;\n\n\ + import java.util.Map;\n\n\ + import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;\n\ + import com.fasterxml.jackson.annotation.JsonProperty;\n\ + import java.io.IOException;\n\n\ /**\n\ \ * This class holds enum types and exceptions.\n\ \ */\n\ @@ -784,14 +784,14 @@ let gen_types_class folder = \ * returned an invalid response, throws a BadServerResponse. \ Otherwise, returns the server response as passed in.\n\ \ */\n\ - \ public static void checkError(JsonRpcResponseError response) throws XenAPIException, \ - BadServerResponse\n\ + \ public static void checkError(JsonRpcResponseError response) throws \ + XenAPIException, BadServerResponse\n\ \ {\n\ - \ var errorData = response.data; - \ if(errorData.length == 0){ - \ throw new BadServerResponse(response); - \ } - \ var errorName = errorData[0];\n\n" ; + \ var errorData = response.data;\n\ + \ if(errorData.length == 0){\n\ + \ throw new BadServerResponse(response);\n\ + \ }\n\ + \ var errorName = errorData[0];\n\n" ; Hashtbl.iter (gen_method_error_throw file) Datamodel.errors ; From c33b98166d63efc7f483a5352bef75e706abfe33 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 16 Nov 2023 15:19:35 +0000 Subject: [PATCH 63/99] Add missing `mli` files under `sdk-gen/` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/c/gen_c_binding.mli | 1 + ocaml/sdk-gen/c/helper.mli | 13 ++ ocaml/sdk-gen/common/CommonFunctions.mli | 130 ++++++++++++++++++ ocaml/sdk-gen/common/dune | 1 + ocaml/sdk-gen/common/license.mli | 2 + ocaml/sdk-gen/csharp/friendly_error_names.mli | 1 + ocaml/sdk-gen/csharp/gen_csharp_binding.mli | 1 + ocaml/sdk-gen/java/main.mli | 1 + ocaml/sdk-gen/powershell/common_functions.ml | 1 - ocaml/sdk-gen/powershell/common_functions.mli | 98 +++++++++++++ .../powershell/gen_powershell_binding.ml | 2 - .../powershell/gen_powershell_binding.mli | 1 + 12 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 ocaml/sdk-gen/c/gen_c_binding.mli create mode 100644 ocaml/sdk-gen/c/helper.mli create mode 100644 ocaml/sdk-gen/common/CommonFunctions.mli create mode 100644 ocaml/sdk-gen/common/license.mli create mode 100644 ocaml/sdk-gen/csharp/friendly_error_names.mli create mode 100644 ocaml/sdk-gen/csharp/gen_csharp_binding.mli create mode 100644 ocaml/sdk-gen/java/main.mli create mode 100644 ocaml/sdk-gen/powershell/common_functions.mli create mode 100644 ocaml/sdk-gen/powershell/gen_powershell_binding.mli diff --git a/ocaml/sdk-gen/c/gen_c_binding.mli b/ocaml/sdk-gen/c/gen_c_binding.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/c/gen_c_binding.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) diff --git a/ocaml/sdk-gen/c/helper.mli b/ocaml/sdk-gen/c/helper.mli new file mode 100644 index 00000000000..1d8b90de096 --- /dev/null +++ b/ocaml/sdk-gen/c/helper.mli @@ -0,0 +1,13 @@ +val formatted_wrap : Format.formatter -> string -> unit +(** Recursively formats the input string + based on spaces and newlines, ensuring proper indentation. + + @param formatter The formatter to output the formatted string. + @param s The input string to be formatted. *) + +val comment : bool -> ?indent:int -> string -> string +(** Formats a comment block with optional indentation. + @param indent Optional indentation level. + @param doc Include an extra '*' for documentation. + @param s String content of the comment. + @return Formatted comment block. *) diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli new file mode 100644 index 00000000000..18a52821189 --- /dev/null +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -0,0 +1,130 @@ +(** Exception for unknown wire protocol. *) +exception Unknown_wire_protocol + +(** Type representing supported protocols. *) +type wireProtocol = XmlRpc | JsonRpc + +val get_release_branding : string -> string +(** [get_release_branding codename] Gets the branding for a release codename. + @param codename Release codename to lookup. + @return Branding for the release codename, or the original codename if not found. *) + +val is_setter : Datamodel_types.message -> bool +(** [is_setter message] Checks if a message is a setter based on its name. + @param message Message to check. + @return [true] if the message is a setter, [false] otherwise. *) + +val finally : (unit -> 'a) -> always:(unit -> unit) -> 'a +(** [finally f ~always] Executes [f] and then [always] regardless of success or failure. + @param f Function to execute. + @param always Function to execute always. *) + +val is_getter : Datamodel_types.message -> bool +(** [is_getter message] Checks if a message is a getter based on its name. + @param message Message to check. + @return [true] if the message is a getter, [false] otherwise. *) + +val get_deprecated_info_message : Datamodel_types.message -> string +(** [get_deprecated_info_message message] Returns a deprecated information message + based on the internal_deprecated_since version in the input message. + @param message Message containing version information. + @return Deprecated information message or an empty string if not deprecated. *) + +val is_adder : Datamodel_types.message -> bool +(** [is_adder message] Checks if a message is an adder based on its name. + @param message Message to check. + @return [true] if the message is an adder, [false] otherwise. *) + +val get_published_info_class : Datamodel_types.obj -> string +(** [get_published_info_class cls] Returns information about the first publication + of a class based on its lifecycle transitions. + @param cls Class to retrieve publication information for. + @return Information string with the first published version. *) + +val is_method_static : Datamodel_types.message -> bool +(** [is_method_static message] Checks if a method is static based on its parameters. + @param message Message to check. + @return [true] if the method is static, [false] otherwise. *) + +val escape_xml : string -> string +(** [escape_xml s] Escapes XML special characters in a string. + @param s String to escape. + @return String with XML special characters escaped. *) + +val get_published_info_message : + Datamodel_types.message -> Datamodel_types.obj -> string +(** Gets information about the publication status of a message. + @param message - Message to check. + @param cls - Class containing the message. + @return Information about the publication status. *) + +val is_remover : Datamodel_types.message -> bool +(** [is_remover message] Checks if a message is a remover based on its name. + @param message Message to check. + @return [true] if the message is a remover, [false] otherwise. *) + +val gen_param_groups : + Datamodel_types.message + -> Datamodel_types.param list + -> Datamodel_types.param list list +(** Generates parameter groups based on a message and its parameters. + @param message - Message containing the parameters. + @param params - List of parameters. + @return List of parameter groups. *) + +val get_published_info_param : + Datamodel_types.message -> Datamodel_types.param -> string +(** Gets information about the publication status of a parameter. + @param message - Message containing the parameter. + @param param - Parameter to check. + @return Information about the publication status. *) + +val get_published_info_field : + Datamodel_types.field -> Datamodel_types.obj -> string +(** Gets information about the publication status of a field within a class. + @param field - Field to check. + @param cls - Class containing the field. + @return Information about the publication status. *) + +val string_of_file : string -> string +(** [string_of_file filename] Reads the content of a file into a string. + @param filename Name of the file to read. + @return String containing the file content. *) + +val is_constructor : Datamodel_types.message -> bool +(** [is_constructor message] Checks if a message is a constructor. + @param message Message to check. + @return [true] if the message is a constructor, [false] otherwise. *) + +val with_output : string -> (out_channel -> 'a) -> 'a +(** [with_output filename f] Opens a file for writing and executes a function with the output channel. + @param filename Name of the file to open. + @param f Function to execute with the output channel. *) + +val is_destructor : Datamodel_types.message -> bool +(** [is_destructor message] Checks if a message is a destructor. + @param message Message to check. + @return [true] if the message is a destructor, [false] otherwise. *) + +val is_real_constructor : Datamodel_types.message -> bool +(** [is_real_constructor message] Checks if a message is a real constructor. + @param message Message to check. + @return [true] if the message is a real constructor, [false] otherwise. *) + +val render_file : string * string -> Mustache.Json.t -> string -> string -> unit +(** [render_file (infile, outfile) json templates_dir dest_dir] Renders a file using a JSON data model and templates. + @param infile Input file name. + @param outfile Output file name. + @param json JSON data model. + @param templates_dir Directory containing templates. + @param dest_dir Directory for the rendered output. *) + +val json_releases : + [> `O of + ( string + * [> `A of + [> `O of (string * [> `Float of float | `String of string]) list] list + | `Float of float ] + ) + list ] +(** JSON structure representing release information. *) diff --git a/ocaml/sdk-gen/common/dune b/ocaml/sdk-gen/common/dune index 2278becc3e0..7cda0194598 100644 --- a/ocaml/sdk-gen/common/dune +++ b/ocaml/sdk-gen/common/dune @@ -6,5 +6,6 @@ xapi-datamodel mustache ) + (modules_without_implementation license) ) diff --git a/ocaml/sdk-gen/common/license.mli b/ocaml/sdk-gen/common/license.mli new file mode 100644 index 00000000000..6ab5ae4cc08 --- /dev/null +++ b/ocaml/sdk-gen/common/license.mli @@ -0,0 +1,2 @@ +(* Conent of BSD 2 License *) +val bsd_two_clause : string diff --git a/ocaml/sdk-gen/csharp/friendly_error_names.mli b/ocaml/sdk-gen/csharp/friendly_error_names.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/csharp/friendly_error_names.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.mli b/ocaml/sdk-gen/csharp/gen_csharp_binding.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) diff --git a/ocaml/sdk-gen/java/main.mli b/ocaml/sdk-gen/java/main.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/java/main.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index d81f9f687db..d3b69b7d24a 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -197,7 +197,6 @@ and cut_msg_name message_name fn_type = else message_name -(* True if an object has a uuid (and therefore should have a get_by_uuid message *) and has_uuid x = let all_fields = DU.fields_of_obj x in List.filter (fun fld -> fld.full_name = ["uuid"]) all_fields <> [] diff --git a/ocaml/sdk-gen/powershell/common_functions.mli b/ocaml/sdk-gen/powershell/common_functions.mli new file mode 100644 index 00000000000..706264469f4 --- /dev/null +++ b/ocaml/sdk-gen/powershell/common_functions.mli @@ -0,0 +1,98 @@ +val get_http_action_stem : string -> string +(** Gets the HTTP action stem based on the name. + @param name - Name to analyze. + @return HTTP action stem. *) + +val get_http_action_verb : string -> Datamodel.http_meth -> string +(** Gets the HTTP action verb based on the name and method. + @param name - Name to analyze. + @param meth - HTTP method. + @return HTTP action verb. *) + +val get_common_verb_category : string -> string +(** Gets the common verb category based on the HTTP action verb. + @param verb - HTTP action verb. + @return Common verb category. *) + +val pascal_case_ : string -> string +(** Recursively converts a string to PascalCase. + @param s - String to convert. + @return PascalCase formatted string. *) + +val qualified_class_name : string -> string +(** Gets the qualified class name by prepending "XenAPI." to the exposed class name. + @param classname - Class name. + @return Qualified class name. *) + +val ocaml_class_to_csharp_local_var : string -> string +(** Converts an OCaml class name to a C# local variable. + @param classname - OCaml class name. + @return C# local variable name. *) + +val ocaml_class_to_csharp_property : string -> string +(** Converts an OCaml class name to a C# property name. + @param classname - OCaml class name. + @return C# property name. *) + +val ocaml_class_to_csharp_class : string -> string +(** Converts an OCaml class name to a C# class name. + @param classname - OCaml class name. + @return C# class name. *) + +val is_invoke : Datamodel_types.message -> bool +(** Checks if a message is an invoke operation. + @param message - Message to check. + @return true if it's an invoke operation, false otherwise. *) + +val has_uuid : Datamodel_types.obj -> bool +(** Checks if an object has a UUID field, and therefore should have a get_by_uuid message. + @param x - Object to check. + @return true if the object has a UUID field, false otherwise. *) + +val has_name : Datamodel_types.obj -> bool +(** Checks if an object has a name field. + @param x - Object to check. + @return true if the object has a name field, false otherwise. *) + +val full_name : Datamodel_types.field -> string +(** Gets the full name of a field as a single string with underscores escaped. + @param field - Field to extract the full name from. + @return Full name with underscores escaped. *) + +val obj_internal_type : Datamodel_types.ty -> string +(** Gets the internal type representation of an object. + @param x - Object to determine the internal type for. + @return Internal type representation as a string. *) + +val ocaml_field_to_csharp_property : Datamodel_types.field -> string +(** Converts an OCaml field to its corresponding C# property name. + @param field - OCaml field. + @return C# property name. *) + +val exposed_type : Datamodel_types.ty -> string +(** Converts an exposed type from OCaml to its corresponding C# type. + @param ty - OCaml type. + @return Corresponding C# type. *) + +val pascal_case : string -> string +(** Converts a string to PascalCase. + @param s - Input string. + @return PascalCase formatted string. *) + +val exposed_class_name : string -> string +(** Converts an OCaml class name to a corresponding exposed class name in C#. + @param classname - OCaml class name. + @return Exposed class name in C#. *) + +val cut_msg_name : string -> string -> string +(** Extracts the base name from an OCaml message name by removing the specified prefix. + Some adders/removers are just prefixed by Add or RemoveFrom + and some are prefixed by AddTo or RemoveFrom + @param message_name - OCaml message name. + @param fn_type - Prefix to remove ("Add" or "Remove"). + @return Base name after removing the specified prefix. *) + +val lower_and_underscore_first : string -> string +(** Converts a string to lowercase and adds an underscore at the beginning. + @param s - Input string. + @return String in lowercase with an underscore at the beginning. *) diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 2e1fb55e5fb..406a8ddbfc1 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -19,8 +19,6 @@ end) let destdir = "autogen/src" -let templdir = "templates" - type cmdlet = {filename: string; content: string} let api = diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.mli b/ocaml/sdk-gen/powershell/gen_powershell_binding.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) From 909aeaaee561bb9380a8fa1559b9f4d900ec0999 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 16 Nov 2023 15:36:47 +0000 Subject: [PATCH 64/99] Remove all unused SDK generation functions As reported by `make check` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/c/gen_c_binding.ml | 5 --- ocaml/sdk-gen/common/CommonFunctions.ml | 3 -- ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 38 ------------------ ocaml/sdk-gen/powershell/common_functions.ml | 32 --------------- .../powershell/gen_powershell_binding.ml | 40 ------------------- 5 files changed, 118 deletions(-) diff --git a/ocaml/sdk-gen/c/gen_c_binding.ml b/ocaml/sdk-gen/c/gen_c_binding.ml index 0c84af4ac93..7121ea70f34 100644 --- a/ocaml/sdk-gen/c/gen_c_binding.ml +++ b/ocaml/sdk-gen/c/gen_c_binding.ml @@ -1412,15 +1412,10 @@ and internal_decl_filename name = and impl_filename name = sprintf "xen_%s.c" (String.lowercase_ascii name) -and internal_impl_filename name = - sprintf "xen_%s_internal.c" (String.lowercase_ascii name) - and protector classname = sprintf "XEN_%s_H" (String.uppercase_ascii classname) and typename classname = sprintf "xen_%s" (String.lowercase_ascii classname) -and variablename classname = sprintf "%s" (String.lowercase_ascii classname) - and record_typename classname = sprintf "%s_record" (typename classname) and record_opt_typename classname = sprintf "%s_record_opt" (typename classname) diff --git a/ocaml/sdk-gen/common/CommonFunctions.ml b/ocaml/sdk-gen/common/CommonFunctions.ml index 2d046933bf1..cd262792b57 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.ml +++ b/ocaml/sdk-gen/common/CommonFunctions.ml @@ -40,9 +40,6 @@ let with_output filename f = let io = open_out filename in finally (fun () -> f io) ~always:(fun () -> close_out io) -let joined sep f l = - l |> List.map f |> List.filter (fun x -> x <> "") |> String.concat sep - let escape_xml s = s |> Astring.String.cuts ~sep:"<" ~empty:true diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index 2d7f254fad9..db12da58b29 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -137,10 +137,6 @@ and gen_relations () = Hashtbl.iter (gen_relations_by_type out_chan) relations ; print "\n return relations;\n }\n }\n}\n" -and string_ends str en = - let len = String.length en in - String.sub str (String.length str - len) len = en - and process_relations ((oneClass, oneField), (manyClass, manyField)) = let value = try (manyField, oneClass, oneField) :: Hashtbl.find relations manyClass @@ -451,16 +447,6 @@ and get_constructor_body' content elements = | Namespace (_, c) :: others -> get_constructor_body' (c @ others) elements -and gen_constructor_line out_chan content = - let print format = fprintf out_chan format in - - match content with - | Field fr -> - print " %s = %s;\n" (full_name fr) - (convert_from_proxy ("proxy." ^ full_name fr) fr.ty) - | Namespace (_, c) -> - List.iter (gen_constructor_line out_chan) c - and gen_hashtable_constructor_line out_chan content = let print format = fprintf out_chan format in @@ -560,9 +546,6 @@ and gen_exposed_method cls msg curParams = in sync ^ async -and returns_xenobject msg = - match msg.msg_result with Some (Record _, _) -> true | _ -> false - and get_params_doc msg classname params = let sessionDoc = "\n /// The session" @@ -688,18 +671,6 @@ and gen_save_changes_to_field out_chan exposed_class_name fr = \ }\n" equality exposed_class_name full_name_fr full_name_fr -and ctor_call classname = - let fields = - Datamodel_utils.fields_of_obj (Dm_api.get_obj_by_name api ~objname:classname) - in - let fields2 = - List.filter - (function {DT.qualifier= DT.StaticRO | DT.RW; _} -> true | _ -> false) - fields - in - let args = List.map (fun fr -> "p." ^ full_name fr) fields2 in - String.concat ", " ("session.opaque_ref" :: args) - and gen_exposed_field out_chan cls content = match content with | Field fr -> @@ -868,15 +839,6 @@ and gen_enum' name contents = ("enum", `String name); ("enum_members", `A (List.map enum_member members)) ] -and has_unknown_entry contents = - let rec f = function - | x :: xs -> - if String.lowercase_ascii (fst x) = "unknown" then true else f xs - | [] -> - false - in - f contents - (* ------------------- category: maps *) and gen_maps () = let out_chan = open_out (Filename.concat destdir "Maps.cs") in diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index d3b69b7d24a..a2832c14494 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -59,9 +59,6 @@ and ocaml_class_to_csharp_local_var classname = else String.lowercase_ascii (exposed_class_name classname) -and ocaml_field_to_csharp_local_var field = - String.lowercase_ascii (full_name field) - and ocaml_field_to_csharp_property field = ocaml_class_to_csharp_property (full_name field) @@ -86,39 +83,10 @@ and exposed_class_name classname = and qualified_class_name classname = "XenAPI." ^ exposed_class_name classname -and type_default ty = - match ty with - | Int -> - "" - | SecretString | String -> - "" - | Float -> - "" - | Bool -> - "" - | Enum _ -> - "" - | Record _ -> - "" - | Ref _ -> - "" - | Map (_, _) -> - " = new Hashtable()" - | Set String -> - " = new string[0]" - | _ -> - sprintf " = new %s()" (exposed_type ty) - and escaped = function "params" -> "paramz" | s -> s and full_name field = escaped (String.concat "_" field.full_name) -and exposed_type_opt = function - | Some (typ, _) -> - exposed_type typ - | None -> - "void" - and exposed_type = function | SecretString | String -> "string" diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 406a8ddbfc1..96d0782bf67 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -552,15 +552,6 @@ and print_methods_constructor message obj classname = (gen_shouldprocess "New" message classname) (gen_csharp_api_call message classname "New" "passthru") -and create_param_parse param paramName = - match param.param_type with - | Ref _ -> - sprintf "\n string %s = %s.opaque_ref;\n" - (String.lowercase_ascii param.param_name) - paramName - | _ -> - "" - and gen_make_record obj classname = sprintf "\n\ @@ -1145,37 +1136,6 @@ and print_cmdlet_methods_dynamic classname messages enum commonVerb = enum (cut_message_name hd) (cut_message_name hd) localVar (print_cmdlet_methods_dynamic classname tl enum commonVerb) -and print_async_param_getter classname asyncMessages = - let properties = - List.map - (fun x -> - sprintf " case Xen%sProperty.%s:" - (ocaml_class_to_csharp_class classname) - x - ) - asyncMessages - in - match asyncMessages with - | [] -> - "" - | _ -> - sprintf - "\n\ - \ protected override bool GenerateAsyncParam\n\ - \ {\n\ - \ get\n\ - \ {\n\ - \ switch (XenProperty)\n\ - \ {\n\ - %s\n\ - \ return true;\n\ - \ default:\n\ - \ return false;\n\ - \ }\n\ - \ }\n\ - \ }\n" - (String.concat "\n" properties) - (**************************************) (* Common to more than one generators *) (**************************************) From 64525404d26dc74d69b72f9f8910dd759d7d9f31 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 16 Nov 2023 15:40:13 +0000 Subject: [PATCH 65/99] Update number of `mli` files in quality gate script Signed-off-by: Danilo Del Busso --- quality-gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quality-gate.sh b/quality-gate.sh index 224e852aa32..77238f4ab93 100755 --- a/quality-gate.sh +++ b/quality-gate.sh @@ -25,7 +25,7 @@ verify-cert () { } mli-files () { - N=530 + N=522 # do not count ml files from the tests in ocaml/{tests/perftest/quicktest} MLIS=$(git ls-files -- '**/*.mli' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) MLS=$(git ls-files -- '**/*.ml' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) From 16e84547dc16bf3a8a360dc72e2f7d4547b700b3 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 20 Nov 2023 09:29:50 +0000 Subject: [PATCH 66/99] Use `Mustache.Json.t` as type for `json_releases` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/common/CommonFunctions.mli | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli index 18a52821189..fd2ba766568 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.mli +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -119,12 +119,5 @@ val render_file : string * string -> Mustache.Json.t -> string -> string -> unit @param templates_dir Directory containing templates. @param dest_dir Directory for the rendered output. *) -val json_releases : - [> `O of - ( string - * [> `A of - [> `O of (string * [> `Float of float | `String of string]) list] list - | `Float of float ] - ) - list ] +val json_releases : Mustache.Json.t (** JSON structure representing release information. *) From 3c9b98b1a670a5ea00f82d21906a3bc55eff4952 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 20 Nov 2023 09:30:25 +0000 Subject: [PATCH 67/99] Remove use of custom `finally` in `sdk-gen` Instead use `Fun.protect` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/c/gen_c_binding.ml | 18 ++++++++++-------- ocaml/sdk-gen/common/CommonFunctions.ml | 17 +++++------------ ocaml/sdk-gen/common/CommonFunctions.mli | 5 ----- ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 8 +++++--- 4 files changed, 20 insertions(+), 28 deletions(-) diff --git a/ocaml/sdk-gen/c/gen_c_binding.ml b/ocaml/sdk-gen/c/gen_c_binding.ml index 7121ea70f34..5203b86f2e0 100644 --- a/ocaml/sdk-gen/c/gen_c_binding.ml +++ b/ocaml/sdk-gen/c/gen_c_binding.ml @@ -138,14 +138,16 @@ let rec main () = and gen_class f g clas targetdir = let out_chan = open_out (Filename.concat targetdir (g clas.name)) in - finally (fun () -> f clas out_chan) ~always:(fun () -> close_out out_chan) + Fun.protect (fun () -> f clas out_chan) ~finally:(fun () -> close_out out_chan) and gen_enum f g targetdir = function | Enum (name, _) as x -> if not (List.mem name !all_headers) then all_headers := name :: !all_headers ; let out_chan = open_out (Filename.concat targetdir (g name)) in - finally (fun () -> f x out_chan) ~always:(fun () -> close_out out_chan) + Fun.protect + (fun () -> f x out_chan) + ~finally:(fun () -> close_out out_chan) | _ -> assert false @@ -155,9 +157,9 @@ and gen_map f g targetdir = function if not (List.mem name !all_headers) then all_headers := name :: !all_headers ; let out_chan = open_out (Filename.concat targetdir (g name)) in - finally + Fun.protect (fun () -> f name l r out_chan) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) | _ -> assert false @@ -972,14 +974,14 @@ and gen_failure_h () = let out_chan = open_out (Filename.concat destdir "include/xen/api/xen_api_failure.h") in - finally + Fun.protect (fun () -> print_h_header out_chan protect ; gen_failure_enum out_chan ; gen_failure_funcs out_chan ; print_h_footer out_chan ) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) and gen_failure_enum out_chan = let print format = fprintf out_chan format in @@ -1029,7 +1031,7 @@ and gen_failure_funcs out_chan = and gen_failure_c () = let out_chan = open_out (Filename.concat destdir "src/xen_api_failure.c") in let print format = fprintf out_chan format in - finally + Fun.protect (fun () -> print "%s\n\n\ @@ -1055,7 +1057,7 @@ and gen_failure_c () = Licence.bsd_two_clause (String.concat ",\n " (failure_lookup_entries ())) ) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) and failure_lookup_entries () = List.sort String.compare diff --git a/ocaml/sdk-gen/common/CommonFunctions.ml b/ocaml/sdk-gen/common/CommonFunctions.ml index cd262792b57..fe89dd9600e 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.ml +++ b/ocaml/sdk-gen/common/CommonFunctions.ml @@ -11,13 +11,6 @@ type wireProtocol = XmlRpc | JsonRpc type rpm_version = {major: int; minor: int; micro: int} -let finally f ~(always : unit -> unit) = - match f () with - | result -> - always () ; result - | exception e -> - always () ; raise e - let parse_to_rpm_version_option inputStr = try Scanf.sscanf inputStr "%d.%d.%d" (fun x y z -> @@ -27,18 +20,18 @@ let parse_to_rpm_version_option inputStr = let string_of_file filename = let in_channel = open_in filename in - finally + Fun.protect (fun () -> let rec read_lines acc = try read_lines (input_line in_channel :: acc) with End_of_file -> acc in read_lines [] |> List.rev |> String.concat "\n" ) - ~always:(fun () -> close_in in_channel) + ~finally:(fun () -> close_in in_channel) let with_output filename f = let io = open_out filename in - finally (fun () -> f io) ~always:(fun () -> close_out io) + Fun.protect (fun () -> f io) ~finally:(fun () -> close_out io) let escape_xml s = s @@ -269,9 +262,9 @@ and render_template template_file json output_file = let templ = string_of_file template_file |> Mustache.of_string in let rendered = Mustache.render templ json in let out_chan = open_out output_file in - finally + Fun.protect (fun () -> output_string out_chan rendered) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) let render_file (infile, outfile) json templates_dir dest_dir = let input_path = Filename.concat templates_dir infile in diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli index fd2ba766568..d114f6ea69a 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.mli +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -14,11 +14,6 @@ val is_setter : Datamodel_types.message -> bool @param message Message to check. @return [true] if the message is a setter, [false] otherwise. *) -val finally : (unit -> 'a) -> always:(unit -> unit) -> 'a -(** [finally f ~always] Executes [f] and then [always] regardless of success or failure. - @param f Function to execute. - @param always Function to execute always. *) - val is_getter : Datamodel_types.message -> bool (** [is_getter message] Checks if a message is a getter based on its name. @param message Message to check. diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index db12da58b29..0c70b85c6d6 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -240,9 +240,9 @@ and gen_class_file cls = let out_chan = open_out (Filename.concat destdir (exposed_class_name cls.name) ^ ".cs") in - finally + Fun.protect (fun () -> gen_class out_chan cls) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) and gen_class out_chan cls = let print format = fprintf out_chan format in @@ -842,7 +842,9 @@ and gen_enum' name contents = (* ------------------- category: maps *) and gen_maps () = let out_chan = open_out (Filename.concat destdir "Maps.cs") in - finally (fun () -> gen_maps' out_chan) ~always:(fun () -> close_out out_chan) + Fun.protect + (fun () -> gen_maps' out_chan) + ~finally:(fun () -> close_out out_chan) and gen_maps' out_chan = let print format = fprintf out_chan format in From 3d9dc50b895af9bb305d26f8f6e00f622ffe44da Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 20 Nov 2023 11:34:12 +0000 Subject: [PATCH 68/99] Replace usage of escaped strings with `{||}` in Java's `main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 213 +++++++++++++++++++------------------ 1 file changed, 108 insertions(+), 105 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index b0f3538978c..3fddff31b96 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -520,16 +520,18 @@ let gen_class cls folder = let publishInfo = get_published_info_class cls in print_license file ; fprintf file - "package com.xensource.xenapi;\n\n\ - import com.fasterxml.jackson.annotation.JsonProperty;\n\ - import com.fasterxml.jackson.core.JsonProcessingException;\n\ - import com.fasterxml.jackson.core.type.TypeReference;\n\ - import com.xensource.xenapi.Types.BadServerResponse;\n\ - import com.xensource.xenapi.Types.XenAPIException;\n\n\ - import java.io.PrintWriter;\n\ - import java.io.StringWriter;\n\ - import java.util.*;\n\ - import java.io.IOException;\n\n" ; + {|package com.xensource.xenapi; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.xensource.xenapi.Types.BadServerResponse; +import com.xensource.xenapi.Types.XenAPIException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.*; +import java.io.IOException; + +|} ; fprintf file "/**\n" ; fprintf file " * %s\n" cls.description ; if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; @@ -684,122 +686,123 @@ let gen_method_error_throw file name error = ) in - fprintf file " if (errorName.equals(\"%s\"))\n" name ; - fprintf file " {\n" ; + fprintf file " if (errorName.equals(\"%s\")){\n" name ; (* Prepare the parameters to the Exception constructor *) List.iter (fun i -> fprintf file - " String p%i = errorData.length > %i ? errorData[%i] : \ - \"\";\n" + " String p%i = errorData.length > %i ? errorData[%i] : \"\";\n" i i i ) (range (List.length error.err_params)) ; - fprintf file " throw new Types.%s(%s);\n" class_name paramsStr ; - fprintf file " }\n" + fprintf file " throw new Types.%s(%s);\n" class_name paramsStr ; + fprintf file " }\n" let gen_types_class folder = let class_name = "Types" in let file = open_out (Filename.concat folder class_name ^ ".java") in print_license file ; fprintf file - "package com.xensource.xenapi;\n\n\ - import java.util.Map;\n\n\ - import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;\n\ - import com.fasterxml.jackson.annotation.JsonProperty;\n\ - import java.io.IOException;\n\n\ - /**\n\ - \ * This class holds enum types and exceptions.\n\ - \ */\n\ - public class Types\n\ - {\n\ - \ /**\n\ - \ * Interface for all Record classes\n\ - \ */\n\ - \ public interface Record\n\ - \ {\n\ - \ /**\n\ - \ * Convert a Record to a Map\n\ - \ */\n\ - \ Map toMap();\n\ - \ }\n\n\ - \ /**\n\ - \ * Base class for all XenAPI Exceptions\n\ - \ */\n\ - \ public static class XenAPIException extends IOException {\n\ - \ public final String shortDescription;\n\ - \ public final String[] errorDescription;\n\n\ - \ XenAPIException(String shortDescription)\n\ - \ {\n\ - \ this.shortDescription = shortDescription;\n\ - \ this.errorDescription = null;\n\ - \ }\n\n\ - \ XenAPIException(String[] errorDescription)\n\ - \ {\n\ - \ this.errorDescription = errorDescription;\n\n\ - \ if (errorDescription.length > 0)\n\ - \ {\n\ - \ shortDescription = errorDescription[0];\n\ - \ } else\n\ - \ {\n\ - \ shortDescription = \"\";\n\ - \ }\n\ - \ }\n\n\ - \ public String toString()\n\ - \ {\n\ - \ if (errorDescription == null)\n\ - \ {\n\ - \ return shortDescription;\n\ - \ } else if (errorDescription.length == 0)\n\ - \ {\n\ - \ return \"\";\n\ - \ }\n\ - \ StringBuilder sb = new StringBuilder();\n\ - \ for (int i = 0; i < errorDescription.length - 1; i++)\n\ - \ {\n\ - \ sb.append(errorDescription[i]);\n\ - \ }\n\ - \ sb.append(errorDescription[errorDescription.length - 1]);\n\n\ - \ return sb.toString();\n\ - \ }\n\ - \ }\n\ - \ /**\n\ - \ * Thrown if the response from the server contains an invalid status.\n\ - \ */\n\ - \ public static class BadServerResponse extends XenAPIException\n\ - \ {\n\ - \ public BadServerResponse(JsonRpcResponseError responseError)\n\ - \ {\n\ - \ super(String.valueOf(responseError));\n\ - \ }\n\ - \ }\n\n\ - \ " ; + {|package com.xensource.xenapi; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonEnumDefaultValue; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; + +/** + * This class holds enum types and exceptions. + */ +public class Types +{ + /** + * Interface for all Record classes + */ + public interface Record + { + /** + * Convert a Record to a Map + */ + Map toMap(); + } + /** + * Base class for all XenAPI Exceptions + */ + public static class XenAPIException extends IOException { + public final String shortDescription; + public final String[] errorDescription; + XenAPIException(String shortDescription) + { + this.shortDescription = shortDescription; + this.errorDescription = null; + } + XenAPIException(String[] errorDescription) + { + this.errorDescription = errorDescription; + if (errorDescription.length > 0) + { + shortDescription = errorDescription[0]; + } else + { + shortDescription = ""; + } + } + public String toString() + { + if (errorDescription == null) + { + return shortDescription; + } else if (errorDescription.length == 0) + { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < errorDescription.length - 1; i++) + { + sb.append(errorDescription[i]); + } + sb.append(errorDescription[errorDescription.length - 1]); + return sb.toString(); + } + } + /** + * Thrown if the response from the server contains an invalid status. + */ + public static class BadServerResponse extends XenAPIException + { + public BadServerResponse(JsonRpcResponseError responseError) + { + super(String.valueOf(responseError)); + } + } +|} ; fprintf file - " /**\n\ - \ * Checks the provided server response was successful. If the call \ - failed, throws a XenAPIException. If the server\n\ - \ * returned an invalid response, throws a BadServerResponse. \ - Otherwise, returns the server response as passed in.\n\ - \ */\n\ - \ public static void checkError(JsonRpcResponseError response) throws \ - XenAPIException, BadServerResponse\n\ - \ {\n\ - \ var errorData = response.data;\n\ - \ if(errorData.length == 0){\n\ - \ throw new BadServerResponse(response);\n\ - \ }\n\ - \ var errorName = errorData[0];\n\n" ; + {| /** + * Checks the provided server response was successful. If the call + * failed, throws a XenAPIException. If the server + * returned an invalid response, throws a BadServerResponse. + * Otherwise, returns the server response as passed in. + */ + public static void checkError(JsonRpcResponseError response) throws XenAPIException, BadServerResponse + { + var errorData = response.data; + if(errorData.length == 0){ + throw new BadServerResponse(response); + } + var errorName = errorData[0]; +|} ; Hashtbl.iter (gen_method_error_throw file) Datamodel.errors ; fprintf file - "\n\ - \ // An unknown error occurred\n\ - \ throw new Types.XenAPIException(errorData);\n\ - \ }\n\n" ; + {| + // An unknown error occurred + throw new Types.XenAPIException(errorData); +} + +|} ; gen_enums file ; fprintf file "\n" ; From 1224222420015470c7bfe1c54e0f2f7ec08656a9 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 20 Nov 2023 11:39:36 +0000 Subject: [PATCH 69/99] Replace mentions of `java.net.http.HttpClient` in `JsonRpcClient.java` Signed-off-by: Danilo Del Busso --- .../src/main/java/com/xensource/xenapi/JsonRpcClient.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index 8747822672c..0dad4b7dc0a 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -44,7 +44,6 @@ import java.io.IOException; import java.net.URL; -import java.net.http.HttpClient; import java.text.SimpleDateFormat; import java.util.concurrent.TimeUnit; @@ -61,7 +60,7 @@ * By default, the timeout for requests is set to 10 minutes (600 seconds). The default timeout for connecting to the * JSON-RPC backend is set to 5 seconds. * - * @see HttpClient HttpClient is used to make requests and connect to the backend + * @see CloseableHttpClient CloseableHttpClient is used to make requests and connect to the backend * @see ObjectMapper ObjectMapper is used to marshall requests and responses */ public class JsonRpcClient { @@ -120,12 +119,12 @@ public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTi } /** - * Initialize a JsonRpcClient using a custom HttpClient instance. + * Initialize a JsonRpcClient using a custom CloseableHttpClient instance. * * @param client the custom HttpClient to use for all requests * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. * @param requestTimeout the timeout value for requests. - * @see HttpClient + * @see CloseableHttpClient CloseableHttpClient the client that will be used for dispatching requests * @see JsonRpcClient JsonRpcClient for more info on using this class */ public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requestTimeout) { From 3f9ed3956f4b772e616d72ea907f7a4e3e8fc846 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 27 Nov 2023 08:44:05 +0000 Subject: [PATCH 70/99] CP-45888: Do not expose `JsonProcessingException` to SDK users Exception is already subclass of stdlib exception `IOException` and there's no need to expose it explicitly Signed-off-by: Danilo Del Busso --- .../main/java/com/xensource/xenapi/Connection.java | 11 ++++------- ocaml/sdk-gen/java/main.ml | 9 ++++----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index f568c452894..fcf64b78485 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -29,7 +29,6 @@ package com.xensource.xenapi; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.xensource.xenapi.Types.BadServerResponse; import com.xensource.xenapi.Types.XenAPIException; @@ -203,10 +202,9 @@ public String getSessionReference() { * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records * @return The result of the call with the type specified under T. * @throws XenAPIException if the call failed. - * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON - * @throws IOException if an I/O error occurs when sending or receiving + * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. */ - public T dispatch(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws XenAPIException, JsonProcessingException, IOException { + public T dispatch(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws XenAPIException, IOException { var result = client.sendRequest(methodCall, methodParameters, responseTypeReference); if (result.error != null) { throw new XenAPIException(String.valueOf(result.error)); @@ -231,10 +229,9 @@ public T dispatch(String methodCall, Object[] methodParameters, TypeReferenc * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password * @param methodParameters the methodParameters of the method call * @throws XenAPIException if the call failed. - * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON - * @throws IOException if an I/O error occurs when sending or receiving + * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. */ - public void dispatch(String methodCall, Object[] methodParameters) throws XenAPIException, JsonProcessingException, IOException { + public void dispatch(String methodCall, Object[] methodParameters) throws XenAPIException, IOException { var typeReference = new TypeReference() { }; this.dispatch(methodCall, methodParameters, typeReference); diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 3fddff31b96..03d3ab73b8e 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -233,11 +233,11 @@ let gen_method file cls message params async_version = , "Thrown if the response from the server contains an invalid status." ) ; ("XenAPIException", "if the call failed.") - ; ( "JsonProcessingException" - , "if the request's payload or the response's payload cannot be written \ - or read as valid JSON." + ; ( "IOException" + , "if an I/O error occurs when sending or receiving, includes cases when \ + the request's payload or the response's payload cannot be written or \ + read as valid JSON." ) - ; ("IOException", "if an I/O error occurs when sending or receiving.") ] in let publishInfo = get_published_info_message message cls in @@ -522,7 +522,6 @@ let gen_class cls folder = fprintf file {|package com.xensource.xenapi; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.xensource.xenapi.Types.BadServerResponse; import com.xensource.xenapi.Types.XenAPIException; From 13c1a29b44a4cf992721d174cadcc1ce6b9f91d7 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 8 Jan 2024 15:36:45 +0000 Subject: [PATCH 71/99] CP-45888: Fix misc typos and comments Also remove a redundant `List.map` in `java/main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/c/helper.mli | 2 +- ocaml/sdk-gen/common/license.mli | 2 +- .../main/java/com/xensource/xenapi/Connection.java | 2 +- .../java/com/xensource/xenapi/JsonRpcClient.java | 2 +- ocaml/sdk-gen/java/main.ml | 3 +-- ocaml/sdk-gen/powershell/common_functions.ml | 2 +- ocaml/sdk-gen/powershell/common_functions.mli | 14 +++++++------- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ocaml/sdk-gen/c/helper.mli b/ocaml/sdk-gen/c/helper.mli index 1d8b90de096..ea32b78f207 100644 --- a/ocaml/sdk-gen/c/helper.mli +++ b/ocaml/sdk-gen/c/helper.mli @@ -1,6 +1,6 @@ val formatted_wrap : Format.formatter -> string -> unit (** Recursively formats the input string - based on spaces and newlines, ensuring proper indentation. + based on spaces and new lines, ensuring proper indentation. @param formatter The formatter to output the formatted string. @param s The input string to be formatted. *) diff --git a/ocaml/sdk-gen/common/license.mli b/ocaml/sdk-gen/common/license.mli index 6ab5ae4cc08..092b79357a6 100644 --- a/ocaml/sdk-gen/common/license.mli +++ b/ocaml/sdk-gen/common/license.mli @@ -1,2 +1,2 @@ -(* Conent of BSD 2 License *) +(* Content of BSD 2 License *) val bsd_two_clause : string diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index fcf64b78485..6005982aaa6 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -199,7 +199,7 @@ public String getSessionReference() { * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password * @param methodParameters the methodParameters of the method call * @param responseTypeReference the type of the response, wrapped with a TypeReference - * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records + * @param The type of the response's payload. For instance, a map of opaque references to VM objects is expected when calling VM.get_all_records * @return The result of the call with the type specified under T. * @throws XenAPIException if the call failed. * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index 0dad4b7dc0a..7172e148f4c 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -143,7 +143,7 @@ public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requ * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password * @param methodParameters the parameters of the method call * @param responseTypeReference the type of the response, wrapped with a TypeReference - * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records + * @param The type of the response's payload. For instance, a map of opaque references to VM objects is expected when calling VM.get_all_records * @return a JsonRpcResponse object. If its error field is empty, the response was successful. * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON * @throws IOException if an I/O error occurs when sending or receiving diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 03d3ab73b8e..b45f02c2fc8 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -336,8 +336,7 @@ let gen_method file cls message params async_version = get_method_params_for_xml message params in - output_string file - (String.concat ", " (List.map (fun s -> sprintf "%s" s) methodParamsList)) ; + output_string file (String.concat ", " methodParamsList) ; fprintf file "};\n" ; diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index a2832c14494..fd47311eba5 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -144,7 +144,7 @@ and is_invoke message = && (not (is_constructor message)) && not (is_destructor message) -(* Some adders/removers are just prefixed by Add or RemoveFrom +(* Some adders/removers are just prefixed by Add or Remove and some are prefixed by AddTo or RemoveFrom *) and cut_msg_name message_name fn_type = let name_len = String.length message_name in diff --git a/ocaml/sdk-gen/powershell/common_functions.mli b/ocaml/sdk-gen/powershell/common_functions.mli index 706264469f4..f947139bec0 100644 --- a/ocaml/sdk-gen/powershell/common_functions.mli +++ b/ocaml/sdk-gen/powershell/common_functions.mli @@ -86,13 +86,13 @@ val exposed_class_name : string -> string val cut_msg_name : string -> string -> string (** Extracts the base name from an OCaml message name by removing the specified prefix. - Some adders/removers are just prefixed by Add or RemoveFrom - and some are prefixed by AddTo or RemoveFrom - @param message_name - OCaml message name. - @param fn_type - Prefix to remove ("Add" or "Remove"). - @return Base name after removing the specified prefix. *) + Some adders/removers are just prefixed by Add or Remove and some are prefixed by + AddTo or RemoveFrom. + @param message_name - OCaml message name. + @param fn_type - Prefix to remove ("Add" or "Remove"). + @return Base name after removing the specified prefix. *) val lower_and_underscore_first : string -> string (** Converts a string to lowercase and adds an underscore at the beginning. - @param s - Input string. - @return String in lowercase with an underscore at the beginning. *) + @param s - Input string. + @return String in lowercase with an underscore at the beginning. *) From dbe7e11bd3e78ba4b763b44497c6f6c3b7d4b1e8 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 9 Jan 2024 15:18:11 +0000 Subject: [PATCH 72/99] Add missing function signature in `CommonFunctions.mli` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/common/CommonFunctions.mli | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli index d114f6ea69a..2b2312fafb7 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.mli +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -58,6 +58,13 @@ val is_remover : Datamodel_types.message -> bool @param message Message to check. @return [true] if the message is a remover, [false] otherwise. *) +val get_minimum_allowed_role : Datamodel_types.message -> string +(** [msg message] Get the minimum RBAC role required to run the procedure of the input message. + This function ignores internal roles. + If no matching role is found, the string "Not Applicable" is returned + @param message Input message. + @return string the name of the RBAC role if a matching one is found, "Not Applicable" otherwise. *) + val gen_param_groups : Datamodel_types.message -> Datamodel_types.param list From e8e589f57312a71868acdc278020d60dba1591b6 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 9 Jan 2024 15:42:53 +0000 Subject: [PATCH 73/99] CP-45888: Bump Java SDK dependencies to latest stable versions `jackson-databind` to `2.16.1` and `httpclient5` to `5.4` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/autogen/xen-api/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml index c78ec926540..abaed44d833 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml +++ b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml @@ -55,12 +55,12 @@ com.fasterxml.jackson.core jackson-databind - 2.15.1 + 2.16.1 org.apache.httpcomponents.client5 httpclient5 - 5.2.1 + 5.3 From b876a3ea429850593ea9bf54388aea3003657373 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 10 Jan 2024 08:39:18 +0000 Subject: [PATCH 74/99] Apply misc formatting and style changes - Shorten docs for Java's `IOException` - Use `String.concat` instead of multiple `^` calls in Java's `main.ml` - Rename `pascal_case_` to `pascal_case_rec` to avoid confusion for the fuinction's consumers - Use `{| |}` to replace escaped quotes when possible in Java's `main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 18 +++++++----------- ocaml/sdk-gen/powershell/common_functions.ml | 6 +++--- ocaml/sdk-gen/powershell/common_functions.mli | 2 +- .../powershell/gen_powershell_binding.ml | 12 ++++++------ 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index b45f02c2fc8..a79ffd724f8 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -234,9 +234,8 @@ let gen_method file cls message params async_version = ) ; ("XenAPIException", "if the call failed.") ; ( "IOException" - , "if an I/O error occurs when sending or receiving, includes cases when \ - the request's payload or the response's payload cannot be written or \ - read as valid JSON." + , "if an error occurs during a send or receive. This includes cases \ + where a payload is invalid JSON." ) ] in @@ -420,7 +419,7 @@ and gen_record_tostring_contents file prefix = function let field_default = function | SecretString | String -> - "\"\"" + {|""|} | Int -> "0" | Float -> @@ -438,7 +437,7 @@ let field_default = function | Map (t1, t2) -> sprintf "new HashMap<%s, %s>()" (get_java_type t1) (get_java_type t2) | Ref ty -> - sprintf "new %s(\"OpaqueRef:NULL\")" (class_case ty) + sprintf {|new %s("OpaqueRef:NULL")|} (class_case ty) | Record _ -> assert false | Option _ -> @@ -611,15 +610,12 @@ let gen_enum file name ls = global_replace (regexp_string "\n") "\n * " escaped_description in let comment = - " /**\n" - ^ " * " - ^ final_description - ^ "\n" - ^ " */\n" + String.concat "\n" + [" /**"; " * " ^ final_description; " */"] in let json_property = if name != "UNRECOGNIZED" then - "@JsonProperty(\"" ^ name ^ "\")" + {|@JsonProperty("|} ^ name ^ {|"|} else "@JsonEnumDefaultValue" in diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index fd47311eba5..edff4bf5c70 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -8,7 +8,7 @@ open Datamodel_types open CommonFunctions module DU = Datamodel_utils -let rec pascal_case_ s = +let rec pascal_case_rec s = let ss = Astring.String.cuts ~sep:"_" ~empty:true s |> List.map String.capitalize_ascii @@ -30,7 +30,7 @@ let rec pascal_case_ s = h' ^ String.concat "" tl and pascal_case s = - let str = pascal_case_ s in + let str = pascal_case_rec s in if String.starts_with ~prefix:"set" (String.lowercase_ascii str) || String.starts_with ~prefix:"get" (String.lowercase_ascii str) @@ -197,7 +197,7 @@ and get_http_action_stem name = let parts = Astring.String.cuts ~sep:"_" name in let filtered = List.filter trim_http_action_stem parts in let trimmed = String.concat "_" filtered in - match trimmed with "" -> pascal_case_ "vm" | _ -> pascal_case_ trimmed + match trimmed with "" -> pascal_case_rec "vm" | _ -> pascal_case_rec trimmed and trim_http_action_stem x = match x with diff --git a/ocaml/sdk-gen/powershell/common_functions.mli b/ocaml/sdk-gen/powershell/common_functions.mli index f947139bec0..b13bd53aea2 100644 --- a/ocaml/sdk-gen/powershell/common_functions.mli +++ b/ocaml/sdk-gen/powershell/common_functions.mli @@ -14,7 +14,7 @@ val get_common_verb_category : string -> string @param verb - HTTP action verb. @return Common verb category. *) -val pascal_case_ : string -> string +val pascal_case_rec : string -> string (** Recursively converts a string to PascalCase. @param s - String to convert. @return PascalCase formatted string. *) diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 96d0782bf67..26b94fba23e 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -171,14 +171,14 @@ and gen_arg_param = function else "" ) - (pascal_case_ x) + (pascal_case_rec x) | Int64_query_arg x -> sprintf "\n [Parameter]\n public long? %s { get; set; }\n" - (pascal_case_ x) + (pascal_case_rec x) | Bool_query_arg x -> let y = if x = "host" then "is_host" else x in sprintf "\n [Parameter]\n public bool? %s { get; set; }\n" - (pascal_case_ y) + (pascal_case_rec y) | Varargs_query_arg -> sprintf "\n\ @@ -214,12 +214,12 @@ and gen_call_arg_params args = and gen_call_arg_param = function | String_query_arg x -> - sprintf ", %s" (pascal_case_ x) + sprintf ", %s" (pascal_case_rec x) | Int64_query_arg x -> - sprintf ", %s" (pascal_case_ x) + sprintf ", %s" (pascal_case_rec x) | Bool_query_arg x -> let y = if x = "host" then "is_host" else x in - sprintf ", %s" (pascal_case_ y) + sprintf ", %s" (pascal_case_rec y) | Varargs_query_arg -> sprintf ", Args" From 83994a9b37e45d5b378a31c9cb6a3c5bb1b449c8 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 10 Jan 2024 10:13:06 +0000 Subject: [PATCH 75/99] CP-45888: Use public setters for client settings in `JsonRpcClient` - Deprecates `Connection` constructors that take in `requestTimeout` or `connectionTimeout` - Add docs to tell user to use `Connection.client#setXYZ` instead - Add setters to `JsonRpcClient` - Replace `protected` with `private` in `JsonRpcClient` when possible, since API consumers will now be able to access `Connection.client` - Make `Connection.client` `public` - Fix misc formatting in `Connection` Signed-off-by: Danilo Del Busso --- .../java/com/xensource/xenapi/Connection.java | 93 ++++++++++++++----- .../com/xensource/xenapi/JsonRpcClient.java | 93 ++++++++++++------- 2 files changed, 134 insertions(+), 52 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index 6005982aaa6..356d5dbfe22 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -33,9 +33,11 @@ import com.xensource.xenapi.Types.BadServerResponse; import com.xensource.xenapi.Types.XenAPIException; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.util.Timeout; import java.io.IOException; import java.net.URL; +import java.util.concurrent.TimeUnit; /** * Represents a connection to a XenServer. Creating a new instance of this class initialises a new JsonRpcClient that is @@ -43,7 +45,8 @@ * method call, and dispatches it on the Connection's client via the dispatch method. */ public class Connection { - private final JsonRpcClient client; + + public final JsonRpcClient client; private APIVersion apiVersion; /** * The opaque reference to the session used by this connection @@ -75,12 +78,17 @@ public Connection(JsonRpcClient jsonRpcClient) { * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. * - * @param httpClient The HttpClient used to make calls, this will be used by JsonRpcClient for handling requests - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param httpClient The HttpClient used to make calls, this will be used by the underlying {@link #client} for handling requests + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. * @param requestTimeout The reply timeout for JSON-RPC calls in seconds + * @deprecated This constructor is deprecated. To set the {@code requestTimeout} please {@link #setRequestTimeout(int)}. You may also use the {@link com.xensource.xenapi.JsonRpcClient#setRequestTimeout(int)} + * method of this object's {@link #client}. This option is only advisable if you are managing your own {@link com.xensource.xenapi.JsonRpcClient} as the underlying + * {@link #client} for this object. */ + @Deprecated public Connection(CloseableHttpClient httpClient, URL url, int requestTimeout) { - this.client = new JsonRpcClient(httpClient, url, requestTimeout); + this.client = new JsonRpcClient(httpClient, url); + this.client.setRequestTimeout(requestTimeout); } /** @@ -96,7 +104,7 @@ public Connection(CloseableHttpClient httpClient, URL url, int requestTimeout) { * This constructor uses the default values of the reply and connection timeouts for the JSON-RPC calls * (600 seconds and 5 seconds respectively). * - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. */ public Connection(URL url) { this.client = new JsonRpcClient(url); @@ -112,12 +120,19 @@ public Connection(URL url) { * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. * - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. * @param requestTimeout The reply timeout for JSON-RPC calls in seconds * @param connectionTimeout The connection timeout for JSON-RPC calls in seconds + * @deprecated This constructor is deprecated. To set {@code requestTimeout} or {@code connectionTimeout} please use {@link #setRequestTimeout(int)} or {@link #setConnectionTimeout(int)} respectively. + * You may also use the {@link com.xensource.xenapi.JsonRpcClient#setRequestTimeout(int)} method of this object's {@link #client}. + * This option is only advisable if you are managing your own {@link com.xensource.xenapi.JsonRpcClient} as the underlying + * {@link #client} for this object. */ + @Deprecated public Connection(URL url, int requestTimeout, int connectionTimeout) { - this.client = new JsonRpcClient(url, requestTimeout, connectionTimeout); + this.client = new JsonRpcClient(url); + this.client.setRequestTimeout(requestTimeout); + this.client.setConnectionTimeout(connectionTimeout); } /** @@ -133,7 +148,7 @@ public Connection(URL url, int requestTimeout, int connectionTimeout) { * This constructor uses the default values of the reply and connection timeouts for the JSON-RPC calls * (600 seconds and 5 seconds respectively). * - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. * @param sessionReference A reference to a logged-in Session. Any method calls on this * Connection will use it. This constructor does not call Session.loginWithPassword, and dispose() on the resulting * Connection object does not call Session.logout. The programmer is responsible for ensuring the Session is logged @@ -154,19 +169,52 @@ public Connection(URL url, String sessionReference) { * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. * - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. * @param sessionReference A reference to a logged-in Session. Any method calls on this Connection will use it. * This constructor does not call Session.loginWithPassword, and dispose() on the resulting * Connection object does not call Session.logout. The programmer is responsible for * ensuring the Session is logged in and out correctly. * @param requestTimeout The reply timeout for JSON-RPC calls in seconds * @param connectionTimeout The connection timeout for JSON-RPC calls in seconds + * @deprecated This constructor is deprecated. To set {@code requestTimeout} or {@code connectionTimeout} please use {@link #setRequestTimeout(int)} or {@link #setConnectionTimeout(int)} respectively. + * You may also use the {@link com.xensource.xenapi.JsonRpcClient#setRequestTimeout(int)} method of this object's {@link #client}. + * This option is only advisable if you are managing your own {@link com.xensource.xenapi.JsonRpcClient} as the underlying + * {@link #client} for this object. */ + @Deprecated public Connection(URL url, String sessionReference, int requestTimeout, int connectionTimeout) { - this.client = new JsonRpcClient(url, requestTimeout, connectionTimeout); + this.client = new JsonRpcClient(url); + this.client.setRequestTimeout(requestTimeout); + this.client.setConnectionTimeout(connectionTimeout); this.sessionReference = sessionReference; } + /** + * Set the timeout in seconds for every request made by this object's {@link #client}. + * If not set the value defaults to {@value JsonRpcClient#DEFAULT_REQUEST_TIMEOUT}. + * You may also pass your own {@link JsonRpcClient} in the constructor for more control. + * + * @param requestTimeout the timeout value in seconds + * @throws NullPointerException if the {@link #client} is null + * @see org.apache.hc.client5.http.config.RequestConfig.Builder#setConnectionRequestTimeout(long, TimeUnit) + */ + public void setRequestTimeout(int requestTimeout) throws NullPointerException { + this.client.setRequestTimeout(requestTimeout); + } + + /** + * Set the connection timeout in seconds for its {@link #client}'s {@link org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager}. + * If not set the value defaults to {@value JsonRpcClient#DEFAULT_CONNECTION_TIMEOUT}. + * You may also pass your own {@link JsonRpcClient} in the constructor for more control. + * + * @param connectionTimeout the client's connection timeout in seconds. + * @throws NullPointerException if the {@link #client} is null + * @see org.apache.hc.client5.http.config.ConnectionConfig.Builder#setConnectTimeout(Timeout) + */ + public void setConnectionTimeout(int connectionTimeout) { + this.client.setConnectionTimeout(connectionTimeout); + } + /** * Updated when Session.login_with_password() is called. */ @@ -176,9 +224,12 @@ public APIVersion getAPIVersion() { private void setAPIVersion(Session session) throws IOException { try { - long major = session.getThisHost(this).getAPIVersionMajor(this); - long minor = session.getThisHost(this).getAPIVersionMajor(this); - apiVersion = APIVersion.fromMajorMinor(major, minor); + var pools = Pool.getAllRecords(this); + var pool = pools.values().stream().findFirst(); + if (pool.isPresent()) { + var host = pool.get().master.getRecord(this); + apiVersion = APIVersion.fromMajorMinor(host.APIVersionMajor, host.APIVersionMinor); + } } catch (BadServerResponse exn) { apiVersion = APIVersion.UNKNOWN; } @@ -196,13 +247,13 @@ public String getSessionReference() { /** * Send a method call to xapi's backend. You need to provide the type of the data returned by a successful response. * - * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password - * @param methodParameters the methodParameters of the method call - * @param responseTypeReference the type of the response, wrapped with a TypeReference + * @param methodCall The JSON-RPC xapi method call. e.g.: session.login_with_password + * @param methodParameters The methodParameters of the method call + * @param responseTypeReference The type of the response, wrapped with a TypeReference * @param The type of the response's payload. For instance, a map of opaque references to VM objects is expected when calling VM.get_all_records - * @return The result of the call with the type specified under T. - * @throws XenAPIException if the call failed. - * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. + * @return The result of the call with the type specified under T. + * @throws XenAPIException if the call failed. + * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. */ public T dispatch(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws XenAPIException, IOException { var result = client.sendRequest(methodCall, methodParameters, responseTypeReference); @@ -228,8 +279,8 @@ public T dispatch(String methodCall, Object[] methodParameters, TypeReferenc * * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password * @param methodParameters the methodParameters of the method call - * @throws XenAPIException if the call failed. - * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. + * @throws XenAPIException if the call failed. + * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. */ public void dispatch(String methodCall, Object[] methodParameters) throws XenAPIException, IOException { var typeReference = new TypeReference() { diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index 7172e148f4c..346ea62aba1 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -41,6 +41,7 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.util.Timeout; import java.io.IOException; import java.net.URL; @@ -57,8 +58,10 @@ * The client can be customised by passing it as a parameter to corresponding constructor, enabling custom * handling of requests. *
- * By default, the timeout for requests is set to 10 minutes (600 seconds). The default timeout for connecting to the - * JSON-RPC backend is set to 5 seconds. + *
+ * By default, the timeout for requests is set to {@value #DEFAULT_REQUEST_TIMEOUT}. The default timeout for connecting to the + * JSON-RPC backend is set to {@value #DEFAULT_CONNECTION_TIMEOUT} seconds. The maximum number of concurrent connections handled + * by the underlying {@link PoolingHttpClientConnectionManager} is {@value #MAX_CONCURRENT_CONNECTIONS}. * * @see CloseableHttpClient CloseableHttpClient is used to make requests and connect to the backend * @see ObjectMapper ObjectMapper is used to marshall requests and responses @@ -67,18 +70,20 @@ public class JsonRpcClient { private static final int DEFAULT_REQUEST_TIMEOUT = 600; private static final int DEFAULT_CONNECTION_TIMEOUT = 5; - protected static final int MAX_CONCURRENT_CONNECTIONS = 10; - - protected static final String JSON_BACKEND_PATH = "/jsonrpc"; + private static final int MAX_CONCURRENT_CONNECTIONS = 10; - protected final CloseableHttpClient httpClient; - protected final String jsonRpcBackendUrl; - protected final ObjectMapper objectMapper; - protected final int requestTimeout; + private static final String JSON_BACKEND_PATH = "/jsonrpc"; + private final CloseableHttpClient httpClient; + private final String jsonRpcBackendUrl; + private final ObjectMapper objectMapper; private final RequestConfig defaultRequestConfig = RequestConfig.custom() .setCookieSpec(StandardCookieSpec.IGNORE) .build(); + private int requestTimeout; + private PoolingHttpClientConnectionManager connectionManager; + + //region Constructors /** * Create a JsonRpcClient with default settings. @@ -87,24 +92,12 @@ public class JsonRpcClient { * @see JsonRpcClient JsonRpcClient for more info on using this class */ public JsonRpcClient(URL jsonRpcBackendUrl) { - this(jsonRpcBackendUrl, DEFAULT_REQUEST_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT); - } - - /** - * Create a JsonRpcClient with the option to define the request and connection timeout values. - * - * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. - * @param requestTimeout the timeout value for requests. - * @param connectionTimeout the timeout value for the initial connection to the host. - * @see JsonRpcClient JsonRpcClient for more info on using this class - */ - public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTimeout) { var connectionConfig = ConnectionConfig .custom() - .setConnectTimeout(connectionTimeout, TimeUnit.SECONDS) + .setConnectTimeout(DEFAULT_CONNECTION_TIMEOUT, TimeUnit.SECONDS) .build(); - var connectionManager = new PoolingHttpClientConnectionManager(); + this.connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setDefaultConnectionConfig(connectionConfig); connectionManager.setMaxTotal(MAX_CONCURRENT_CONNECTIONS); @@ -113,7 +106,7 @@ public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTi .setConnectionManager(connectionManager) .build(); this.jsonRpcBackendUrl = formatBackendUrl(jsonRpcBackendUrl); - this.requestTimeout = requestTimeout; + this.requestTimeout = DEFAULT_REQUEST_TIMEOUT; this.objectMapper = new ObjectMapper(); initializeObjectMapperConfiguration(); } @@ -123,20 +116,58 @@ public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTi * * @param client the custom HttpClient to use for all requests * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. - * @param requestTimeout the timeout value for requests. * @see CloseableHttpClient CloseableHttpClient the client that will be used for dispatching requests * @see JsonRpcClient JsonRpcClient for more info on using this class */ - public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requestTimeout) { + public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl) { httpClient = client; - - this.requestTimeout = requestTimeout; this.jsonRpcBackendUrl = formatBackendUrl(jsonRpcBackendUrl); - this.objectMapper = new ObjectMapper(); initializeObjectMapperConfiguration(); } + //endregion + + //region Public Setters + + /** + * Set the timeout in seconds for every request made by this client. + * If not set the value defaults to {@value #DEFAULT_REQUEST_TIMEOUT}. + * + * @param requestTimeout the timeout value in seconds + * @see org.apache.hc.client5.http.config.RequestConfig.Builder#setConnectionRequestTimeout(long, TimeUnit) + */ + public void setRequestTimeout(int requestTimeout) { + this.requestTimeout = requestTimeout; + } + + /** + * Set the connection timeout in seconds for this client's {@link org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager}. + * If not set the value defaults to {@value #DEFAULT_CONNECTION_TIMEOUT}. + * + * @param connectionTimeout the client's connection timeout in seconds. + * @see org.apache.hc.client5.http.config.ConnectionConfig.Builder#setConnectTimeout(Timeout) + */ + public void setConnectionTimeout(int connectionTimeout) { + connectionManager.setDefaultConnectionConfig(ConnectionConfig + .custom() + .setConnectTimeout(connectionTimeout, TimeUnit.SECONDS) + .build() + ); + } + + /** + * Set the maximum number of connections that this client's {@link org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager} will keep open. + * If not set the value defaults to {@value #MAX_CONCURRENT_CONNECTIONS}. + * + * @param maxConcurrentConnections the maximum number of connections managed by the connection manager + * @see org.apache.hc.core5.pool.ConnPoolControl#setMaxTotal(int) + */ + public void setMaxConcurrentConnections(int maxConcurrentConnections) { + connectionManager.setMaxTotal(maxConcurrentConnections); + } + //endregion + /** * Send a method call to xapi's backend. You need to provide the type of the data returned by a successful response. * @@ -144,11 +175,11 @@ public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requ * @param methodParameters the parameters of the method call * @param responseTypeReference the type of the response, wrapped with a TypeReference * @param The type of the response's payload. For instance, a map of opaque references to VM objects is expected when calling VM.get_all_records - * @return a JsonRpcResponse object. If its error field is empty, the response was successful. + * @return a {@link JsonRpcResponse} object. If its error field is empty, the response was successful. * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON * @throws IOException if an I/O error occurs when sending or receiving */ - public JsonRpcResponse sendRequest(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws IOException { + protected JsonRpcResponse sendRequest(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws IOException { var requestBody = objectMapper .writeValueAsString(new JsonRpcRequest(methodCall, methodParameters)); From 1cf6b2d50074ed513f50c94190f9b116314fad9b Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 10 Jan 2024 10:28:21 +0000 Subject: [PATCH 76/99] CP-45888: Use `Pool.getAllRecords` and `Host.getRecord` to set API version This version replicates what we do in the C# SDK, and uses fewer calls to fetch the API version Signed-off-by: Danilo Del Busso --- .../src/main/java/com/xensource/xenapi/Connection.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index 356d5dbfe22..53bdc1d8b83 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -222,7 +222,8 @@ public APIVersion getAPIVersion() { return apiVersion; } - private void setAPIVersion(Session session) throws IOException { + private void setAPIVersion() throws IOException { + apiVersion = APIVersion.UNKNOWN; try { var pools = Pool.getAllRecords(this); var pool = pools.values().stream().findFirst(); @@ -231,7 +232,7 @@ private void setAPIVersion(Session session) throws IOException { apiVersion = APIVersion.fromMajorMinor(host.APIVersionMajor, host.APIVersionMinor); } } catch (BadServerResponse exn) { - apiVersion = APIVersion.UNKNOWN; + // ignore, we default to UNKNOWN } } @@ -264,7 +265,7 @@ public T dispatch(String methodCall, Object[] methodParameters, TypeReferenc if (methodCall.equals("session.login_with_password")) { var session = ((Session) result.result); sessionReference = session.ref; - setAPIVersion(session); + setAPIVersion(); } else if (methodCall.equals("session.slave_local_login_with_password")) { var session = ((Session) result.result); sessionReference = session.ref; From 517ef1a5c3bb4b6822e182a24eed6470d22e4142 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 15 Jan 2024 15:47:18 +0000 Subject: [PATCH 77/99] CP-45888: Fix incorrect URL presence check in `JsonRpcClient` Signed-off-by: Danilo Del Busso --- .../src/main/java/com/xensource/xenapi/JsonRpcClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index 346ea62aba1..e3513908116 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -222,7 +222,7 @@ private String formatBackendUrl(URL url) { // We only replace it when it's empty. // If the user purposely set the path // we use the given value even if incorrect - if (!url.getPath().isEmpty()) { + if (url.getPath().isEmpty()) { return url.getProtocol() + "://" + url.getHost() + JSON_BACKEND_PATH; } return url.toString(); From 9677795da60f6723129819df4e66ec1c82f396d3 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 15 Jan 2024 15:48:44 +0000 Subject: [PATCH 78/99] CP-45888: Restore `toX` methods in `Types.java` - Restore use of `Set` instead of `HashSet` to avoid consumer implementation breaking - Remove use of raw types to avoid runtime exceptions - Deprecate `toX(Object)` methods in `Types` since they should be internal - Restore `toX(Object, Connection)` methods - Supress unchecked cast warnings with annotation to remove them during compilation. Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 251 +++++++++++++++++++++++++++++++++++-- 1 file changed, 243 insertions(+), 8 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index a79ffd724f8..254ce1c35f8 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -142,7 +142,7 @@ let rec get_java_type ty = Hashtbl.replace enums name ls ; sprintf "Types.%s" (class_case name) | Set t1 -> - sprintf "HashSet<%s>" (get_java_type t1) + sprintf "Set<%s>" (get_java_type t1) | Map (t1, t2) -> sprintf "Map<%s, %s>" (get_java_type t1) (get_java_type t2) | Ref x -> @@ -157,6 +157,36 @@ let rec get_java_type ty = let switch_enum = Enum ("XenAPIObjects", List.map (fun x -> (x.name, x.description)) classes) +(*Helper function for get_marshall_function*) +let rec get_marshall_function_rec = function + | SecretString | String -> + "String" + | Int -> + "Long" + | Float -> + "Double" + | Bool -> + "Boolean" + | DateTime -> + "Date" + | Enum (name, _) -> + class_case name + | Set t1 -> + sprintf "SetOf%s" (get_marshall_function_rec t1) + | Map (t1, t2) -> + sprintf "MapOf%s%s" + (get_marshall_function_rec t1) + (get_marshall_function_rec t2) + | Ref ty -> + class_case ty (* We want to hide all refs *) + | Record ty -> + sprintf "%sRecord" (class_case ty) + | Option ty -> + get_marshall_function_rec ty + +(*get_marshall_function (Set(Map(Float,Bool)));; -> "toSetOfMapOfDoubleBoolean"*) +let get_marshall_function ty = "to" ^ get_marshall_function_rec ty + let _ = get_java_type switch_enum (* Generate the methods *) @@ -321,8 +351,7 @@ let gen_method file cls message params async_version = List.iter (fun {param_name= s; _} -> let name = camel_case s in - fprintf file " Map %s_map = %s.toMap();\n" name - name + fprintf file " var %s_map = %s.toMap();\n" name name ) record_params ; @@ -484,7 +513,7 @@ let gen_record file cls = fprintf file " * Convert a %s.Record to a Map\n" cls.name ; fprintf file " */\n" ; fprintf file " public Map toMap() {\n" ; - fprintf file " Map map = new HashMap<>();\n" ; + fprintf file " var map = new HashMap();\n" ; List.iter (gen_record_tomap_contents file []) contents ; if cls.name = "event" then @@ -596,6 +625,194 @@ import java.io.IOException; fprintf file "}" ; close_out file +(**?*) +(* Generate Marshalling Class *) + +(*This generates the special case code for marshalling the snapshot field in an Event.Record*) +let generate_snapshot_hack file = + fprintf file "\n" ; + fprintf file "\n" ; + fprintf file " Object a,b;\n" ; + fprintf file " a=map.get(\"snapshot\");\n" ; + fprintf file " switch(%s(record.clazz))\n" + (get_marshall_function switch_enum) ; + fprintf file " {\n" ; + List.iter + (fun x -> + fprintf file " case %17s: b = %25s(a); break;\n" + (String.uppercase_ascii x) + (get_marshall_function (Record x)) + ) + (List.map + (fun x -> x.name) + (List.filter (fun x -> not (class_is_empty x)) classes) + ) ; + fprintf file + " default: throw new RuntimeException(\"Internal error in \ + auto-generated code whilst unmarshalling event snapshot\");\n" ; + fprintf file " }\n" ; + fprintf file " record.snapshot = b;\n" + +let gen_marshall_record_field file prefix field = + let ty = get_marshall_function field.ty in + let name = String.concat "_" (List.rev (field.field_name :: prefix)) in + let name' = camel_case name in + fprintf file " record.%s = %s(map.get(\"%s\"));\n" name' ty name + +let rec gen_marshall_record_namespace file prefix (name, contents) = + List.iter (gen_marshall_record_contents file (name :: prefix)) contents + +and gen_marshall_record_contents file prefix = function + | Field f -> + gen_marshall_record_field file prefix f + | Namespace (n, cs) -> + gen_marshall_record_namespace file prefix (n, cs) ; + () + +(*Every type which may be returned by a function may also be the result of the*) +(* corresponding asynchronous task. We therefore need to generate corresponding*) +(* marshalling functions which can take the raw xml of the tasks result field*) +(* and turn it into the corresponding type. Luckily, the only things returned by*) +(* asynchronous tasks are object references and strings, so rather than implementing*) +(* the general recursive structure we'll just make one for each of the classes*) +(* that's been registered as a marshall-needing type*) + +let generate_reference_task_result_func file clstr = + fprintf file {| /** + * Attempt to convert the {@link Task}'s result to a {@link %s} object. + * Will return null if the method cannot fetch a valid value from the {@link Task} object. + * @param task The task from which to fetch the result. + * @param connection The connection + * @return the instantiated object if a valid value was found, null otherwise. + * @throws BadServerResponse Thrown if the response from the server contains an invalid status. + * @throws XenAPIException if the call failed. + * @throws IOException if an error occurs during a send or receive. This includes cases where a payload is invalid JSON. + */ +|} clstr; + fprintf file + " public static %s to%s(Task task, Connection connection) throws IOException {\n" + clstr clstr ; + fprintf file + " return Types.to%s(task.getResult(connection));\n" + clstr ; + fprintf file " }\n" ; + fprintf file "\n" + +let gen_task_result_func file = function + | Ref ty -> + generate_reference_task_result_func file (class_case ty) + | _ -> + () + +(*don't generate for complicated types. They're not needed.*) + +let rec gen_marshall_body file = function + | SecretString | String -> + fprintf file " return (String) object;\n" + | Int -> + fprintf file " return Long.valueOf((String) object);\n" + | Float -> + fprintf file " return (Double) object;\n" + | Bool -> + fprintf file " return (Boolean) object;\n" + | DateTime -> + fprintf file + " try {\n\ + \ return (Date) object;\n\ + \ } catch (ClassCastException e){\n\ + \ //Occasionally the date comes back as an ocaml float \ + rather than\n\ + \ //in the xmlrpc format! Catch this and convert.\n\ + \ return (new Date((long) (1000*Double.parseDouble((String) \ + object))));\n\ + \ }\n" + | Ref ty -> + fprintf file " return new %s((String) object);\n" (class_case ty) + | Enum (name, _) -> + fprintf file " try {\n" ; + fprintf file + " return %s.valueOf(((String) \ + object).toUpperCase().replace('-','_'));\n" + (class_case name) ; + fprintf file " } catch (IllegalArgumentException ex) {\n" ; + fprintf file " return %s.UNRECOGNIZED;\n" (class_case name) ; + fprintf file " }\n" + | Set ty -> + let ty_name = get_java_type ty in + let marshall_fn = get_marshall_function ty in + fprintf file " Object[] items = (Object[]) object;\n" ; + fprintf file " Set<%s> result = new LinkedHashSet<>();\n" ty_name ; + fprintf file " for(Object item: items) {\n" ; + fprintf file " %s typed = %s(item);\n" ty_name marshall_fn ; + fprintf file " result.add(typed);\n" ; + fprintf file " }\n" ; + fprintf file " return result;\n" + | Map (ty, ty') -> + let ty_name = get_java_type ty in + let ty_name' = get_java_type ty' in + let marshall_fn = get_marshall_function ty in + let marshall_fn' = get_marshall_function ty' in + fprintf file " var map = (Map)object;\n" ; + fprintf file " var result = new HashMap<%s,%s>();\n" ty_name + ty_name' ; + fprintf file " for(var entry: map.entrySet()) {\n" ; + fprintf file " var key = %s(entry.getKey());\n" marshall_fn ; + fprintf file " var value = %s(entry.getValue());\n" + marshall_fn' ; + fprintf file " result.put(key, value);\n" ; + fprintf file " }\n" ; + fprintf file " return result;\n" + | Record ty -> + let contents = Hashtbl.find records ty in + let cls_name = class_case ty in + fprintf file + " Map map = (Map) object;\n" ; + fprintf file " %s.Record record = new %s.Record();\n" cls_name + cls_name ; + List.iter (gen_marshall_record_contents file []) contents ; + (*Event.Record needs a special case to handle snapshots*) + if ty = "event" then generate_snapshot_hack file ; + fprintf file " return record;\n" + | Option ty -> + gen_marshall_body file ty + +let rec gen_marshall_func file ty = + match ty with + | Option x -> + if TypeSet.mem x !types then + () + else + gen_marshall_func file ty + | _ -> + let type_string = get_java_type ty in + fprintf file {| /** + * Converts an {@link Object} to a {@link %s} object. + *
+ * This method takes an {@link Object} as input and attempts to convert it into a {@link %s} object. + * If the input object is null, the method returns null. Otherwise, it creates a new {@link %s} + * object using the input object's {@link String} representation. + *
+ * @param object The {@link Object} to be converted to a {@link %s} object. + * @return A {@link %s} object created from the input {@link Object}'s {@link String} representation, + * or null if the input object is null. + * @deprecated this method will not be publicly exposed in future releases of this package. + */ + @Deprecated +|} type_string type_string type_string type_string type_string ; + let fn_name = get_marshall_function ty in + + if match ty with | Map _ | Record _ -> true | _ -> false then + fprintf file " @SuppressWarnings(\"unchecked\")\n"; + + fprintf file " public static %s %s(Object object) {\n" type_string + fn_name ; + fprintf file " if (object == null) {\n" ; + fprintf file " return null;\n" ; + fprintf file " }\n" ; + gen_marshall_body file ty ; + fprintf file " }\n\n" +(***) + let gen_enum file name ls = let name = class_case name in let ls = @@ -615,11 +832,11 @@ let gen_enum file name ls = in let json_property = if name != "UNRECOGNIZED" then - {|@JsonProperty("|} ^ name ^ {|"|} + {|@JsonProperty("|} ^ name ^ {|")|} else "@JsonEnumDefaultValue" in - comment ^ " " ^ json_property ^ "\n" ^ " " ^ enum_of_wire name + comment ^ " \n" ^ json_property ^ "\n" ^ " " ^ enum_of_wire name in fprintf file "%s" (String.concat ",\n" (List.map to_member_declaration ls)) ; fprintf file ";\n" ; @@ -700,7 +917,7 @@ let gen_types_class folder = print_license file ; fprintf file {|package com.xensource.xenapi; -import java.util.Map; +import java.util.*; import com.fasterxml.jackson.annotation.JsonEnumDefaultValue; import com.fasterxml.jackson.annotation.JsonProperty; import java.io.IOException; @@ -760,6 +977,7 @@ public class Types return sb.toString(); } } + /** * Thrown if the response from the server contains an invalid status. */ @@ -802,7 +1020,24 @@ public class Types fprintf file "\n" ; Hashtbl.iter (gen_error file) Datamodel.errors ; fprintf file "\n" ; - fprintf file "}\n" + TypeSet.iter (gen_marshall_func file) !types ; + fprintf file "\n" ; + TypeSet.iter (gen_task_result_func file) !types ; + fprintf file +{| + public static EventBatch toEventBatch(Object object) { + if (object == null) { + return null; + } + Map map = (Map) object; + EventBatch batch = new EventBatch(); + batch.token = toString(map.get("token")); + batch.validRefCounts = map.get("valid_ref_counts"); + batch.events = toSetOfEventRecord(map.get("events")); + return batch; + } +} +|} (* Now run it *) From 102c825bc5a50c14a171596391c1041ea58ec424 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 16 Jan 2024 10:37:43 +0000 Subject: [PATCH 79/99] CP-45888: Improve deprecation info Adds "@Deprecated (since = x )" annotations, and populates it for methods and fields. Also adds "@deprecated xyz" in method JavaDocs. Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/common/CommonFunctions.ml | 3 + ocaml/sdk-gen/common/CommonFunctions.mli | 6 ++ .../java/com/xensource/xenapi/EventBatch.java | 2 + ocaml/sdk-gen/java/main.ml | 79 +++++++++++++------ 4 files changed, 64 insertions(+), 26 deletions(-) diff --git a/ocaml/sdk-gen/common/CommonFunctions.ml b/ocaml/sdk-gen/common/CommonFunctions.ml index fe89dd9600e..12ef3420d31 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.ml +++ b/ocaml/sdk-gen/common/CommonFunctions.ml @@ -122,6 +122,9 @@ let get_prototyped_release lifecycle = let get_published_release lifecycle = lifecycle_matcher Lifecycle.Published lifecycle +let get_deprecated_release lifecycle = + lifecycle_matcher Lifecycle.Published lifecycle + let get_release_branding codename = try let found = diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli index 2b2312fafb7..9a88b5cd5bd 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.mli +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -4,6 +4,12 @@ exception Unknown_wire_protocol (** Type representing supported protocols. *) type wireProtocol = XmlRpc | JsonRpc +val get_deprecated_release : + (Datamodel_types.Lifecycle.change * string * 'a) list -> string +(** [get_deprecated_release codename] Gets the non-branded release name for a lifecycle if it's deprecated + @param lifecycle The lifecycle transitions to check. + @return The non-branded release name for a lifecycle if it's deprecated, empty if not deprecated *) + val get_release_branding : string -> string (** [get_release_branding codename] Gets the branding for a release codename. @param codename Release codename to lookup. diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/EventBatch.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/EventBatch.java index 2734ca18840..e70823f2638 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/EventBatch.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/EventBatch.java @@ -30,6 +30,7 @@ package com.xensource.xenapi; import java.util.Set; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Class used to map the output of Event.from(). @@ -46,6 +47,7 @@ public class EventBatch /** * The number of valid objects of all types in the database. */ + @JsonProperty("valid_ref_counts") public Object validRefCounts; /** diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 254ce1c35f8..496de17a075 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -204,14 +204,20 @@ let get_java_type_or_void = function (* it has a self parameter or not.*) (*Similar functions for deprecation of methods*) -let get_method_deprecated message = - message.msg_release.internal_deprecated_since <> None -let get_method_deprecated_string message = - if get_method_deprecated message then - "@Deprecated" - else - "" +let get_method_deprecated_release_name message = + match message.msg_release.internal_deprecated_since with + | Some version -> + Some (get_release_branding version) + | None -> + None + +let get_method_deprecated_annotation message = + match get_method_deprecated_release_name message with + | Some version -> + {|@Deprecated(since = "|} ^ version ^ {|")|} + | None -> + "" let get_method_param {param_type= ty; param_name= name; _} = let ty = get_java_type ty in @@ -244,7 +250,6 @@ let rec range = function 0 -> [] | i -> range (i - 1) @ [i] (* Here is the main method generating function.*) let gen_method file cls message params async_version = - let deprecated_string = get_method_deprecated_string message in let return_type = if String.lowercase_ascii cls.name = "event" @@ -276,7 +281,14 @@ let gen_method file cls message params async_version = fprintf file " * Minimum allowed role: %s\n" (get_minimum_allowed_role message) ; if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; - if get_method_deprecated message then fprintf file " * @deprecated\n" ; + let deprecated_info = + match get_method_deprecated_release_name message with + | Some version -> + " * @deprecated since " ^ version ^ "\n" + | None -> + "" + in + fprintf file "%s" deprecated_info ; fprintf file " *\n" ; fprintf file " * @param c The connection the call is made on\n" ; @@ -313,13 +325,20 @@ let gen_method file cls message params async_version = ) message.msg_errors ; - fprintf file " */\n" ; + fprintf file " */\n" ; + let deprecated_string = + match get_method_deprecated_annotation message with + | "" -> + "" + | other -> + " " ^ other ^ "\n" + in if async_version then - fprintf file " %s public %sTask %sAsync(%s) throws\n" deprecated_string + fprintf file "%s public %sTask %sAsync(%s) throws\n" deprecated_string method_static method_name paramString else - fprintf file " %s public %s%s %s(%s) throws\n" deprecated_string + fprintf file "%s public %s%s %s(%s) throws\n" deprecated_string method_static return_type method_name paramString ; let all_errors = @@ -415,7 +434,12 @@ let gen_record_field file prefix field cls = if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; fprintf file " */\n" ; fprintf file " @JsonProperty(\"%s\")\n" full_name ; - fprintf file " public %s %s;\n" ty name + + if field.lifecycle.state = Lifecycle.Deprecated_s then + fprintf file " @Deprecated(since = \"%s\")\n" + (get_release_branding (get_deprecated_release field.lifecycle.transitions)) ; + + fprintf file " public %s %s;\n\n" ty name let rec gen_record_namespace file prefix (name, contents) cls = List.iter (gen_record_contents file (name :: prefix) cls) contents @@ -678,7 +702,8 @@ and gen_marshall_record_contents file prefix = function (* that's been registered as a marshall-needing type*) let generate_reference_task_result_func file clstr = - fprintf file {| /** + fprintf file + {| /** * Attempt to convert the {@link Task}'s result to a {@link %s} object. * Will return null if the method cannot fetch a valid value from the {@link Task} object. * @param task The task from which to fetch the result. @@ -688,13 +713,13 @@ let generate_reference_task_result_func file clstr = * @throws XenAPIException if the call failed. * @throws IOException if an error occurs during a send or receive. This includes cases where a payload is invalid JSON. */ -|} clstr; +|} + clstr ; fprintf file - " public static %s to%s(Task task, Connection connection) throws IOException {\n" + " public static %s to%s(Task task, Connection connection) throws \ + IOException {\n" clstr clstr ; - fprintf file - " return Types.to%s(task.getResult(connection));\n" - clstr ; + fprintf file " return Types.to%s(task.getResult(connection));\n" clstr ; fprintf file " }\n" ; fprintf file "\n" @@ -785,7 +810,8 @@ let rec gen_marshall_func file ty = gen_marshall_func file ty | _ -> let type_string = get_java_type ty in - fprintf file {| /** + fprintf file + {| /** * Converts an {@link Object} to a {@link %s} object. *
* This method takes an {@link Object} as input and attempts to convert it into a {@link %s} object. @@ -798,12 +824,13 @@ let rec gen_marshall_func file ty = * @deprecated this method will not be publicly exposed in future releases of this package. */ @Deprecated -|} type_string type_string type_string type_string type_string ; +|} + type_string type_string type_string type_string type_string ; let fn_name = get_marshall_function ty in - if match ty with | Map _ | Record _ -> true | _ -> false then - fprintf file " @SuppressWarnings(\"unchecked\")\n"; - + if match ty with Map _ | Record _ -> true | _ -> false then + fprintf file " @SuppressWarnings(\"unchecked\")\n" ; + fprintf file " public static %s %s(Object object) {\n" type_string fn_name ; fprintf file " if (object == null) {\n" ; @@ -836,7 +863,7 @@ let gen_enum file name ls = else "@JsonEnumDefaultValue" in - comment ^ " \n" ^ json_property ^ "\n" ^ " " ^ enum_of_wire name + comment ^ "\n " ^ json_property ^ "\n " ^ enum_of_wire name in fprintf file "%s" (String.concat ",\n" (List.map to_member_declaration ls)) ; fprintf file ";\n" ; @@ -1024,7 +1051,7 @@ public class Types fprintf file "\n" ; TypeSet.iter (gen_task_result_func file) !types ; fprintf file -{| + {| public static EventBatch toEventBatch(Object object) { if (object == null) { return null; From fc3ac0e8deba1ac0a33346985dd88658dc289aec Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 16 Jan 2024 11:29:37 +0000 Subject: [PATCH 80/99] CP-45888: Add a custom date deserializer to handle outliers xapi returns strings of the form "0.0" for `event.timestamp` fields. This commit adds a new custom JSON deserializer to handle such values. Signed-off-by: Danilo Del Busso --- .../xenapi/CustomDateDeserializer.java | 94 +++++++++++++++++++ .../com/xensource/xenapi/JsonRpcClient.java | 7 +- 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/CustomDateDeserializer.java diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/CustomDateDeserializer.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/CustomDateDeserializer.java new file mode 100644 index 00000000000..a0e9bff1a3d --- /dev/null +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/CustomDateDeserializer.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.xensource.xenapi; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * {@link CustomDateDeserializer} is a Jackson JSON deserializer for parsing {@link Date} objects + * from custom date formats used in Xen-API responses. + */ +public class CustomDateDeserializer extends StdDeserializer { + + /** + * Array of {@link SimpleDateFormat} objects representing the custom date formats + * used in XenServer API responses. + */ + private final SimpleDateFormat[] dateFormatters + = new SimpleDateFormat[]{ + new SimpleDateFormat("yyyyMMdd'T'HH:mm:ss'Z'"), + new SimpleDateFormat("ss.SSS") + }; + + /** + * Constructs a {@link CustomDateDeserializer} instance. + */ + public CustomDateDeserializer() { + this(null); + } + + /** + * Constructs a {@link CustomDateDeserializer} instance with the specified value type. + * + * @param t The value type to handle (can be null, handled by superclass) + */ + public CustomDateDeserializer(Class t) { + super(t); + } + + /** + * Deserializes a {@link Date} object from the given JSON parser. + * + * @param jsonParser The JSON parser containing the date value to deserialize + * @param deserializationContext The deserialization context + * @return The deserialized {@link Date} object + * @throws IOException if an I/O error occurs during deserialization + */ + @Override + public Date deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + + for (SimpleDateFormat formatter : dateFormatters) { + try { + return formatter.parse(jsonParser.getText()); + } catch (ParseException e) { + // ignore + } + } + + throw new IOException("Failed to deserialize a Date value."); + } +} diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index e3513908116..38ba22db148 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -32,6 +32,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.config.RequestConfig; @@ -45,7 +46,7 @@ import java.io.IOException; import java.net.URL; -import java.text.SimpleDateFormat; +import java.util.Date; import java.util.concurrent.TimeUnit; /** @@ -209,7 +210,9 @@ protected JsonRpcResponse sendRequest(String methodCall, Object[] methodP * Helper method to initialize jackson's ObjectMapper. */ private void initializeObjectMapperConfiguration() { - this.objectMapper.setDateFormat(new SimpleDateFormat("yyyyMMdd'T'HH:mm:ss'Z'")); + var dateHandlerModule = new SimpleModule("DateHandler"); + dateHandlerModule.addDeserializer(Date.class, new CustomDateDeserializer()); + this.objectMapper.registerModule(dateHandlerModule); } /** From dc3b6d4b8c43eadfef2ba5a7e296e3a83d284921 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 8 Mar 2024 11:52:12 +0000 Subject: [PATCH 81/99] shellscripts: fix the worse complaints from shellcheck These were errors, even if they were accidentally working Signed-off-by: Pau Ruiz Safont --- ocaml/libs/http-lib/client_server_test.sh | 2 ++ ocaml/message-switch/.coverage.sh | 34 ----------------------- ocaml/sdk-gen/windows-line-endings.sh | 9 +++--- ocaml/xenopsd/scripts/vif-real | 6 ++-- scripts/install-sdk-pool | 10 +++---- scripts/test-ha-sr2 | 2 +- scripts/xe-install-supplemental-pack | 2 +- 7 files changed, 17 insertions(+), 48 deletions(-) delete mode 100644 ocaml/message-switch/.coverage.sh diff --git a/ocaml/libs/http-lib/client_server_test.sh b/ocaml/libs/http-lib/client_server_test.sh index 6757878f963..601ed257f99 100644 --- a/ocaml/libs/http-lib/client_server_test.sh +++ b/ocaml/libs/http-lib/client_server_test.sh @@ -1,3 +1,5 @@ +#!/bin/bash + set -eux trap 'kill $(jobs -p)' EXIT diff --git a/ocaml/message-switch/.coverage.sh b/ocaml/message-switch/.coverage.sh deleted file mode 100644 index 2c8f7be72b7..00000000000 --- a/ocaml/message-switch/.coverage.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -ex - -COVERAGE_DIR=.coverage -rm -rf $COVERAGE_DIR -mkdir -p $COVERAGE_DIR -pushd $COVERAGE_DIR -if [ -z "$KEEP" ]; then trap "popd; rm -rf $COVERAGE_DIR" EXIT; fi - -$(which cp) -r ../* . - -opam pin add bisect_ppx 1.3.3 -y -opam install ocveralls -y - -# install test deps -opam install message-switch-async cohttp-async -y - -export BISECT_ENABLE=YES -jbuilder runtest - -outs=($(find . | grep bisect.*.out)) -bisect-ppx-report -I $(dirname ${outs[1]}) -text report ${outs[@]} -bisect-ppx-report -I $(dirname ${outs[1]}) -summary-only -text summary ${outs[@]} -if [ -n "$HTML" ]; then bisect-ppx-report -I $(dirname ${outs[1]}) -html ../html-report ${outs[@]}; fi - -if [ -n "$TRAVIS" ]; then - echo "\$TRAVIS set; running ocveralls and sending to coveralls.io..." - ocveralls --prefix _build/default ${outs[@]} --send -else - echo "\$TRAVIS not set; displaying results of bisect-report..." - cat report - cat summary -fi diff --git a/ocaml/sdk-gen/windows-line-endings.sh b/ocaml/sdk-gen/windows-line-endings.sh index 0b11db3ba3a..f61801e1e83 100644 --- a/ocaml/sdk-gen/windows-line-endings.sh +++ b/ocaml/sdk-gen/windows-line-endings.sh @@ -1,18 +1,19 @@ +#!/bin/bash # # Copyright (c) Cloud Software Group, Inc. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: -# +# # 1) Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. -# +# # 2) Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials # provided with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS diff --git a/ocaml/xenopsd/scripts/vif-real b/ocaml/xenopsd/scripts/vif-real index 848104ca76d..25662c9bf02 100755 --- a/ocaml/xenopsd/scripts/vif-real +++ b/ocaml/xenopsd/scripts/vif-real @@ -70,7 +70,7 @@ handle_promiscuous() if [ $? -eq 0 -a -n "${arg}" ] ; then case $NETWORK_MODE in bridge) - case "${arg}" in + case "${arg}" in true|on) echo 1 > /sys/class/net/${dev}/brport/promisc ;; *) echo 0 > /sys/class/net/${dev}/brport/promisc ;; esac @@ -198,7 +198,7 @@ PRIVATE=/xapi/${DOMUUID}/private/vif/${DEVID} HOTPLUG_STATUS="${XENBUS_PATH}/hotplug-status" HOTPLUG_ERROR="${XENBUS_PATH}/hotplug-error" -NETWORK_MODE=bridge +NETWORK_MODE=bridge if [ -e /sys/module/openvswitch ]; then NETWORK_MODE=openvswitch fi @@ -221,7 +221,7 @@ if [ -n "${UDEV_CALL}" ] && \ exit 0 fi -logger -t scripts-vif "Called as \"$@\" domid:$DOMID devid:$DEVID mode:$NETWORK_MODE" +logger -t scripts-vif "Called as \"$*\" domid:$DOMID devid:$DEVID mode:$NETWORK_MODE" case "${ACTION}" in online) if [ "${TYPE}" = "vif" ] ; then diff --git a/scripts/install-sdk-pool b/scripts/install-sdk-pool index 4cc5c6809ac..fedce55cfd4 100755 --- a/scripts/install-sdk-pool +++ b/scripts/install-sdk-pool @@ -49,7 +49,7 @@ echo "* starting VMS" echo " starting: template ${VMTEMPLATE}" ${XE} vm-start uuid=${VMTEMPLATE} -for vm in ${vms[@]} +for vm in "${vms[@]}" do VM=${vms[$i]} echo " starting: $vm" @@ -69,11 +69,11 @@ ${XEBIN} -s ${masterip} -p ${port} -u root -pw "xensource" ${NOSSL} \ host-license-add license-file="${license}" # get all ips and update a license to it -for vm in ${vms[@]} +for vm in "${vms[@]}" do ip=`${XE} vm-param-get --minimal uuid="${vm}" param-name=networks | awk '{print $2}'` echo " Applying license to ${vm} at IP (${ip})" - + ${XEBIN} -s ${ip} -p ${port} -u root -pw "xensource" ${NOSSL} \ host-license-add license-file="${license}" done @@ -82,13 +82,13 @@ sleep 40 echo "* Pooling all VMs to $masterip" -for vm in ${vms[@]} +for vm in "${vms[@]}" do ip=`${XE} vm-param-get --minimal uuid="${vm}" param-name=networks | awk '{print $2}'` echo " Pooling slave VM ${vm} to (${masterip})" ${XEBIN} -s ${ip} -p ${port} -u root -pw "xensource" ${NOSSL} \ pool-join master-address=${masterip} \ - master-username=root master-password=xensource master + master-username=root master-password=xensource master done diff --git a/scripts/test-ha-sr2 b/scripts/test-ha-sr2 index 2f35bf844ad..fc2f95980fe 100755 --- a/scripts/test-ha-sr2 +++ b/scripts/test-ha-sr2 @@ -72,7 +72,7 @@ echo -n "Waiting for Survival Rule 1: " wait_for_sr 1 echo OK -while [ /bin/true ]; do +while true; do # Block statefile everywhere remote_exec_all "iptables -I OUTPUT -p tcp --dport 3260 -j DROP" diff --git a/scripts/xe-install-supplemental-pack b/scripts/xe-install-supplemental-pack index 97f93a55188..3d830efdf9a 100755 --- a/scripts/xe-install-supplemental-pack +++ b/scripts/xe-install-supplemental-pack @@ -4,7 +4,7 @@ set -e die() { - echo $@ >&2 + echo "$@" >&2 exit 1 } From 90e4c3ca845544522c813f6d62001a72600e717e Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Fri, 15 Dec 2023 21:14:09 +0000 Subject: [PATCH 82/99] The C# project files do not need to be rendered from mustache templates any more. Signed-off-by: Konstantina Chremmou --- .../src/XenServer.csproj} | 0 ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 12 ------------ 2 files changed, 12 deletions(-) rename ocaml/sdk-gen/csharp/{templates/XenServer.csproj.mustache => autogen/src/XenServer.csproj} (100%) diff --git a/ocaml/sdk-gen/csharp/templates/XenServer.csproj.mustache b/ocaml/sdk-gen/csharp/autogen/src/XenServer.csproj similarity index 100% rename from ocaml/sdk-gen/csharp/templates/XenServer.csproj.mustache rename to ocaml/sdk-gen/csharp/autogen/src/XenServer.csproj diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index 0c70b85c6d6..d216de9e0e5 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -89,18 +89,6 @@ let rec main () = ("HTTP_actions.mustache", "HTTP_actions.cs") (gen_http_actions ()) templdir destdir ; gen_relations () ; - let sorted_members = List.sort String.compare !api_members in - let json = - `O - [ - ( "api_members" - , `A (List.map (fun x -> `O [("api_member", `String x)]) sorted_members) - ) - ] - in - render_file - ("XenServer.csproj.mustache", "XenServer.csproj") - json templdir destdir ; render_file ("ApiVersion.mustache", "ApiVersion.cs") json_releases templdir destdir From 04efef123f90c0fc251e0c1476d68e75ad4492b1 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Sat, 16 Dec 2023 00:55:59 +0000 Subject: [PATCH 83/99] Replaced the generation code of a few more files with mustache templates. Signed-off-by: Konstantina Chremmou --- ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 178 +++++------- ocaml/sdk-gen/csharp/templates/Maps.mustache | 64 +++++ .../csharp/templates/Relation.mustache | 64 +++++ .../powershell/gen_powershell_binding.ml | 271 +++++------------- .../templates/ConvertTo-XenRef.mustache | 63 ++++ .../powershell/templates/HttpAction.mustache | 80 ++++++ 6 files changed, 420 insertions(+), 300 deletions(-) create mode 100644 ocaml/sdk-gen/csharp/templates/Maps.mustache create mode 100644 ocaml/sdk-gen/csharp/templates/Relation.mustache create mode 100644 ocaml/sdk-gen/powershell/templates/ConvertTo-XenRef.mustache create mode 100644 ocaml/sdk-gen/powershell/templates/HttpAction.mustache diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index d216de9e0e5..56243c59fcd 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -97,33 +97,42 @@ let rec main () = and relations = Hashtbl.create 10 and gen_relations () = - let out_chan = open_out (Filename.concat destdir "Relation.cs") in - let print format = fprintf out_chan format in List.iter process_relations (relations_of_api api) ; - print - "%s\n\n\ - using System;\n\ - using System.Collections.Generic;\n\n\ - namespace XenAPI\n\ - {\n\ - \ public partial class Relation\n\ - \ {\n\ - \ public readonly String field;\n\ - \ public readonly String manyType;\n\ - \ public readonly String manyField;\n\n\ - \ public Relation(String field, String manyType, String manyField)\n\ - \ {\n\ - \ this.field = field;\n\ - \ this.manyField = manyField;\n\ - \ this.manyType = manyType;\n\ - \ }\n\n\ - \ public static Dictionary GetRelations()\n\ - \ {\n\ - \ Dictionary relations = new Dictionary();\n\n" - Licence.bsd_two_clause ; - Hashtbl.iter (gen_relations_by_type out_chan) relations ; - print "\n return relations;\n }\n }\n}\n" + let typelist = + List.rev (Hashtbl.fold (fun k v acc -> (k, v) :: acc) relations []) + in + let json = + `O + [ + ( "types" + , `A + (List.map + (fun (k, v) -> + `O + [ + ("type", `String (exposed_class_name k)) + ; ( "relations" + , `A + (List.map + (fun (x, y, z) -> + `O + [ + ("field", `String x) + ; ("manyType", `String y) + ; ("manyField", `String z) + ] + ) + v + ) + ) + ] + ) + typelist + ) + ) + ] + in + render_file ("Relation.mustache", "Relation.cs") json templdir destdir and process_relations ((oneClass, oneField), (manyClass, manyField)) = let value = @@ -132,20 +141,6 @@ and process_relations ((oneClass, oneField), (manyClass, manyField)) = in Hashtbl.replace relations manyClass value -and gen_relations_by_type out_chan manyClass relations = - let print format = fprintf out_chan format in - print " relations.Add(typeof(%s), new Relation[] {\n" - (exposed_class_name manyClass) ; - - List.iter (gen_relation out_chan) relations ; - - print " });\n\n" - -and gen_relation out_chan (manyField, oneClass, oneField) = - let print format = fprintf out_chan format in - print " new Relation(\"%s\", \"%s\", \"%s\"),\n" manyField - oneClass oneField - (* ------------------- category: http_actions *) and gen_http_actions () = (* Each action has: @@ -829,70 +824,44 @@ and gen_enum' name contents = (* ------------------- category: maps *) and gen_maps () = - let out_chan = open_out (Filename.concat destdir "Maps.cs") in - Fun.protect - (fun () -> gen_maps' out_chan) - ~finally:(fun () -> close_out out_chan) - -and gen_maps' out_chan = - let print format = fprintf out_chan format in - - print - "%s\n\n\ - using System;\n\ - using System.Collections;\n\ - using System.Collections.Generic;\n\n\ - \ namespace XenAPI\n\ - {\n\ - \ internal class Maps\n\ - \ {" Licence.bsd_two_clause ; - - TypeSet.iter (gen_map_conversion out_chan) !maps ; - - print "\n }\n}\n" - -and gen_map_conversion out_chan = function - | Map (l, r) -> - let print format = fprintf out_chan format in - let el = exposed_type l in - let el_literal = exposed_type_as_literal l in - let er = exposed_type r in - let er_literal = exposed_type_as_literal r in - - print - "\n\ - \ internal static Dictionary<%s, %s> \ - ToDictionary_%s_%s(Hashtable table)\n\ - \ {\n\ - \ Dictionary<%s, %s> result = new Dictionary<%s, %s>();\n\ - \ if (table != null)\n\ - \ {\n\ - \ foreach (string key in table.Keys)\n\ - \ {\n\ - \ try\n\ - \ {\n\ - \ %s k = %s;\n\ - \ %s v = %s;\n\ - \ result[k] = v;\n\ - \ }\n\ - \ catch\n\ - \ {\n\ - \ // continue\n\ - \ }\n\ - \ }\n\ - \ }\n\ - \ return result;\n\ - \ }\n\n" - el er - (sanitise_function_name el_literal) - (sanitise_function_name er_literal) - el er el er el - (simple_convert_from_proxy "key" l) - er - (convert_from_proxy_hashtable_value "table[key]" r) - (***) - | _ -> - assert false + let mapList = List.rev (TypeSet.fold (fun x acc -> x :: acc) !maps []) in + let json = + `O + [ + ( "all_maps" + , `A + (List.map + (function + | Map (l, r) -> + `O + [ + ("map_key", `String (exposed_type l)) + ; ("map_value", `String (exposed_type r)) + ; ( "sanitised_key" + , `String + (sanitise_function_name (exposed_type_as_literal l)) + ) + ; ( "sanitised_value" + , `String + (sanitise_function_name (exposed_type_as_literal r)) + ) + ; ( "proxy_key" + , `String (simple_convert_from_proxy "key" l) + ) + ; ( "proxy_value" + , `String + (convert_from_proxy_hashtable_value "table[key]" r) + ) + ] + | _ -> + `Null + ) + mapList + ) + ) + ] + in + render_file ("Maps.mustache", "Maps.cs") json templdir destdir (* ------------------- category: utility *) and exposed_type_opt = function @@ -979,7 +948,6 @@ and convert_from_proxy_hashtable_value thing ty = convert_from_proxy thing ty and convert_from_proxy thing ty = - (*function*) match ty with | DateTime -> thing diff --git a/ocaml/sdk-gen/csharp/templates/Maps.mustache b/ocaml/sdk-gen/csharp/templates/Maps.mustache new file mode 100644 index 00000000000..b8942e88731 --- /dev/null +++ b/ocaml/sdk-gen/csharp/templates/Maps.mustache @@ -0,0 +1,64 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace XenAPI +{ + internal class Maps + { +{{#all_maps}} + internal static Dictionary<{{{map_key}}}, {{{map_value}}}> ToDictionary_{{{sanitised_key}}}_{{{sanitised_value}}}(Hashtable table) + { + var result = new Dictionary<{{{map_key}}}, {{{map_value}}}>(); + if (table != null) + { + foreach (string key in table.Keys) + { + try + { + {{{map_key}}} k = {{{proxy_key}}}; + {{{map_value}}} v = {{{proxy_value}}}; + result[k] = v; + } + catch + { + // continue + } + } + } + return result; + } + +{{/all_maps}} + } +} diff --git a/ocaml/sdk-gen/csharp/templates/Relation.mustache b/ocaml/sdk-gen/csharp/templates/Relation.mustache new file mode 100644 index 00000000000..69f3cd8c834 --- /dev/null +++ b/ocaml/sdk-gen/csharp/templates/Relation.mustache @@ -0,0 +1,64 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +using System; +using System.Collections.Generic; + +namespace XenAPI +{ + public partial class Relation + { + public readonly String field; + public readonly String manyType; + public readonly String manyField; + + public Relation(String field, String manyType, String manyField) + { + this.field = field; + this.manyField = manyField; + this.manyType = manyType; + } + + public static Dictionary GetRelations() + { + Dictionary relations = new Dictionary(); +{{#types}} + + relations.Add(typeof({{type}}), new Relation[] { + {{#relations}} + new Relation("{{field}}", "{{manyType}}", "{{manyField}}"), + {{/relations}} + }); +{{/types}} + + return relations; + } + } +} diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 26b94fba23e..3f49c7deff8 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -18,6 +18,7 @@ module TypeSet = Set.Make (struct end) let destdir = "autogen/src" +let templdir = "templates" type cmdlet = {filename: string; content: string} @@ -67,17 +68,38 @@ let generated x = not (List.mem x.name ["blob"; "session"; "debug"; "event"; "vtpm"]) let rec main () = - gen_xenref_converters classes ; + let json = + `O + [ + ( "all_classes" + , `A + (List.map + (fun x -> + `O + [ + ("exposed_name", `String (exposed_class_name x.name)) + ; ( "var_name" + , `String (ocaml_class_to_csharp_local_var x.name) + ) + ] + ) + classes + ) + ) + ] + in + render_file + ("ConvertTo-XenRef.mustache", "ConvertTo-XenRef.cs") + json templdir destdir ; + + http_actions + |> List.filter (fun (_, (_, _, sdk, _, _, _)) -> sdk) + |> List.iter gen_http_action ; + let cmdlets = classes |> List.filter generated |> List.map gen_cmdlets |> List.concat in - let http_cmdlets = - http_actions - |> List.filter (fun (_, (_, _, sdk, _, _, _)) -> sdk) - |> List.map gen_http_action - in - let all_cmdlets = cmdlets @ http_cmdlets in - List.iter (fun x -> write_file x.filename x.content) all_cmdlets + List.iter (fun x -> write_file x.filename x.content) cmdlets (****************) (* Http actions *) @@ -87,196 +109,55 @@ and gen_http_action action = let commonVerb = get_http_action_verb name meth in let verbCategory = get_common_verb_category commonVerb in let stem = get_http_action_stem name in - let content = - sprintf - "%s\n\n\ - using System;\n\ - using System.Collections;\n\ - using System.Collections.Generic;\n\ - using System.Management.Automation;\n\ - using XenAPI;\n\n\ - namespace Citrix.XenServer.Commands\n\ - {\n\ - \ [Cmdlet(%s.%s, \"Xen%s\"%s)]\n\ - \ [OutputType(typeof(void))]\n\ - \ public class %sXen%sCommand : XenServerHttpCmdlet\n\ - \ {\n\ - \ #region Cmdlet Parameters\n\ - %s%s\n\ - \ #endregion\n\n\ - \ #region Cmdlet Methods\n\n\ - \ protected override void ProcessRecord()\n\ - \ {\n\ - \ GetSession();\n\ - %s\n\ - \ RunApiCall(() => %s);\n\ - \ }\n\n\ - \ #endregion\n\ - \ }\n\ - }\n" - Licence.bsd_two_clause verbCategory commonVerb stem - (gen_should_process_http_decl meth) - commonVerb stem - (gen_progress_tracker meth) - (gen_arg_params args) - (gen_should_process_http meth uri) - (gen_http_action_call action) + let arg_name = function + | String_query_arg x | Int64_query_arg x -> + pascal_case_rec x + | Bool_query_arg x -> + if String.lowercase_ascii x = "host" then "IsHost" else pascal_case_rec x + | Varargs_query_arg -> + "Args" in - {filename= sprintf "%s-Xen%s.cs" commonVerb stem; content} - -and gen_should_process_http_decl meth = - match meth with - | Put -> - ", SupportsShouldProcess = true" - | Get -> - ", SupportsShouldProcess = false" - | _ -> - assert false - -and gen_should_process_http meth uri = - match meth with - | Put -> - sprintf - "\n if (!ShouldProcess(\"%s\"))\n return;\n" - uri - | _ -> - "" - -and gen_progress_tracker meth = - match meth with - | Get -> - "\n\ - \ [Parameter]\n\ - \ public HTTP.DataCopiedDelegate DataCopiedDelegate { get; set; }\n" - | Put -> - "\n\ - \ [Parameter]\n\ - \ public HTTP.UpdateProgressDelegate ProgressDelegate { get; set; }\n" - | _ -> - assert false - -and gen_arg_params args = - match args with - | [] -> - "" - | hd :: tl -> - sprintf "%s%s" (gen_arg_param hd) (gen_arg_params tl) - -and gen_arg_param = function - | String_query_arg x -> - sprintf - "\n [Parameter%s]\n public string %s { get; set; }\n" - ( if String.lowercase_ascii x = "uuid" then - "(ValueFromPipelineByPropertyName = true)" - else - "" + let arg_type = function + | String_query_arg _ -> + "string" + | Int64_query_arg _ -> + "long?" + | Bool_query_arg _ -> + "bool?" + | Varargs_query_arg -> + "string[]" + in + let json = + `O + [ + ("verb_category", `String verbCategory) + ; ("common_verb", `String commonVerb) + ; ("stem", `String stem) + ; ("isPut", `Bool (meth == Put)) + ; ("isGet", `Bool (meth == Get)) + ; ("uri", `String uri) + ; ("action_name", `String name) + ; ( "args" + , `A + (List.map + (fun x -> + `O + [ + ("arg_type", `String (arg_type x)) + ; ("arg_name", `String (arg_name x)) + ; ( "from_pipeline" + , `Bool (String.lowercase_ascii (arg_name x) = "uuid") + ) + ] + ) + args + ) ) - (pascal_case_rec x) - | Int64_query_arg x -> - sprintf "\n [Parameter]\n public long? %s { get; set; }\n" - (pascal_case_rec x) - | Bool_query_arg x -> - let y = if x = "host" then "is_host" else x in - sprintf "\n [Parameter]\n public bool? %s { get; set; }\n" - (pascal_case_rec y) - | Varargs_query_arg -> - sprintf - "\n\ - \ ///

\n\ - \ /// Alternate names and values\n\ - \ ///\n\ - \ [Parameter]\n\ - \ public string[] Args { get; set; }\n" - -and gen_http_action_call (name, (meth, _, _, args, _, _)) = - let progressTracker = - match meth with - | Get -> - "DataCopiedDelegate" - | Put -> - "ProgressDelegate" - | _ -> - assert false + ] in - sprintf - "XenAPI.HTTP_actions.%s(%s,\n\ - \ CancellingDelegate, TimeoutMs, XenHost, Proxy, Path, \ - TaskRef,\n\ - \ session.opaque_ref%s)" name progressTracker - (gen_call_arg_params args) - -and gen_call_arg_params args = - match args with - | [] -> - "" - | hd :: tl -> - sprintf "%s%s" (gen_call_arg_param hd) (gen_call_arg_params tl) - -and gen_call_arg_param = function - | String_query_arg x -> - sprintf ", %s" (pascal_case_rec x) - | Int64_query_arg x -> - sprintf ", %s" (pascal_case_rec x) - | Bool_query_arg x -> - let y = if x = "host" then "is_host" else x in - sprintf ", %s" (pascal_case_rec y) - | Varargs_query_arg -> - sprintf ", Args" - -(***********************************) -(* Utility cmdlet ConvertTo-XenRef *) -(***********************************) -and gen_xenref_converters classes = - write_file "ConvertTo-XenRef.cs" (gen_body_xenref_converters classes) - -and gen_body_xenref_converters classes = - sprintf - "%s\n\n\ - using System;\n\ - using System.Collections;\n\ - using System.Collections.Generic;\n\ - using System.Management.Automation;\n\ - using XenAPI;\n\n\ - namespace Citrix.XenServer.Commands\n\ - {\n\ - \ [Cmdlet(VerbsData.ConvertTo, \"XenRef\")]\n\ - \ [OutputType(typeof(IXenObject))]\n\ - \ public class ConvertToXenRefCommand : PSCmdlet\n\ - \ {\n\ - \ #region Cmdlet Parameters\n\n\ - \ [Parameter(Mandatory = true, ValueFromPipeline = true, Position = \ - 0)]\n\ - \ public IXenObject XenObject { get; set; }\n\n\ - \ #endregion\n\n\ - \ #region Cmdlet Methods\n\n\ - \ protected override void ProcessRecord()\n\ - \ {%s\n\ - \ }\n\n\ - \ #endregion\n\n\ - \ }\n\ - }\n" - Licence.bsd_two_clause (print_converters classes) - -and print_converters classes = - match classes with - | [] -> - "" - | hd :: tl -> - sprintf - "\n\ - \ %s %s = XenObject as %s;\n\ - \ if (%s != null)\n\ - \ {\n\ - \ WriteObject(new XenRef<%s>(%s));\n\ - \ return;\n\ - \ }%s" - (qualified_class_name hd.name) - (ocaml_class_to_csharp_local_var hd.name) - (qualified_class_name hd.name) - (ocaml_class_to_csharp_local_var hd.name) - (qualified_class_name hd.name) - (ocaml_class_to_csharp_local_var hd.name) - (print_converters tl) + render_file + ("HttpAction.mustache", sprintf "%s-Xen%s.cs" commonVerb stem) + json templdir destdir (*************************) (* Autogenerated cmdlets *) diff --git a/ocaml/sdk-gen/powershell/templates/ConvertTo-XenRef.mustache b/ocaml/sdk-gen/powershell/templates/ConvertTo-XenRef.mustache new file mode 100644 index 00000000000..669704fa3d1 --- /dev/null +++ b/ocaml/sdk-gen/powershell/templates/ConvertTo-XenRef.mustache @@ -0,0 +1,63 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +using System.Management.Automation; +using XenAPI; + +namespace Citrix.XenServer.Commands +{ + [Cmdlet(VerbsData.ConvertTo, "XenRef")] + [OutputType(typeof(IXenObject))] + public class ConvertToXenRefCommand : PSCmdlet + { + #region Cmdlet Parameters + + [Parameter(Mandatory = true, ValueFromPipeline = true, Position = 0, + HelpMessage = "The API object to convert")] + public IXenObject XenObject { get; set; } + + #endregion + + #region Cmdlet Methods + + protected override void ProcessRecord() + { +{{#all_classes}} + if (XenObject is {{exposed_name}} {{var_name}}) + { + WriteObject(new XenRef<{{exposed_name}}>({{var_name}})); + return; + } +{{/all_classes}} + } + + #endregion + } +} diff --git a/ocaml/sdk-gen/powershell/templates/HttpAction.mustache b/ocaml/sdk-gen/powershell/templates/HttpAction.mustache new file mode 100644 index 00000000000..e346a68b8fe --- /dev/null +++ b/ocaml/sdk-gen/powershell/templates/HttpAction.mustache @@ -0,0 +1,80 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Management.Automation; +using XenAPI; + +namespace Citrix.XenServer.Commands +{ + [Cmdlet({{verb_category}}.{{common_verb}}, "Xen{{stem}}", SupportsShouldProcess = {{#isPut}}true{{/isPut}}{{#isGet}}false{{/isGet}})] + [OutputType(typeof(void))] + public class {{common_verb}}Xen{{stem}}Command : XenServerHttpCmdlet + { + #region Cmdlet Parameters +{{#isPut}} + + [Parameter] + public HTTP.UpdateProgressDelegate ProgressDelegate { get; set; } +{{/isPut}} +{{#isGet}} + + [Parameter] + public HTTP.DataCopiedDelegate DataCopiedDelegate { get; set; } +{{/isGet}} +{{#args}} + + [Parameter{{#from_pipeline}}(ValueFromPipelineByPropertyName = true){{/from_pipeline}}] + public {{arg_type}} {{arg_name}} { get; set; } +{{/args}} + + #endregion + + #region Cmdlet Methods + + protected override void ProcessRecord() + { + GetSession(); +{{#isPut}} + + if (!ShouldProcess("{{uri}}")) + return; +{{/isPut}} + + RunApiCall(() => HTTP_actions.{{action_name}}({{#isPut}}ProgressDelegate{{/isPut}}{{#isGet}}DataCopiedDelegate{{/isGet}}, + CancellingDelegate, TimeoutMs, XenHost, Proxy, Path, TaskRef, + session.opaque_ref{{#args}}, {{arg_name}}{{/args}})); + } + + #endregion + } +} From 64a78b4d5c05d83146860a8056f66b3d4e6a9a84 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Thu, 7 Mar 2024 15:53:10 +0100 Subject: [PATCH 84/99] GitHub Actions: Update actions/checkout to v4 to fix Node 16 warnings Signed-off-by: Bernhard Kaindl --- .github/workflows/1.249-lcm.yml | 6 +++--- .github/workflows/docs.yml | 2 +- .github/workflows/format.yml | 2 +- .github/workflows/generate-and-build-sdks.yml | 2 +- .github/workflows/hugo.yml | 2 +- .github/workflows/main.yml | 8 ++++---- .github/workflows/release.yml | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/1.249-lcm.yml b/.github/workflows/1.249-lcm.yml index 5fec8bef8d8..084750684b6 100644 --- a/.github/workflows/1.249-lcm.yml +++ b/.github/workflows/1.249-lcm.yml @@ -15,7 +15,7 @@ jobs: test: ["", "-3"] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: '1.249-lcm' @@ -30,7 +30,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: '1.249-lcm' @@ -49,7 +49,7 @@ jobs: - name: Restore opam cache id: opam-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: "~/.opam" # invalidate cache every week, gets built using a scheduled job diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5a05a5dfc81..0ceb3016ec7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Pull configuration from xs-opam run: | diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index b15173805cf..60f954ffcaa 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Pull configuration from xs-opam run: | diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 92f0f52b854..d314654adde 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup XenAPI environment uses: ./.github/workflows/setup-xapi-environment diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index c76cc3d9487..2d9bfe2bcb2 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Hugo uses: peaceiris/actions-hugo@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d38a825a23b..9db781fd626 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,14 +21,14 @@ jobs: python-version: ["2.7", "3.11"] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 # To check which files changed: origin/master..HEAD - uses: LizardByte/setup-python-action@master with: python-version: ${{matrix.python-version}} - - uses: actions/cache@v3 + - uses: actions/cache@v4 name: Setup cache for running pre-commit fast with: path: ~/.cache/pre-commit @@ -128,7 +128,7 @@ jobs: XAPI_VERSION: "v0.0.0" steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup XenAPI environment uses: ./.github/workflows/setup-xapi-environment @@ -165,7 +165,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Generate empty configuration for make to be happy run: touch config.mk diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9045949aea2..a2b07694d7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Use python uses: actions/setup-python@v4 From 8db28fdb7345bc20e3b98f2b115e6e2e947afe96 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Thu, 7 Mar 2024 16:24:18 +0100 Subject: [PATCH 85/99] Github Actions: Update up/download-artifact to v4 to fix Node 16 warnings Signed-off-by: Bernhard Kaindl --- .github/workflows/generate-and-build-sdks.yml | 20 +++++++++---------- .github/workflows/release.yml | 12 +++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 92f0f52b854..847748e429e 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -25,13 +25,13 @@ jobs: run: opam exec -- make sdk - name: Store C# SDK source - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Source_CSharp path: _build/install/default/xapi/sdk/csharp/* - name: Store PowerShell SDK source - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Source_PowerShell path: _build/install/default/xapi/sdk/powershell/* @@ -49,7 +49,7 @@ jobs: run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve C# SDK source - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Source_CSharp path: source/ @@ -64,7 +64,7 @@ jobs: --verbosity=normal - name: Store C# SDK - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Binaries_CSharp path: source/src/bin/Release/XenServer.NET.${{ env.XAPI_VERSION_NUMBER }}-prerelease-unsigned.nupkg @@ -81,13 +81,13 @@ jobs: run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve PowerShell SDK source - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Source_PowerShell path: source/ - name: Retrieve C# SDK binaries - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_CSharp path: csharp/ @@ -135,7 +135,7 @@ jobs: ForEach-Object -Process { Copy-Item -Verbose $_.FullName -Destination "output" } - name: Store PowerShell SDK (.NET Framework 4.5) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Binaries_XenServerPowerShell_NET45 path: output/**/* @@ -155,13 +155,13 @@ jobs: run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve PowerShell SDK source - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Source_PowerShell path: source/ - name: Retrieve C# SDK binaries - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_CSharp path: csharp/ @@ -205,7 +205,7 @@ jobs: ForEach-Object -Process { Copy-Item -Verbose $_.FullName -Destination "output" } - name: Store PowerShell SDK (.NET ${{ matrix.dotnet }}) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Binaries_XenServerPowerShell_NET${{ matrix.dotnet }} path: output/**/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9045949aea2..543b3ab5a18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: make python - name: Store python distribution artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: XenAPI path: scripts/examples/python/dist/ @@ -47,25 +47,25 @@ jobs: needs: [build-python, build-sdks] steps: - name: Retrieve Python SDK distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: XenAPI path: dist/ - name: Retrieve C# SDK distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_CSharp path: dist/ - name: Retrieve PowerShell 5.x SDK distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_XenServerPowerShell_NET45 path: sdk_powershell_5x/ - name: Retrieve PowerShell 7.x SDK distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_XenServerPowerShell_NET6 path: sdk_powershell_7x/ @@ -96,7 +96,7 @@ jobs: id-token: write steps: - name: Retrieve python distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: XenAPI path: dist/ From 38d1924413fa0b600e6b70a073a2778ed469e2b9 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 8 Mar 2024 12:00:00 +0100 Subject: [PATCH 86/99] GitHub Actions: Node 16 warning fixes: update setup-dotnet to v4 Signed-off-by: Bernhard Kaindl --- .github/workflows/generate-and-build-sdks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 92f0f52b854..eddd9f1a052 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -95,7 +95,7 @@ jobs: # Following needed for restoring packages # when calling dotnet add package - name: Set up dotnet CLI (.NET 6.0 and 8.0) - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v4 with: dotnet-version: | 6 @@ -167,7 +167,7 @@ jobs: path: csharp/ - name: Set up dotnet CLI (.NET ${{ matrix.dotnet }}) - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ matrix.dotnet }} From 8ec322ab01d54a994758c81f6c8c8a02d7c34f9d Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Wed, 20 Dec 2023 22:07:44 +0000 Subject: [PATCH 87/99] Split the API reference markdown into smaller files and use templates to generate it. Signed-off-by: Konstantina Chremmou --- Makefile | 11 +- ocaml/doc/basics.md | 4 + ocaml/doc/vm-lifecycle.md | 4 + ocaml/doc/wire-protocol.md | 4 + ocaml/idl/autogen/api-ref-autogen.md | 12 + ocaml/idl/autogen/dune | 6 + ocaml/idl/datamodel.ml | 14 +- ocaml/idl/datamodel_errors.ml | 2 +- ocaml/idl/datamodel_host.ml | 2 +- ocaml/idl/datamodel_main.ml | 2 +- ocaml/idl/dune | 11 + ocaml/idl/markdown_backend.ml | 815 ++++++++++----------- ocaml/idl/templates/api_errors.mustache | 136 ++++ ocaml/idl/templates/class.mustache | 89 +++ ocaml/idl/templates/classes.mustache | 13 + ocaml/idl/templates/relationships.mustache | 19 + ocaml/idl/templates/toc.mustache | 15 + ocaml/idl/templates/types.mustache | 40 + 18 files changed, 749 insertions(+), 450 deletions(-) create mode 100644 ocaml/idl/autogen/api-ref-autogen.md create mode 100644 ocaml/idl/autogen/dune create mode 100644 ocaml/idl/templates/api_errors.mustache create mode 100644 ocaml/idl/templates/class.mustache create mode 100644 ocaml/idl/templates/classes.mustache create mode 100644 ocaml/idl/templates/relationships.mustache create mode 100644 ocaml/idl/templates/toc.mustache create mode 100644 ocaml/idl/templates/types.mustache diff --git a/Makefile b/Makefile index f272cd8e766..991ce87c812 100644 --- a/Makefile +++ b/Makefile @@ -74,15 +74,20 @@ schema: dune runtest ocaml/idl doc: - dune build --profile=$(PROFILE) ocaml/idl/datamodel_main.exe +#html dune build --profile=$(PROFILE) -f @ocaml/doc/jsapigen mkdir -p $(XAPIDOC)/html cp -r _build/default/ocaml/doc/api $(XAPIDOC)/html cp _build/default/ocaml/doc/branding.js $(XAPIDOC)/html cp ocaml/doc/*.js ocaml/doc/*.html ocaml/doc/*.css $(XAPIDOC)/html - dune exec --profile=$(PROFILE) -- ocaml/idl/datamodel_main.exe -closed -markdown $(XAPIDOC)/markdown - cp ocaml/doc/*.dot ocaml/doc/doc-convert.sh $(XAPIDOC) +#markdown + dune build --profile=$(PROFILE) -f @ocaml/idl/markdowngen + mkdir -p $(XAPIDOC)/markdown + cp -r _build/default/ocaml/idl/autogen/*.md $(XAPIDOC)/markdown + cp -r _build/default/ocaml/idl/autogen/*.yml $(XAPIDOC)/markdown find ocaml/doc -name "*.md" -not -name "README.md" -exec cp {} $(XAPIDOC)/markdown/ \; +#other + cp ocaml/doc/*.dot ocaml/doc/doc-convert.sh $(XAPIDOC) # Build manpages, networkd generated these dune build --profile=$(PROFILE) -f @man diff --git a/ocaml/doc/basics.md b/ocaml/doc/basics.md index f659a795a14..9a75a16087b 100644 --- a/ocaml/doc/basics.md +++ b/ocaml/doc/basics.md @@ -1,3 +1,7 @@ +--- + layout: doc +--- + # API Basics This document defines the XenServer Management API - an interface for remotely diff --git a/ocaml/doc/vm-lifecycle.md b/ocaml/doc/vm-lifecycle.md index 31f31889f36..68664730617 100644 --- a/ocaml/doc/vm-lifecycle.md +++ b/ocaml/doc/vm-lifecycle.md @@ -1,3 +1,7 @@ +--- + layout: doc +--- + # VM Lifecycle The following diagram shows the states that a VM can be in diff --git a/ocaml/doc/wire-protocol.md b/ocaml/doc/wire-protocol.md index 260039ab495..e42a0f2da2c 100644 --- a/ocaml/doc/wire-protocol.md +++ b/ocaml/doc/wire-protocol.md @@ -1,3 +1,7 @@ +--- + layout: doc +--- + # Wire Protocol for Remote API Calls API calls are sent over a network to a Xen-enabled host using an RPC protocol. diff --git a/ocaml/idl/autogen/api-ref-autogen.md b/ocaml/idl/autogen/api-ref-autogen.md new file mode 100644 index 00000000000..7abf22b6872 --- /dev/null +++ b/ocaml/idl/autogen/api-ref-autogen.md @@ -0,0 +1,12 @@ +--- + layout: doc +--- + +# API Reference + +Version **@xapi-version@** + +- [Classes](@root@management-api/classes.html) +- [Relationships Between Classes](@root@management-api/relationships-between-classes.html) +- [Types](@root@management-api/types.html) +- [ErrorHandling](@root@management-api/api-ref-autogen-errors.html) diff --git a/ocaml/idl/autogen/dune b/ocaml/idl/autogen/dune new file mode 100644 index 00000000000..483a0dbdef8 --- /dev/null +++ b/ocaml/idl/autogen/dune @@ -0,0 +1,6 @@ +(alias + (name markdowngen) + (deps + (source_tree .) + ) +) \ No newline at end of file diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index 15f1c4c66c6..d7bff0266a9 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -5985,7 +5985,7 @@ module DR_task = struct ) ; (Set String, "whitelist", "The devices to use for disaster recovery") ] - ~result:(Ref _dr_task, "The reference to the created task") + ~result:(Ref _dr_task, "The reference of the created DR_task") ~doc: "Create a disaster recovery task which will query the supplied list of \ devices" @@ -6202,7 +6202,7 @@ module Blob = struct } ] ~doc:"Create a placeholder for a binary blob" ~flags:[`Session] - ~result:(Ref _blob, "The reference to the created blob") + ~result:(Ref _blob, "The reference of the created blob") ~allowed_roles:_R_POOL_OP () let destroy = @@ -6889,7 +6889,8 @@ module GPU_group = struct ; param_default= Some (VMap []) } ] - ~result:(Ref _gpu_group, "") ~allowed_roles:_R_POOL_OP () + ~result:(Ref _gpu_group, "The reference of the created GPU_group") + ~allowed_roles:_R_POOL_OP () let destroy = call ~name:"destroy" @@ -7041,7 +7042,7 @@ module VGPU = struct ; param_default= Some (VRef null_ref) } ] - ~result:(Ref _vgpu, "reference to the newly created object") + ~result:(Ref _vgpu, "The reference of the created VGPU object") ~allowed_roles:_R_POOL_OP () let destroy = @@ -7356,7 +7357,7 @@ module PVS_proxy = struct let create = call ~name:"create" ~doc:"Configure a VM/VIF to use a PVS proxy" - ~result:(Ref _pvs_proxy, "the new PVS proxy") + ~result:(Ref _pvs_proxy, "The reference of the created PVS proxy") ~params: [ (Ref _pvs_site, "site", "PVS site that we proxy for") @@ -7626,7 +7627,8 @@ module USB_group = struct ; param_default= Some (VMap []) } ] - ~result:(Ref _usb_group, "") ~allowed_roles:_R_POOL_ADMIN () + ~result:(Ref _usb_group, "The reference of the created USB_group") + ~allowed_roles:_R_POOL_ADMIN () let destroy = call ~name:"destroy" ~lifecycle diff --git a/ocaml/idl/datamodel_errors.ml b/ocaml/idl/datamodel_errors.ml index 0a0166dfe93..0bfbaa21039 100644 --- a/ocaml/idl/datamodel_errors.ml +++ b/ocaml/idl/datamodel_errors.ml @@ -1455,7 +1455,7 @@ let _ = ~doc: "The requested update could not be found. Please upload the update \ again. This can occur when you run xe update-pool-clean before xe \ - update-apply. " + update-apply." () ; error Api_errors.update_pool_apply_failed ["hosts"] ~doc:"The update cannot be applied for the following host(s)." () ; diff --git a/ocaml/idl/datamodel_host.ml b/ocaml/idl/datamodel_host.ml index 6c7895ec901..2f9d1d7ed83 100644 --- a/ocaml/idl/datamodel_host.ml +++ b/ocaml/idl/datamodel_host.ml @@ -1369,7 +1369,7 @@ let set_power_on_mode = ; (Changed, rel_stockholm, "Removed iLO script") ] ~in_product_since:rel_midnight_ride - ~doc:"Set the power-on-mode, host, user and password " + ~doc:"Set the power-on-mode, host, user and password" ~params: [ (Ref _host, "self", "The host") diff --git a/ocaml/idl/datamodel_main.ml b/ocaml/idl/datamodel_main.ml index 77250738817..fa22d3b9d09 100644 --- a/ocaml/idl/datamodel_main.ml +++ b/ocaml/idl/datamodel_main.ml @@ -86,7 +86,7 @@ let _ = in if !markdown_mode then - Markdown_backend.all api !dirname ; + Markdown_backend.all api ; if !dirname <> "" then Unix.chdir !dirname ; if !dot_mode then diff --git a/ocaml/idl/dune b/ocaml/idl/dune index 3dfa75af8c4..713462e7ffa 100644 --- a/ocaml/idl/dune +++ b/ocaml/idl/dune @@ -30,6 +30,7 @@ (modules datamodel_main dot_backend dtd_backend markdown_backend) (libraries dune-build-info + mustache xapi-datamodel xapi-stdext-std xapi-stdext-pervasives @@ -37,6 +38,16 @@ ) ) +(rule + (alias markdowngen) + (deps + (:x datamodel_main.exe) + (source_tree templates) + ) + (package xapi-datamodel) + (action (run %{x} -closed -markdown)) +) + (test (name schematest) (modes exe) diff --git a/ocaml/idl/markdown_backend.ml b/ocaml/idl/markdown_backend.ml index edd95d95d50..c4bbd538fe1 100644 --- a/ocaml/idl/markdown_backend.ml +++ b/ocaml/idl/markdown_backend.ml @@ -15,7 +15,7 @@ open Printf open Datamodel_types open Datamodel_utils open Dm_api -open Xapi_stdext_pervasives.Pervasiveext +module Unixext = Xapi_stdext_unix.Unixext (*column widths for the autogenerated tables*) let col_width_15 = 15 @@ -28,6 +28,10 @@ let col_width_40 = 40 let col_width_70 = 70 +let destdir = "autogen" + +let templatesdir = "templates" + let pad_right x max_width = let length = String.length x in if String.length x < max_width then @@ -78,14 +82,6 @@ let escape s = let escaped_list = List.map esc_char sl in String.concat "" escaped_list -let is_prim_type = function - | String | Int | Float | Bool | DateTime -> - true - | _ -> - false - -let is_prim_opt_type = function None -> true | Some (ty, _) -> is_prim_type ty - let rec of_ty_verbatim = function | SecretString | String -> "string" @@ -152,231 +148,383 @@ let string_of_qualifier = function | RW -> "_RW_" -let is_removal_marker x = - match x with Lifecycle.Removed, _, _ -> true | _ -> false +let render_file (infile, outfile) json templates_dir dest_dir = + let templ = + Unixext.string_of_file (Filename.concat templates_dir infile) + |> Mustache.of_string + in + let rendered = Mustache.render templ json in + let io = open_out (Filename.concat dest_dir outfile) in + Fun.protect + (fun () -> output_string io rendered) + ~finally:(fun () -> close_out io) -let is_deprecation_marker x = - match x with Lifecycle.Deprecated, _, _ -> true | _ -> false +let generate_class cls = + let class_json = + `O + [ + ("class_name", `String (escape cls.name)) + ; ("class_descr", `String (escape cls.description)) + ; ("has_descr", `Bool (cls.description <> "")) + ; ("class_deprecated", `Bool (cls.obj_lifecycle.state = Deprecated_s)) + ; ("class_removed", `Bool (cls.obj_lifecycle.state = Removed_s)) + ; ("is_event", `Bool (String.lowercase_ascii cls.name = "event")) + ; ("has_fields", `Bool (Datamodel_utils.fields_of_obj cls <> [])) + ; ( "fields" + , `A + (cls + |> Datamodel_utils.fields_of_obj + |> List.sort (fun x y -> + compare_case_ins + (Datamodel_utils.wire_name_of_field x) + (Datamodel_utils.wire_name_of_field y) + ) + |> List.map (fun field -> + `O + [ + ( "field_name" + , `String + (pad_right + (escape (Datamodel_utils.wire_name_of_field field)) + col_width_20 + ) + ) + ; ( "field_type" + , `String + (pad_right + ("`" ^ of_ty_verbatim field.ty ^ "`") + col_width_20 + ) + ) + ; ( "field_ctor" + , `String + (pad_right + (string_of_qualifier field.qualifier) + col_width_15 + ) + ) + ; ( "field_descr" + , `String + (pad_right + (escape field.field_description) + col_width_40 + ) + ) + ; ( "field_deprecated" + , `Bool + (field.lifecycle.state = Deprecated_s + || cls.obj_lifecycle.state = Deprecated_s + ) + ) + ; ( "field_removed" + , `Bool + (field.lifecycle.state = Removed_s + || cls.obj_lifecycle.state = Removed_s + ) + ) + ] + ) + ) + ) + ; ("has_rpcs", `Bool (cls.messages <> [])) + ; ( "all_rpcs" + , `A + (cls.messages + |> List.sort (fun x y -> compare_case_ins x.msg_name y.msg_name) + |> List.map (fun msg -> + let is_event_from = + String.lowercase_ascii cls.name = "event" + && String.lowercase_ascii msg.msg_name = "from" + in + let rpc_param_csv = + msg.msg_params + |> List.map (fun p -> + of_ty_verbatim p.param_type ^ " " ^ p.param_name + ) + |> String.concat ", " + in + let error_codes_csv = + msg.msg_errors + |> List.map (fun x -> sprintf "`%s`" x.err_name) + |> String.concat ", " + in + let rbac x = + match x.msg_allowed_roles with + | Some y when y <> [] -> + List.hd (List.rev y) + | _ -> + "" + in + `O + [ + ("rpc_name_escaped", `String (escape msg.msg_name)) + ; ("rpc_name", `String msg.msg_name) + ; ("rpc_descr", `String (escape msg.msg_doc)) + ; ("rpc_has_descr", `Bool (msg.msg_doc <> "")) + ; ( "rpc_deprecated" + , `Bool + (msg.msg_lifecycle.state = Lifecycle.Deprecated_s + || cls.obj_lifecycle.state = Deprecated_s + ) + ) + ; ( "rpc_removed" + , `Bool + (msg.msg_lifecycle.state = Lifecycle.Removed_s + || cls.obj_lifecycle.state = Removed_s + ) + ) + ; ("returns_void", `Bool (msg.msg_result = None)) + ; ( "return_type" + , `String + ( if is_event_from then + "event batch" + else + of_ty_opt_verbatim msg.msg_result + ) + ) + ; ( "return_descr" + , `String (escape (desc_of_ty_opt msg.msg_result)) + ) + ; ("rpc_param_csv", `String rpc_param_csv) + ; ("has_rbac", `Bool (rbac msg <> "")) + ; ("min_role", `String (rbac msg)) + ; ("session", `Bool msg.msg_session) + ; ("has_rpc_params", `Bool (msg.msg_params <> [])) + ; ( "rpc_params" + , `A + (msg.msg_params + |> List.map (fun p -> + `O + [ + ( "param_name" + , `String + (pad_right (escape p.param_name) + col_width_30 + ) + ) + ; ( "param_type" + , `String + (pad_right + ("`" + ^ of_ty_verbatim p.param_type + ^ "`" + ) + col_width_30 + ) + ) + ; ( "param_descr" + , `String + (pad_right (escape p.param_doc) + col_width_40 + ) + ) + ] + ) + ) + ) + ; ("has_error_codes", `Bool (msg.msg_errors <> [])) + ; ("error_codes_csv", `String error_codes_csv) + ] + ) + ) + ) + ] + in + render_file + ("class.mustache", sprintf "class-%s.md" (String.lowercase_ascii cls.name)) + class_json templatesdir destdir -(* Make a markdown section for an API-specified message *) -let markdown_section_of_message printer obj ~is_class_deprecated - ~is_class_removed x = - let is_event_from = - String.lowercase_ascii obj.name = "event" - && String.lowercase_ascii x.msg_name = "from" +let generate_types system = + let type_comparer x y = + match (x, y) with + | Enum (a, _), Enum (b, _) -> + compare_case_ins a b + | _ -> + compare x y in - let return_type = of_ty_opt_verbatim x.msg_result in - printer (sprintf "#### RPC name: %s" (escape x.msg_name)) ; - printer "" ; - if x.msg_lifecycle.state = Lifecycle.Removed_s || is_class_removed then ( - printer "**This message is removed.**" ; - printer "" - ) else if - x.msg_lifecycle.state = Lifecycle.Deprecated_s || is_class_deprecated - then ( - printer "**This message is deprecated.**" ; - printer "" - ) ; - printer "_Overview:_" ; - printer "" ; - printer (escape x.msg_doc) ; - printer "" ; - printer "_Signature:_" ; - printer "" ; - printer "```" ; - let result = - if is_event_from then - "" - else - of_ty_opt_verbatim x.msg_result + let enums = + Types.of_objects system + |> List.filter (function Enum (_, _) -> true | _ -> false) + |> List.sort type_comparer in - printer - (sprintf "%s %s (%s)" result x.msg_name - (String.concat ", " - ((if x.msg_session then ["session ref session_id"] else []) - @ List.map - (fun p -> of_ty_verbatim p.param_type ^ " " ^ p.param_name) - x.msg_params - ) - ) - ) ; - printer "```" ; - printer "" ; - if x.msg_params <> [] then ( - printer "_Arguments:_" ; - printer "" ; - printer - "|type |name \ - |description |" ; - printer - "|:-----------------------------|:-----------------------------|:---------------------------------------|" ; - if x.msg_session then - printer - "|session ref |session_id \ - |Reference to a valid session |" ; - let get_param_row p = - sprintf "|`%s`|%s|%s|" - (pad_right (of_ty_verbatim p.param_type) (col_width_30 - 2)) - (pad_right (escape p.param_name) col_width_30) - (pad_right (escape p.param_doc) col_width_40) - in - List.iter (fun p -> printer (get_param_row p)) x.msg_params ; - printer "" - ) ; - let print_rbac y = - match y.msg_allowed_roles with - | Some yy when yy <> [] -> - printer ("_Minimum Role:_ " ^ List.hd (List.rev yy)) ; - printer "" - | _ -> - () + let types_json = + `O + [ + ( "enums" + , `A + (List.map + (function + | Enum (name, options) -> + `O + [ + ("enum", `String (pad_right name (col_width_40 - 5))) + ; ( "enum_options" + , `A + (options + |> List.sort (fun (x, _) (y, _) -> + compare_case_ins x y + ) + |> List.map (fun (n, c) -> + `O + [ + ( "option_name" + , `String + (pad_right + ("`" ^ n ^ "`") + col_width_40 + ) + ) + ; ( "option_descr" + , `String + (pad_right (escape c) col_width_40) + ) + ] + ) + ) + ) + ] + | _ -> + `Null + ) + enums + ) + ) + ] in - print_rbac x ; - printer - ("_Return Type:_" - ^ if is_event_from then " an event batch" else sprintf " `%s`" return_type - ) ; - printer "" ; - let descr = desc_of_ty_opt x.msg_result in - if descr <> "" then ( - printer (escape descr) ; - printer "" - ) ; - if x.msg_errors <> [] then ( - let error_codes = - List.map (fun err -> sprintf "`%s`" err.err_name) x.msg_errors - in - printer - (sprintf "_Possible Error Codes:_ %s" (String.concat ", " error_codes)) ; - printer "" - ) + render_file ("types.mustache", "types.md") types_json templatesdir destdir -let print_field_table_of_obj printer ~is_class_deprecated ~is_class_removed x = - printer (sprintf "### Fields for class: " ^ escape x.name) ; - printer "" ; - if x.contents = [] then - printer ("Class " ^ escape x.name ^ " has no fields.") - else ( - printer - "|Field |Type |Qualifier \ - |Description |" ; - printer - "|:-------------------|:-------------------|:--------------|:---------------------------------------|" ; - let print_field_content printer - ({qualifier; ty; field_description= description; _} as y) = - let wired_name = Datamodel_utils.wire_name_of_field y in - let descr = - ( if y.lifecycle.state = Removed_s || is_class_removed then - "**Removed**. " - else if y.lifecycle.state = Deprecated_s || is_class_deprecated then - "**Deprecated**. " - else - "" - ) - ^ escape description - in - printer - (sprintf "|%s|`%s`|%s|%s|" - (pad_right (escape wired_name) col_width_20) - (pad_right (of_ty_verbatim ty) (col_width_20 - 2)) - (pad_right (string_of_qualifier qualifier) col_width_15) - (pad_right descr col_width_40) - ) - in - x - |> Datamodel_utils.fields_of_obj - |> List.sort (fun x y -> - compare_case_ins - (Datamodel_utils.wire_name_of_field x) - (Datamodel_utils.wire_name_of_field y) - ) - |> List.iter (print_field_content printer) ; - if String.lowercase_ascii x.name = "event" then - printer - (sprintf "|%s|`%s`|%s|%s|" - (pad_right "snapshot" col_width_20) - (pad_right "" (col_width_20 - 2)) - (pad_right "_RO/runtime_" col_width_15) - (pad_right - "The record of the database object that was added, changed or \ - deleted" - col_width_40 - ) +let generate_relationships api = + let relations = relations_of_api api in + let relationships_json = + `O + [ + ( "relationships" + , `A + (List.map + (function + | ((a, a_field), (b, b_field)) as rel -> + let c = Relations.classify api rel in + let afield = "`" ^ a ^ "." ^ a_field ^ "`" in + let bfield = "`" ^ b ^ "." ^ b_field ^ "`" in + `O + [ + ( "a_field" + , `String (pad_right afield (col_width_40 - 2)) + ) + ; ( "b_field" + , `String (pad_right bfield (col_width_40 - 2)) + ) + ; ( "relationship" + , `String + (pad_right + (Relations.string_of_classification c) + col_width_15 + ) + ) + ] + ) + relations + ) ) - ) + ] + in + render_file + ("relationships.mustache", "relationships-between-classes.md") + relationships_json templatesdir destdir -let of_obj printer x = - printer (sprintf "## Class: %s" (escape x.name)) ; - printer "" ; - let is_class_removed = x.obj_lifecycle.state = Removed_s in - let is_class_deprecated = x.obj_lifecycle.state = Deprecated_s in - if is_class_removed then ( - printer "**This class is removed.**" ; - printer "" - ) else if is_class_deprecated then ( - printer "**This class is deprecated.**" ; - printer "" - ) ; - printer (escape x.description) ; - printer "" ; - print_field_table_of_obj printer ~is_class_deprecated ~is_class_removed x ; - printer "" ; - printer (sprintf "### RPCs associated with class: " ^ escape x.name) ; - printer "" ; - if x.messages = [] then ( - printer - (sprintf "Class %s has no additional RPCs associated with it." - (escape x.name) - ) ; - printer "" - ) else - x.messages - |> List.sort (fun x y -> compare_case_ins x.msg_name y.msg_name) - |> List.iter - (markdown_section_of_message printer x ~is_class_deprecated - ~is_class_removed - ) +let generate_classes system = + let classes_json = + `O + [ + ( "classes" + , `A + (List.map + (fun x -> + let notice y = + match y.obj_lifecycle.state with + | Removed_s -> + "**Removed**. " + | Deprecated_s -> + "**Deprecated**. " + | _ -> + "" + in + `O + [ + ("name", `String x.name) + ; ("name_lower", `String (String.lowercase_ascii x.name)) + ; ( "description" + , `String + (pad_right + (notice x ^ escape x.description) + col_width_70 + ) + ) + ] + ) + system + ) + ) + ] + in + render_file + ("classes.mustache", "classes.md") + classes_json templatesdir destdir -let print_enum printer = function - | Enum (name, options) -> - printer - (sprintf "|`enum %s`| |" - (pad_right name (col_width_40 - 7)) - ) ; - printer - "|:---------------------------------------|:---------------------------------------|" ; - let print_option (opt, description) = - printer - (sprintf "|`%s`|%s|" - (pad_right opt (col_width_40 - 2)) - (pad_right (escape description) col_width_40) - ) - in - options - |> List.sort (fun (x, _) (y, _) -> compare_case_ins x y) - |> List.iter print_option ; - printer "" - | _ -> - () +let generate_toc system = + let classes_json = + `O + [ + ( "classes" + , `A + (List.map + (fun x -> + `O + [ + ("name", `String x.name) + ; ("name_lower", `String (String.lowercase_ascii x.name)) + ] + ) + system + ) + ) + ] + in + render_file ("toc.mustache", "toc.yml") classes_json templatesdir destdir -let error_doc printer {err_name= name; err_params= params; err_doc= doc} = - printer (sprintf "### %s" (escape name)) ; - printer "" ; - printer (escape doc) ; - printer "" ; - if params = [] then - printer "No parameters." - else ( - printer "_Signature:_" ; - printer "" ; - printer "```" ; - printer (sprintf "%s(%s)" name (String.concat ", " params)) ; - printer "```" - ) ; - printer "" +let generate_errors () = + (* Sort the errors alphabetically, then generate one section per code. *) + let errs = + Hashtbl.fold (fun name err acc -> (name, err) :: acc) Datamodel.errors [] + |> List.sort (fun (n1, _) (n2, _) -> compare n1 n2) + |> List.split + |> snd + in + let error_json = + `O + [ + ( "errors" + , `A + (List.map + (fun {err_name; err_params; err_doc} -> + `O + [ + ("error_code", `String (escape err_name)) + ; ("error_code_unescaped", `String err_name) + ; ("error_description", `String (escape err_doc)) + ; ("parameters", `String (String.concat ", " err_params)) + ] + ) + errs + ) + ) + ] + in + render_file + ("api_errors.mustache", "api-ref-autogen-errors.md") + error_json templatesdir destdir -let print_classes api io = - let printer text = fprintf io "%s\n" text in +let all api = (* Remove private messages that are only used internally (e.g. get_record_internal) *) let api = Dm_api.filter @@ -390,219 +538,10 @@ let print_classes api io = let system = objects_of_api api |> List.sort (fun x y -> compare_case_ins x.name y.name) in - let relations = relations_of_api api in - printer - "# API Reference - Types and Classes\n\n\ - ## Classes\n\n\ - The following classes are defined:\n\n\ - |Name \ - |Description |\n\ - |:-------------------|:---------------------------------------------------------------------|" ; - let get_descr obj = - ( if obj.obj_lifecycle.state = Removed_s then - "**Removed**. " - else if obj.obj_lifecycle.state = Deprecated_s then - "**Deprecated**. " - else - "" - ) - ^ escape obj.description - in - List.iter - (fun obj -> - printer - (sprintf "|`%s`|%s|" - (pad_right obj.name (col_width_20 - 2)) - (pad_right (get_descr obj) col_width_70) - ) - ) - system ; - printer - "\n\ - ## Relationships Between Classes\n\n\ - Fields that are bound together are shown in the following table:\n\n\ - |_object.field_ \ - |_object.field_ |_relationship_ |\n\ - |:---------------------------------------|:---------------------------------------|:--------------|" ; - List.iter - (function - | ((a, a_field), (b, b_field)) as rel -> - let c = Relations.classify api rel in - let afield = a ^ "." ^ a_field in - let bfield = b ^ "." ^ b_field in - printer - (sprintf "|`%s`|`%s`|%s|" - (pad_right afield (col_width_40 - 2)) - (pad_right bfield (col_width_40 - 2)) - (pad_right (Relations.string_of_classification c) col_width_15) - ) - ) - relations ; - printer - "\n\ - The following figure represents bound fields (as specified above) \ - diagramatically, using crow's foot notation to specify one-to-one, \ - one-to-many or many-to-many relationships:\n\n\ - ![Class relationships](classes.png 'Class relationships')\n\n\ - ## Types\n\n\ - ### Primitives\n\n\ - The following primitive types are used to specify methods and fields in \ - the API Reference:\n\n\ - |Type |Description |\n\ - |:-------|:-------------------------------------------|\n\ - |string |text strings |\n\ - |int |64-bit integers |\n\ - |float |IEEE double-precision floating-point numbers|\n\ - |bool |boolean |\n\ - |datetime|date and timestamp |\n\n\ - ### Higher-order types\n\n\ - The following type constructors are used:\n\n\ - |Type \ - |Description |\n\ - |:-----------------|:-------------------------------------------------------|\n\ - |_c_ ref |reference to an object of class \ - _c_ |\n\ - |_t_ set |a set of elements of type \ - _t_ |\n\ - |(_a -> b_) map |a table mapping values of type _a_ to values \ - of type _b_|\n\n\ - ### Enumeration types\n\n\ - The following enumeration types are used:\n" ; - let type_comparer x y = - match (x, y) with - | Enum (a, _), Enum (b, _) -> - compare_case_ins a b - | _ -> - compare x y - in - Types.of_objects system - |> List.sort type_comparer - |> List.iter (print_enum printer) ; - List.iter (fun x -> of_obj printer x) system -let print_errors io = - let printer text = fprintf io "%s\n" text in - printer - "# API Reference - Error Handling\n\n\ - When a low-level transport error occurs, or a request is malformed at the \ - HTTP\n\ - or RPC level, the server may send an HTTP 500 error response, or the client\n\ - may simulate the same. The client must be prepared to handle these errors,\n\ - though they may be treated as fatal.\n\n\ - On the wire, these are transmitted in a form similar to this when using the\n\ - XML-RPC protocol:\n\n\ - ```\n\ - $curl -D - -X POST https://server -H 'Content-Type: application/xml' \\\n\ - > -d '\n\ - > \n\ - > session.logout\n\ - > '\n\ - HTTP/1.1 500 Internal Error\n\ - content-length: 297\n\ - content-type:text/html\n\ - connection:close\n\ - cache-control:no-cache, no-store\n\n\ -

HTTP 500 internal server error

An unexpected error \ - occurred;\n\ - \ please wait a while and try again. If the problem persists, please \ - contact your\n\ - \ support representative.

Additional information \ -

Xmlrpc.Parse_error(&quo\n\ - t;close_tag", "open_tag", _)\n\ - ```\n\n\ - When using the JSON-RPC protocol:\n\n\ - ```\n\ - $curl -D - -X POST https://server/jsonrpc -H 'Content-Type: \ - application/json' \\\n\ - > -d '{\n\ - > \"jsonrpc\": \"2.0\",\n\ - > \"method\": \"session.login_with_password\",\n\ - > \"id\": 0\n\ - > }'\n\ - HTTP/1.1 500 Internal Error\n\ - content-length: 308\n\ - content-type:text/html\n\ - connection:close\n\ - cache-control:no-cache, no-store\n\n\ -

HTTP 500 internal server error

An unexpected error \ - occurred;\n\ - \ please wait a while and try again. If the problem persists, please \ - contact your\n\ - \ support representative.

Additional information \ -

Jsonrpc.Malformed_metho\n\ - d_request("{jsonrpc=...,method=...,id=...}")\n\ - ```\n\n\ - All other failures are reported with a more structured error response, to\n\ - allow better automatic response to failures, proper internationalisation of\n\ - any error message, and easier debugging.\n\n\ - On the wire, these are transmitted like this when using the XML-RPC \ - protocol:\n\n\ - ```xml\n\ - \ \n\ - \ \n\ - \ Status\n\ - \ Failure\n\ - \ \n\ - \ \n\ - \ ErrorDescription\n\ - \ \n\ - \ \n\ - \ \n\ - \ MAP_DUPLICATE_KEY\n\ - \ Customer\n\ - \ eSpiel Inc.\n\ - \ eSpiel Incorporated\n\ - \ \n\ - \ \n\ - \ \n\ - \ \n\ - \ \n\ - ```\n\n\ - Note that `ErrorDescription` value is an array of string values. The\n\ - first element of the array is an error code; the remainder of the array are\n\ - strings representing error parameters relating to that code. In this case,\n\ - the client has attempted to add the mapping _Customer ->\n\ - eSpiel Incorporated_ to a Map, but it already contains the mapping\n\ - _Customer -> eSpiel Inc._, and so the request has failed.\n\n\ - When using the JSON-RPC protocol v2.0, the above error is transmitted as:\n\n\ - ```json\n\ - {\n\ - \ \"jsonrpc\": \"2.0\",\n\ - \ \"error\": {\n\ - \ \"code\": 1,\n\ - \ \"message\": \"MAP_DUPLICATE_KEY\",\n\ - \ \"data\": [\n\ - \ \"Customer\",\"eSpiel Inc.\",\"eSpiel Incorporated\"\n\ - \ ]\n\ - \ },\n\ - \ \"id\": 3\n\ - \ }\n\ - ```\n\n\ - Finally, when using the JSON-RPC protocol v1.0:\n\n\ - ```json\n\ - {\n\ - \ \"result\": null,\n\ - \ \"error\": [\n\ - \ \"MAP_DUPLICATE_KEY\",\"Customer\",\"eSpiel Inc.\",\"eSpiel \ - Incorporated\"\n\ - \ ],\n\ - \ \"id\": \"xyz\"\n\ - }\n\ - ```\n\n\ - Each possible error code is documented in the following section.\n\n\ - ## Error Codes\n" ; - (* Sort the errors alphabetically, then generate one section per code. *) - let errs = - Hashtbl.fold (fun name err acc -> (name, err) :: acc) Datamodel.errors [] - in - List.iter (error_doc printer) - (snd (List.split (List.sort (fun (n1, _) (n2, _) -> compare n1 n2) errs))) - -let all api destdir = - Xapi_stdext_unix.Unixext.mkdir_rec destdir 0o755 ; - let with_file filename f = - let io = open_out (Filename.concat destdir filename) in - finally (fun () -> f io) (fun () -> close_out io) - in - with_file "api-ref-autogen.md" (print_classes api) ; - with_file "api-ref-autogen-errors.md" print_errors + List.iter generate_class system ; + generate_classes system ; + generate_relationships api ; + generate_types system ; + generate_errors () ; + generate_toc system diff --git a/ocaml/idl/templates/api_errors.mustache b/ocaml/idl/templates/api_errors.mustache new file mode 100644 index 00000000000..e1a4b95fbaa --- /dev/null +++ b/ocaml/idl/templates/api_errors.mustache @@ -0,0 +1,136 @@ +--- + layout: doc +--- + +# Error Handling + +When a low-level transport error occurs, or a request is malformed at the HTTP +or RPC level, the server may send an HTTP 500 error response, or the client +may simulate the same. The client must be prepared to handle these errors, +though they may be treated as fatal. + +On the wire, these are transmitted in a form similar to this when using the +XML-RPC protocol: + +``` +$curl -D - -X POST https://server -H 'Content-Type: application/xml' \ +> -d ' +> +> session.logout +> ' +HTTP/1.1 500 Internal Error +content-length: 297 +content-type:text/html +connection:close +cache-control:no-cache, no-store + +

HTTP 500 internal server error

An unexpected error occurred; + please wait a while and try again. If the problem persists, please contact your + support representative.

Additional information

Xmlrpc.Parse_error(&quo +t;close_tag", "open_tag", _) +``` + +When using the JSON-RPC protocol: + +``` +$curl -D - -X POST https://server/jsonrpc -H 'Content-Type: application/json' \ +> -d '{ +> "jsonrpc": "2.0", +> "method": "session.login_with_password", +> "id": 0 +> }' +HTTP/1.1 500 Internal Error +content-length: 308 +content-type:text/html +connection:close +cache-control:no-cache, no-store + +

HTTP 500 internal server error

An unexpected error occurred; + please wait a while and try again. If the problem persists, please contact your + support representative.

Additional information

Jsonrpc.Malformed_metho +d_request("{jsonrpc=...,method=...,id=...}") +``` + +All other failures are reported with a more structured error response, to +allow better automatic response to failures, proper internationalisation of +any error message, and easier debugging. + +On the wire, these are transmitted like this when using the XML-RPC protocol: + +```xml + + + Status + Failure + + + ErrorDescription + + + + MAP_DUPLICATE_KEY + Customer + eSpiel Inc. + eSpiel Incorporated + + + + + +``` + +Note that `ErrorDescription` value is an array of string values. The +first element of the array is an error code; the remainder of the array are +strings representing error parameters relating to that code. In this case, +the client has attempted to add the mapping _Customer -> +eSpiel Incorporated_ to a Map, but it already contains the mapping +_Customer -> eSpiel Inc._, hence the request has failed. + +When using the JSON-RPC protocol v2.0, the above error is transmitted as: + +```json +{ + "jsonrpc": "2.0", + "error": { + "code": 1, + "message": "MAP_DUPLICATE_KEY", + "data": [ + "Customer", + "eSpiel Inc.", + "eSpiel Incorporated" + ] + }, + "id": 3 +} +``` + +Finally, when using the JSON-RPC protocol v1.0: + +```json +{ + "result": null, + "error": [ + "MAP_DUPLICATE_KEY", + "Customer", + "eSpiel Inc.", + "eSpiel Incorporated" + ], + "id": "xyz" +} +``` + +Each possible error code is documented in the following section. + +## Error Codes +{{#errors}} + +### {{{error_code}}} + +{{{error_description}}} + +_Signature:_ + +``` +{{{error_code_unescaped}}}({{parameters}}) +``` +{{/errors}} \ No newline at end of file diff --git a/ocaml/idl/templates/class.mustache b/ocaml/idl/templates/class.mustache new file mode 100644 index 00000000000..65c7c294d22 --- /dev/null +++ b/ocaml/idl/templates/class.mustache @@ -0,0 +1,89 @@ +--- + layout: doc +--- + +# Class: {{{class_name}}} +{{#class_deprecated}} + +**This class is deprecated.** +{{/class_deprecated}} +{{#class_removed}} + +**This class is removed.** +{{/class_removed}} +{{#has_descr}} + +{{{class_descr}}} +{{/has_descr}} + +## Fields for class: {{{class_name}}} +{{#has_fields}} + +|Field |Type |Qualifier |Description | +|:-------------------|:-------------------|:--------------|:---------------------------------------| +{{/has_fields}} +{{#fields}} +|{{{field_name}}}|{{{field_type}}}|{{{field_ctor}}}|{{#field_deprecated}}**Deprecated.** {{/field_deprecated}}{{#field_removed}}**Removed.** {{/field_removed}}{{{field_descr}}}| +{{/fields}} +{{#is_event}} +|snapshot |object record |_RO/runtime_ |The record of the database object that was added, changed or deleted| +{{/is_event}} +{{^has_fields}} + +Class {{{class_name}}} has no fields. +{{/has_fields}} + +## RPCs associated with class: {{{class_name}}} +{{#all_rpcs}} + +### RPC name: {{{rpc_name_escaped}}} +{{#rpc_deprecated}} + +**This message is deprecated.** +{{/rpc_deprecated}} +{{#rpc_removed}} + +**This message is removed.** +{{/rpc_removed}} + +_Overview:_ +{{#rpc_has_descr}} + +{{{rpc_descr}}} +{{/rpc_has_descr}} + +_Signature:_ + +``` +{{{return_type}}} {{{rpc_name}}} ({{#session}}session ref session_ref{{#has_rpc_params}}, {{/has_rpc_params}}{{/session}}{{{rpc_param_csv}}}) +``` + +_Arguments:_ + +|type |name |description | +|:-----------------------------|:-----------------------------|:---------------------------------------| +{{#session}} +|session ref |session_ref |Reference to a valid session | +{{/session}} +{{#rpc_params}} +|{{{param_type}}}|{{{param_name}}}|{{{param_descr}}}| +{{/rpc_params}} + +{{#has_rbac}} +_Minimum Role:_ {{min_role}} + +{{/has_rbac}} +_Return Type:_ `{{{return_type}}}` +{{^returns_void}} + +{{{return_descr}}} +{{/returns_void}} +{{#has_error_codes}} + +_Possible Error Codes:_ {{{error_codes_csv}}} +{{/has_error_codes}} +{{/all_rpcs}} +{{^has_rpcs}} + +Class {{{class_name}}} has no RPCs associated with it. +{{/has_rpcs}} \ No newline at end of file diff --git a/ocaml/idl/templates/classes.mustache b/ocaml/idl/templates/classes.mustache new file mode 100644 index 00000000000..3730d77d0bc --- /dev/null +++ b/ocaml/idl/templates/classes.mustache @@ -0,0 +1,13 @@ +--- + layout: doc +--- + +# Classes + +The following classes are defined: + +|Name |Description | +|:-------------------|:---------------------------------------------------------------------| +{{#classes}} +|[{{{name}}}](@root@management-api/class-{{{name_lower}}}.html)|{{{description}}}| +{{/classes}} diff --git a/ocaml/idl/templates/relationships.mustache b/ocaml/idl/templates/relationships.mustache new file mode 100644 index 00000000000..a8567f5c6be --- /dev/null +++ b/ocaml/idl/templates/relationships.mustache @@ -0,0 +1,19 @@ +--- + layout: doc +--- + +# Relationships Between Classes + +Fields that are bound together are shown in the following table: + +|_object.field_ |_object.field_ |_relationship_ | +|:---------------------------------------|:---------------------------------------|:--------------| +{{#relationships}} +|{{{a_field}}}|{{{b_field}}}|{{relationship}}| +{{/relationships}} + +The following figure represents bound fields (as specified above) +diagramatically, using crow's foot notation to specify one-to-one, +one-to-many, or many-to-many relationships: + +![Class relationships](classes.png 'Class relationships') \ No newline at end of file diff --git a/ocaml/idl/templates/toc.mustache b/ocaml/idl/templates/toc.mustache new file mode 100644 index 00000000000..01f81f0982f --- /dev/null +++ b/ocaml/idl/templates/toc.mustache @@ -0,0 +1,15 @@ +- title: API Reference + url: @root@api-ref-autogen.html + subfolderlist: + - title: Classes + url: @root@management-api/classes.html +{{#classes}} + - title: Class:{{{name}}} + url: @root@management-api/class-{{{name_lower}}}.html +{{/classes}} + - title: Relationships Between Classes + url: @root@management-api/relationships-between-classes.html + - title: Types + url: @root@management-api/types.html + - title: Error Handling + url: @root@management-api/api-ref-autogen-errors.html diff --git a/ocaml/idl/templates/types.mustache b/ocaml/idl/templates/types.mustache new file mode 100644 index 00000000000..197a8ca5a0e --- /dev/null +++ b/ocaml/idl/templates/types.mustache @@ -0,0 +1,40 @@ +--- + layout: doc +--- + +# Types + +## Primitives + +The following primitive types are used to specify methods and fields in +the API Reference: + +|Type |Description | +|:-------|:-------------------------------------------| +|string |text strings | +|int |64-bit integers | +|float |IEEE double-precision floating-point numbers| +|bool |boolean | +|datetime|date and timestamp | + +## Higher-order types + +The following type constructors are used: + +|Type |Description | +|:-----------------|:-------------------------------------------------------| +|_c_ ref |reference to an object of class _c_ | +|_t_ set |a set of elements of type _t_ | +|(_a -> b_) map |a table mapping values of type _a_ to values of type _b_| + +## Enumeration types + +The following enumeration types are used: +{{#enums}} + +|enum {{{enum}}}| | +|:---------------------------------------|:---------------------------------------| +{{#enum_options}} +|{{{option_name}}}|{{{option_descr}}}| +{{/enum_options}} +{{/enums}} \ No newline at end of file From ae7480c84960f4ce3693d27858db8eeb58d5cb95 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Sun, 17 Dec 2023 19:24:26 +0000 Subject: [PATCH 88/99] Generate the C enums with templates. Signed-off-by: Konstantina Chremmou --- ocaml/sdk-gen/c/gen_c_binding.ml | 218 ++++-------------- ocaml/sdk-gen/c/templates/xen_enum.c.mustache | 98 ++++++++ ocaml/sdk-gen/c/templates/xen_enum.h.mustache | 91 ++++++++ .../c/templates/xen_enum_internal.h.mustache | 51 ++++ .../powershell/gen_powershell_binding.ml | 8 +- 5 files changed, 285 insertions(+), 181 deletions(-) create mode 100644 ocaml/sdk-gen/c/templates/xen_enum.c.mustache create mode 100644 ocaml/sdk-gen/c/templates/xen_enum.h.mustache create mode 100644 ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache diff --git a/ocaml/sdk-gen/c/gen_c_binding.ml b/ocaml/sdk-gen/c/gen_c_binding.ml index 5203b86f2e0..e9035a88c7f 100644 --- a/ocaml/sdk-gen/c/gen_c_binding.ml +++ b/ocaml/sdk-gen/c/gen_c_binding.ml @@ -83,11 +83,7 @@ let rec main () = all_headers := List.map (fun x -> x.name) filtered_classes ; - TypeSet.iter (gen_enum write_enum_decl decl_filename include_dir) !enums ; - TypeSet.iter (gen_enum write_enum_impl impl_filename src_dir) !enums ; - TypeSet.iter - (gen_enum write_enum_internal_decl internal_decl_filename include_dir) - !enums ; + TypeSet.iter render_enum !enums ; maps := TypeSet.add (Map (String, Int)) !maps ; maps := TypeSet.add (Map (Int, Int)) !maps ; @@ -140,17 +136,6 @@ and gen_class f g clas targetdir = let out_chan = open_out (Filename.concat targetdir (g clas.name)) in Fun.protect (fun () -> f clas out_chan) ~finally:(fun () -> close_out out_chan) -and gen_enum f g targetdir = function - | Enum (name, _) as x -> - if not (List.mem name !all_headers) then - all_headers := name :: !all_headers ; - let out_chan = open_out (Filename.concat targetdir (g name)) in - Fun.protect - (fun () -> f x out_chan) - ~finally:(fun () -> close_out out_chan) - | _ -> - assert false - and gen_map f g targetdir = function | Map (l, r) -> let name = mapname l r in @@ -669,174 +654,49 @@ and hash_include n = else sprintf "#include <%s>" (decl_filename n) -and write_enum_decl x out_chan = - match x with - | Enum (name, contents) -> - let print format = fprintf out_chan format in - let protect = protector name in - let tn = typename name in - - print_h_header out_chan protect ; +and replace_dashes x = + Astring.String.map (fun y -> match y with '-' -> '_' | _ -> y) x - print - "\n\ - %s\n\n\n\ - enum %s\n\ - {\n\ - %s\n\ - };\n\n\n\ - typedef struct %s_set\n\ - {\n\ - \ size_t size;\n\ - \ enum %s contents[];\n\ - } %s_set;\n\n\ - %s\n\ - extern %s_set *\n\ - %s_set_alloc(size_t size);\n\n\ - %s\n\n\n\ - %s\n\ - extern const char *\n\ - %s_to_string(enum %s val);\n\n\n\ - %s\n\ - extern enum %s\n\ - %s_from_string(xen_session *session, const char *str);\n\n" - (hash_include "common") tn - (joined ",\n\n" (enum_entry name) - (contents - @ [("undefined", "Unknown to this version of the bindings.")] - ) - ) - tn tn tn - (Helper.comment true (sprintf "Allocate a %s_set of the given size." tn)) - tn tn - (decl_free (sprintf "%s_set" tn) "*set" false "set") - (Helper.comment true - "Return the name corresponding to the given code. This string must \ - not be modified or freed." - ) - tn tn - (Helper.comment true - "Return the correct code for the given string, or set the session \ - object to failure and return an undefined value if the given \ - string does not match a known code." - ) - tn tn ; - - print_h_footer out_chan - | _ -> - () - -and enum_entry enum_name = function - | n, c -> - sprintf "%s\n XEN_%s_%s" - (Helper.comment true ~indent:4 c) - (String.uppercase_ascii enum_name) - (Astring.String.map - (fun x -> match x with '-' -> '_' | _ -> x) - (String.uppercase_ascii n) - ) - -and write_enum_impl x out_chan = +and render_enum x = match x with | Enum (name, contents) -> - let print format = fprintf out_chan format in - let tn = typename name in - - print - "%s\n\n\ - #include \n\n\ - %s\n\ - %s\n\ - %s\n\n\n\ - /*\n\ - \ * Maintain this in the same order as the enum declaration!\n\ - \ */\n\ - static const char *lookup_table[] =\n\ - {\n\ - %s\n\ - };\n\n\n\ - extern %s_set *\n\ - %s_set_alloc(size_t size)\n\ - {\n\ - \ return calloc(1, sizeof(%s_set) +\n\ - \ size * sizeof(enum %s));\n\ - }\n\n\n\ - extern void\n\ - %s_set_free(%s_set *set)\n\ - {\n\ - \ free(set);\n\ - }\n\n\n\ - const char *\n\ - %s_to_string(enum %s val)\n\ - {\n\ - \ return lookup_table[val];\n\ - }\n\n\n\ - extern enum %s\n\ - %s_from_string(xen_session *session, const char *str)\n\ - {\n\ - \ (void)session;\n\ - \ return ENUM_LOOKUP(str, lookup_table);\n\ - }\n\n\n\ - const abstract_type %s_abstract_type_ =\n\ - \ {\n\ - \ .XEN_API_TYPE = ENUM,\n\ - \ .enum_marshaller =\n\ - \ (const char *(*)(int))&%s_to_string,\n\ - \ .enum_demarshaller =\n\ - \ (int (*)(xen_session *, const char *))&%s_from_string\n\ - \ };\n\n\n" - Licence.bsd_two_clause (hash_include "internal") (hash_include name) - (hash_include (name ^ "_internal")) - (enum_lookup_entries (contents @ [("undefined", "")])) - tn tn tn tn tn tn tn tn tn tn tn tn tn ; - - if name <> "event_operation" then - print - "const abstract_type %s_set_abstract_type_ =\n\ - \ {\n\ - \ .XEN_API_TYPE = SET,\n\ - \ .child = &%s_abstract_type_\n\ - \ };\n\n\n" - tn tn - | _ -> - () - -and enum_lookup_entries contents = joined ",\n" enum_lookup_entry contents - -and enum_lookup_entry = function n, _ -> sprintf " \"%s\"" n - -and write_enum_internal_decl x out_chan = - match x with - | Enum (name, _) -> - let print format = fprintf out_chan format in - let protect = protector (sprintf "%s_internal" name) in - let tn = typename name in - - let set_abstract_type = - if name = "event_operations" then - "" - else - sprintf "extern const abstract_type %s_set_abstract_type_;\n" tn + if not (List.mem name !all_headers) then + all_headers := name :: !all_headers ; + let json = + `O + [ + ("enum_name", `String name) + ; ("enum_name_upper", `String (String.uppercase_ascii name)) + ; ("event_operations", `Bool (name = "event_operation")) + ; ( "enum_values" + , `A + (List.map + (fun (n, c) -> + `O + [ + ("enum_value", `String n) + ; ("enum_value_doc", `String c) + ; ( "enum_value_upper" + , `String (replace_dashes (String.uppercase_ascii n)) + ) + ] + ) + contents + ) + ) + ] in - - print - "%s\n\n\n\ - %s\n\n\n\ - #ifndef %s\n\ - #define %s\n\n\n\ - %s\n\n\n\ - extern const abstract_type %s_abstract_type_;\n\ - %s\n\n\ - #endif\n" - Licence.bsd_two_clause - (Helper.comment false - (sprintf - "Declarations of the abstract types used during demarshalling of \ - enum %s. Internal to this library -- do not use from outside." - tn - ) + render_file + ( "xen_enum_internal.h.mustache" + , sprintf "include/xen_%s_internal.h" name ) - protect protect (hash_include "internal") tn set_abstract_type + json templates_dir destdir ; + render_file + ("xen_enum.h.mustache", sprintf "include/xen/api/xen_%s.h" name) + json templates_dir destdir ; + render_file + ("xen_enum.c.mustache", sprintf "src/xen_%s.c" name) + json templates_dir destdir | _ -> () diff --git a/ocaml/sdk-gen/c/templates/xen_enum.c.mustache b/ocaml/sdk-gen/c/templates/xen_enum.c.mustache new file mode 100644 index 00000000000..421c9015a6f --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_enum.c.mustache @@ -0,0 +1,98 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include + +#include "xen_internal.h" +#include +#include "xen_{{{enum_name}}}_internal.h" + + +/* + * Maintain this in the same order as the enum declaration! + */ +static const char *lookup_table[] = +{ +{{#enum_values}} + "{{{enum_value}}}", +{{/enum_values}} + "undefined" +}; + + +extern xen_{{{enum_name}}}_set * +xen_{{{enum_name}}}_set_alloc(size_t size) +{ + return calloc(1, sizeof(xen_{{{enum_name}}}_set) + + size * sizeof(enum xen_{{{enum_name}}})); +} + + +extern void +xen_{{{enum_name}}}_set_free(xen_{{{enum_name}}}_set *set) +{ + free(set); +} + + +const char * +xen_{{{enum_name}}}_to_string(enum xen_{{{enum_name}}} val) +{ + return lookup_table[val]; +} + + +extern enum xen_{{{enum_name}}} +xen_{{{enum_name}}}_from_string(xen_session *session, const char *str) +{ + (void)session; + return ENUM_LOOKUP(str, lookup_table); +} + + +const abstract_type xen_{{{enum_name}}}_abstract_type_ = + { + .XEN_API_TYPE = ENUM, + .enum_marshaller = + (const char *(*)(int))&xen_{{{enum_name}}}_to_string, + .enum_demarshaller = + (int (*)(xen_session *, const char *))&xen_{{{enum_name}}}_from_string + }; + + +{{^event_operations}} +const abstract_type xen_{{{enum_name}}}_set_abstract_type_ = + { + .XEN_API_TYPE = SET, + .child = &xen_{{{enum_name}}}_abstract_type_ + }; + + +{{/event_operations}} diff --git a/ocaml/sdk-gen/c/templates/xen_enum.h.mustache b/ocaml/sdk-gen/c/templates/xen_enum.h.mustache new file mode 100644 index 00000000000..824179cf2d3 --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_enum.h.mustache @@ -0,0 +1,91 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef XEN_{{{enum_name_upper}}}_H +#define XEN_{{{enum_name_upper}}}_H + + +#include + + +enum xen_{{{enum_name}}} +{ +{{#enum_values}} + /** + * {{{enum_value_doc}}} + */ + XEN_{{{enum_name_upper}}}_{{{enum_value_upper}}}, + +{{/enum_values}} + /** + * Unknown to this version of the bindings. + */ + XEN_{{{enum_name_upper}}}_UNDEFINED +}; + + +typedef struct xen_{{{enum_name}}}_set +{ + size_t size; + enum xen_{{{enum_name}}} contents[]; +} xen_{{{enum_name}}}_set; + +/** + * Allocate a xen_{{{enum_name}}}_set of the given size. + */ +extern xen_{{{enum_name}}}_set * +xen_{{{enum_name}}}_set_alloc(size_t size); + +/** + * Free the given xen_{{{enum_name}}}_set. The given set must have been + * allocated by this library. + */ +extern void +xen_{{{enum_name}}}_set_free(xen_{{{enum_name}}}_set *set); + + +/** + * Return the name corresponding to the given code. This string must + * not be modified or freed. + */ +extern const char * +xen_{{{enum_name}}}_to_string(enum xen_{{{enum_name}}} val); + + +/** + * Return the correct code for the given string, or set the session + * object to failure and return an undefined value if the given string does + * not match a known code. + */ +extern enum xen_{{{enum_name}}} +xen_{{{enum_name}}}_from_string(xen_session *session, const char *str); + + +#endif diff --git a/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache b/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache new file mode 100644 index 00000000000..b9731686edb --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache @@ -0,0 +1,51 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + + +/* + * Declarations of the abstract types used during demarshalling of enum + * xen_{{{enum_name}}}. Internal to this library -- do not use from outside. + */ + + +#ifndef XEN_{{{enum_name_upper}}}_INTERNAL_H +#define XEN_{{{enum_name_upper}}}_INTERNAL_H + + +#include "xen_internal.h" + + +extern const abstract_type xen_{{{enum_name}}}_abstract_type_; +{{^event_operations}} +extern const abstract_type xen_{{{enum_name}}}_set_abstract_type_; +{{/event_operations}} + + +#endif diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 3f49c7deff8..b455d010486 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -18,6 +18,7 @@ module TypeSet = Set.Make (struct end) let destdir = "autogen/src" + let templdir = "templates" type cmdlet = {filename: string; content: string} @@ -111,9 +112,12 @@ and gen_http_action action = let stem = get_http_action_stem name in let arg_name = function | String_query_arg x | Int64_query_arg x -> - pascal_case_rec x + pascal_case_rec x | Bool_query_arg x -> - if String.lowercase_ascii x = "host" then "IsHost" else pascal_case_rec x + if String.lowercase_ascii x = "host" then + "IsHost" + else + pascal_case_rec x | Varargs_query_arg -> "Args" in From 37821d8828a8366c79676027cc73ffac440e01c8 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Fri, 8 Mar 2024 21:46:22 +0000 Subject: [PATCH 89/99] Corrections as per code review. Signed-off-by: Konstantina Chremmou --- ocaml/doc/basics.md | 4 +-- ocaml/doc/wire-protocol.md | 10 ++++---- ocaml/idl/datamodel.ml | 5 +++- ocaml/idl/datamodel_certificate.ml | 3 ++- ocaml/idl/markdown_backend.ml | 10 +++++--- ocaml/idl/templates/api_errors.mustache | 19 ++++++++++---- ocaml/idl/templates/class.mustache | 4 +-- ocaml/idl/templates/relationships.mustache | 9 +++---- ocaml/idl/templates/types.mustache | 30 ++++++++++++---------- 9 files changed, 56 insertions(+), 38 deletions(-) diff --git a/ocaml/doc/basics.md b/ocaml/doc/basics.md index 9a75a16087b..b1812da023d 100644 --- a/ocaml/doc/basics.md +++ b/ocaml/doc/basics.md @@ -5,12 +5,12 @@ # API Basics This document defines the XenServer Management API - an interface for remotely -configuring and controlling virtualised guests running on a Xen-enabled host. +configuring and controlling virtualized guests running on a Xen-enabled host. The API is presented here as a set of Remote Procedure Calls (RPCs). There are two supported wire formats, one based upon [XML-RPC](http://xmlrpc.scripting.com/spec.html) and one based upon [JSON-RPC](http://www.jsonrpc.org) (v1.0 and v2.0 are both -recognised). No specific language bindings are prescribed, although examples +recognized). No specific language bindings are prescribed, although examples will be given in the python programming language. Although we adopt some terminology from object-oriented programming, diff --git a/ocaml/doc/wire-protocol.md b/ocaml/doc/wire-protocol.md index e42a0f2da2c..808d021154d 100644 --- a/ocaml/doc/wire-protocol.md +++ b/ocaml/doc/wire-protocol.md @@ -182,7 +182,7 @@ following manner: * our `void` type is transmitted as an empty string. -Both versions 1.0 and 2.0 of the JSON-RPC wire format are recognised and, +Both versions 1.0 and 2.0 of the JSON-RPC wire format are recognized and, depending on your client library, you can use either of them. ### JSON-RPC v1.0 @@ -490,7 +490,7 @@ session reference is returned under the key `Value` in the resulting dictionary ... "version", "originator")['Value'] ``` -This is what the call looks like when serialised +This is what the call looks like when serialized ```xml @@ -534,7 +534,7 @@ Once a reference to a VM has been acquired, a lifecycle operation may be invoked In this case the `start` message has been rejected, because the VM is a template, and so an error response has been returned. These high-level errors are returned as structured data (rather than as XML-RPC faults), -allowing them to be internationalised. +allowing them to be internationalized. Rather than querying fields individually, whole _records_ may be returned at once. To retrieve the record of a single object as a python dictionary: @@ -579,7 +579,7 @@ reference: ... "user", "passwd", "version", "originator") ``` -`pyjsonrpc` uses the JSON-RPC protocol v2.0, so this is what the serialised +`pyjsonrpc` uses the JSON-RPC protocol v2.0, so this is what the serialized request looks like: ```json @@ -627,7 +627,7 @@ Once a reference to a VM has been acquired, a lifecycle operation may be invoked In this case the `start` message has been rejected because the VM is a template, hence an error response has been returned. These high-level -errors are returned as structured data, allowing them to be internationalised. +errors are returned as structured data, allowing them to be internationalized. Rather than querying fields individually, whole _records_ may be returned at once. To retrieve the record of a single object as a python dictionary: diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index d7bff0266a9..8f452b47069 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -2057,7 +2057,10 @@ module Bond = struct let t = create_obj ~in_db:true ~in_product_since:rel_miami ~in_oss_since:None ~persist:PersistEverything ~gen_constructor_destructor:false ~name:_bond - ~descr:"" ~gen_events:true ~doccomments:[] + ~descr: + "A Network bond that combines physical network interfaces, also known \ + as link aggregation" + ~gen_events:true ~doccomments:[] ~messages_default_allowed_roles:_R_POOL_OP ~doc_tags:[Networking] ~messages:[create; destroy; set_mode; set_property] ~contents: diff --git a/ocaml/idl/datamodel_certificate.ml b/ocaml/idl/datamodel_certificate.ml index b18a20e7656..ac77887b9f0 100644 --- a/ocaml/idl/datamodel_certificate.ml +++ b/ocaml/idl/datamodel_certificate.ml @@ -34,7 +34,8 @@ let certificate_type = ) let t = - create_obj ~name:_certificate ~descr:"Description" ~doccomments:[] + create_obj ~name:_certificate + ~descr:"An X509 certificate used for TLS connections" ~doccomments:[] ~gen_constructor_destructor:false ~gen_events:true ~in_db:true ~lifecycle ~persist:PersistEverything ~in_oss_since:None ~messages_default_allowed_roles:_R_READ_ONLY diff --git a/ocaml/idl/markdown_backend.ml b/ocaml/idl/markdown_backend.ml index c4bbd538fe1..38ec1c8caef 100644 --- a/ocaml/idl/markdown_backend.ml +++ b/ocaml/idl/markdown_backend.ml @@ -185,7 +185,10 @@ let generate_class cls = ( "field_name" , `String (pad_right - (escape (Datamodel_utils.wire_name_of_field field)) + ("`" + ^ Datamodel_utils.wire_name_of_field field + ^ "`" + ) col_width_20 ) ) @@ -298,7 +301,8 @@ let generate_class cls = [ ( "param_name" , `String - (pad_right (escape p.param_name) + (pad_right + ("`" ^ p.param_name ^ "`") col_width_30 ) ) @@ -357,7 +361,7 @@ let generate_types system = | Enum (name, options) -> `O [ - ("enum", `String (pad_right name (col_width_40 - 5))) + ("enum", `String name) ; ( "enum_options" , `A (options diff --git a/ocaml/idl/templates/api_errors.mustache b/ocaml/idl/templates/api_errors.mustache index e1a4b95fbaa..d2e877e25d1 100644 --- a/ocaml/idl/templates/api_errors.mustache +++ b/ocaml/idl/templates/api_errors.mustache @@ -9,15 +9,19 @@ or RPC level, the server may send an HTTP 500 error response, or the client may simulate the same. The client must be prepared to handle these errors, though they may be treated as fatal. -On the wire, these are transmitted in a form similar to this when using the -XML-RPC protocol: +For example, the following malformed request when using the XML-RPC protocol: -``` +```sh $curl -D - -X POST https://server -H 'Content-Type: application/xml' \ > -d ' > > session.logout > ' +``` + +results to the following response: + +```sh HTTP/1.1 500 Internal Error content-length: 297 content-type:text/html @@ -32,13 +36,18 @@ t;close_tag", "open_tag", _) When using the JSON-RPC protocol: -``` +```sh $curl -D - -X POST https://server/jsonrpc -H 'Content-Type: application/json' \ > -d '{ > "jsonrpc": "2.0", > "method": "session.login_with_password", > "id": 0 > }' +``` + +the response is: + +```sh HTTP/1.1 500 Internal Error content-length: 308 content-type:text/html @@ -52,7 +61,7 @@ d_request("{jsonrpc=...,method=...,id=...}") ``` All other failures are reported with a more structured error response, to -allow better automatic response to failures, proper internationalisation of +allow better automatic response to failures, proper internationalization of any error message, and easier debugging. On the wire, these are transmitted like this when using the XML-RPC protocol: diff --git a/ocaml/idl/templates/class.mustache b/ocaml/idl/templates/class.mustache index 65c7c294d22..1112c851642 100644 --- a/ocaml/idl/templates/class.mustache +++ b/ocaml/idl/templates/class.mustache @@ -60,10 +60,10 @@ _Signature:_ _Arguments:_ -|type |name |description | +|Type |Name |Description | |:-----------------------------|:-----------------------------|:---------------------------------------| {{#session}} -|session ref |session_ref |Reference to a valid session | +|`session ref` |`session_ref` |Reference to a valid session | {{/session}} {{#rpc_params}} |{{{param_type}}}|{{{param_name}}}|{{{param_descr}}}| diff --git a/ocaml/idl/templates/relationships.mustache b/ocaml/idl/templates/relationships.mustache index a8567f5c6be..6678c875681 100644 --- a/ocaml/idl/templates/relationships.mustache +++ b/ocaml/idl/templates/relationships.mustache @@ -6,14 +6,13 @@ Fields that are bound together are shown in the following table: -|_object.field_ |_object.field_ |_relationship_ | -|:---------------------------------------|:---------------------------------------|:--------------| +|_object.field_ |_object.field_ |_relationship_ | +|:-------------------------------------|:-------------------------------------|:--------------| {{#relationships}} |{{{a_field}}}|{{{b_field}}}|{{relationship}}| {{/relationships}} -The following figure represents bound fields (as specified above) -diagramatically, using crow's foot notation to specify one-to-one, -one-to-many, or many-to-many relationships: +The following figure represents bound fields (as specified above) using crow's +foot notation to specify one-to-one, one-to-many, or many-to-many relationships: ![Class relationships](classes.png 'Class relationships') \ No newline at end of file diff --git a/ocaml/idl/templates/types.mustache b/ocaml/idl/templates/types.mustache index 197a8ca5a0e..f1ebc136899 100644 --- a/ocaml/idl/templates/types.mustache +++ b/ocaml/idl/templates/types.mustache @@ -9,31 +9,33 @@ The following primitive types are used to specify methods and fields in the API Reference: -|Type |Description | -|:-------|:-------------------------------------------| -|string |text strings | -|int |64-bit integers | -|float |IEEE double-precision floating-point numbers| -|bool |boolean | -|datetime|date and timestamp | +|Type |Description | +|:---------|:-------------------------------------------| +|`string` |text strings | +|`int` |64-bit integers | +|`float` |IEEE double-precision floating-point numbers| +|`bool` |boolean | +|`datetime`|date and timestamp | ## Higher-order types The following type constructors are used: -|Type |Description | -|:-----------------|:-------------------------------------------------------| -|_c_ ref |reference to an object of class _c_ | -|_t_ set |a set of elements of type _t_ | -|(_a -> b_) map |a table mapping values of type _a_ to values of type _b_| +|Type |Description | +|:-------------|:-------------------------------------------------------| +|`c ref` |reference to an object of class `c` | +|`t set` |a set of elements of type `t` | +|`(a -> b) map`|a table mapping values of type `a` to values of type `b`| ## Enumeration types The following enumeration types are used: {{#enums}} -|enum {{{enum}}}| | -|:---------------------------------------|:---------------------------------------| +### Enum {{{enum}}} + +|Named value |Description | +|:---------------------------------------|:----------------------------------------------------| {{#enum_options}} |{{{option_name}}}|{{{option_descr}}}| {{/enum_options}} From 311b79b9fb079e34b25d9246520489d360985ee0 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Mon, 11 Mar 2024 06:38:28 +0100 Subject: [PATCH 90/99] Github Actions: Update to falti/dotenv-action@v1 to fix Node 16 warnings Co-authored-by: Pau Ruiz Safont Signed-off-by: Bernhard Kaindl --- .github/workflows/1.249-lcm.yml | 2 +- .github/workflows/format.yml | 2 +- .github/workflows/setup-xapi-environment/action.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/1.249-lcm.yml b/.github/workflows/1.249-lcm.yml index 5fec8bef8d8..e1e78b7eb4d 100644 --- a/.github/workflows/1.249-lcm.yml +++ b/.github/workflows/1.249-lcm.yml @@ -40,7 +40,7 @@ jobs: - name: Load environment file id: dotenv - uses: falti/dotenv-action@v1.0.2 + uses: falti/dotenv-action@v1 - name: Retrieve date for cache key (year-week) id: cache-key diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index b15173805cf..b7ea7288627 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -22,7 +22,7 @@ jobs: - name: Load environment file id: dotenv - uses: falti/dotenv-action@v1.0.4 + uses: falti/dotenv-action@v1 - name: Update Ubuntu repositories run: sudo apt-get update diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index 702162d42f3..e25e0e184fb 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -25,7 +25,7 @@ runs: - name: Load environment file id: dotenv - uses: falti/dotenv-action@v1.0.4 + uses: falti/dotenv-action@v1 - name: Update Ubuntu repositories shell: bash From 51521bf11de3617b6e43557a9a4d9fe908920fcc Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 11 Mar 2024 16:52:54 +0000 Subject: [PATCH 91/99] CA-389496: Avoid configuration conflicts for rotating xapi logs The custom logrotation setup for xapi sometimes conflicted with the normal one. Remove the former. Install the xapi logrotate configuration has been moved to be managed as part of the normal logrotation, with the global settings removed. Duplicate settings in the configuration have been removed. Signed-off-by: Pau Ruiz Safont --- scripts/Makefile | 4 +--- scripts/xapi-logrotate.conf | 20 -------------------- scripts/xapi-logrotate.cron | 3 --- scripts/xapi-logrotate.sh | 13 ------------- 4 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 scripts/xapi-logrotate.cron delete mode 100755 scripts/xapi-logrotate.sh diff --git a/scripts/Makefile b/scripts/Makefile index 17c8e383fac..d2ab8f5d381 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -45,8 +45,7 @@ install: $(IPROG) xapi-health-check $(DESTDIR)$(LIBEXECDIR) $(IPROG) mail-alarm $(DESTDIR)$(LIBEXECDIR) $(IDATA) audit-logrotate $(DESTDIR)/etc/logrotate.d/audit - $(IDATA) xapi-logrotate.conf $(DESTDIR)$(ETCXENDIR) - $(IPROG) xapi-logrotate.sh $(DESTDIR)$(LIBEXECDIR) + $(IDATA) xapi-logrotate.conf $(DESTDIR)/etc/logrotate.d/xapi $(IPROG) xapi-tracing-log-trim.sh $(DESTDIR)$(LIBEXECDIR) $(IPROG) xapi-wait-init-complete $(DESTDIR)$(OPTDIR)/bin $(IPROG) xapi-autostart-vms $(DESTDIR)$(OPTDIR)/bin @@ -158,7 +157,6 @@ install: $(IPROG) certificate-check $(DESTDIR)/etc/cron.daily/ $(IPROG) certificate-refresh $(DESTDIR)/etc/cron.hourly/ mkdir -p $(DESTDIR)/etc/cron.d - $(IDATA) xapi-logrotate.cron $(DESTDIR)/etc/cron.d/xapi-logrotate.cron $(IDATA) xapi-tracing-log-trim.cron $(DESTDIR)/etc/cron.d/xapi-tracing-log-trim.cron mkdir -p $(DESTDIR)/opt/xensource/gpg # templates diff --git a/scripts/xapi-logrotate.conf b/scripts/xapi-logrotate.conf index b50677e72f2..e9d4845d582 100644 --- a/scripts/xapi-logrotate.conf +++ b/scripts/xapi-logrotate.conf @@ -1,20 +1,3 @@ -# see "man logrotate" for details -# rotate log files daily -daily - -# keep one months worth of backlogs -rotate 31 - -# create new (empty) log files after rotating old ones -create - -# compress log files -compress -delaycompress - -# All the general settings above are copied from /etc/logrotate.conf -# as installed by the logrotate RPM. - /var/log/xensource.log { missingok @@ -26,9 +9,6 @@ delaycompress # When rotating, remove any rotated logs older than this many days. maxage 31 - # Rotate when file exceeds this size, even if we rotated it today already. - maxsize 100M - # Keep up to this many old files. # (When considering total size, expect a compression factor around 10-20.) rotate 100 diff --git a/scripts/xapi-logrotate.cron b/scripts/xapi-logrotate.cron deleted file mode 100644 index 3f4506fb1f6..00000000000 --- a/scripts/xapi-logrotate.cron +++ /dev/null @@ -1,3 +0,0 @@ -# Run at 40 minutes past the hour, -# every hour, day-of-month, month, day-of-week -40 * * * * root /opt/xensource/libexec/xapi-logrotate.sh diff --git a/scripts/xapi-logrotate.sh b/scripts/xapi-logrotate.sh deleted file mode 100755 index b4a5935feae..00000000000 --- a/scripts/xapi-logrotate.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -# If we're using the old partitioning scheme there's no need to run this. -if ! grep -q /var/log /proc/mounts; then - exit 0 -fi - -/usr/sbin/logrotate /etc/xensource/xapi-logrotate.conf -EXITVALUE=$? -if [ $EXITVALUE != 0 ]; then - /usr/bin/logger -t $0 "ALERT exited abnormally with [$EXITVALUE]" -fi -exit 0 From ef9f18ba829cd14b28290409aa8b0549d4866bd7 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 13 Mar 2024 08:39:30 +0000 Subject: [PATCH 92/99] Bump maven plugin versions Also update URL in `pom.xml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/autogen/xen-api/pom.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml index abaed44d833..66e1b633db2 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml +++ b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml @@ -1,6 +1,6 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 com.xenserver xen-api @@ -8,16 +8,16 @@ jar XenServer Java SDK Mavenized build of the XenServer SDK for Java. - https://www.citrix.com/community/citrix-developer/citrix-hypervisor-developer + https://docs.xenserver.com/en-us/xenserver/8/developer Cloud Software Group, Inc. https://www.cloud.com - BSD 2-Clause License - http://opensource.org/licenses/BSD-2-Clause - repo + BSD 2-Clause License + http://opensource.org/licenses/BSD-2-Clause + repo @@ -81,7 +81,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.11.0 + 3.12.1 11 @@ -96,7 +96,7 @@ org.apache.maven.plugins maven-source-plugin - 3.2.1 + 3.3.0 attach-sources @@ -109,7 +109,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.6.3 attach-javadocs From 969b7e7db24e61af3fc0c0d252266aa39e1ce4e3 Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 12 Mar 2024 16:19:47 +0800 Subject: [PATCH 93/99] CA-389840: Bug in parsing output of 'xen-livepatch list' The bug is in parsing the output of 'xen-livepatch list'. The following output can't be parsed correctly: lp_4.17.3-3.11.gf717213.xs8-4.17.3-3.12.xs8| CHECKED The regex pattern doesn't exclude the '|' after 'xs8'. As a result, the vertical line '|' is parsed as part of '3.12.xs8|'. The correct one should be '3.12.xs8'. Signed-off-by: Ming Lu --- ocaml/tests/test_livepatch.ml | 32 ++++++++++++++++++++++++++ ocaml/xapi/livepatch.ml | 43 ++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/ocaml/tests/test_livepatch.ml b/ocaml/tests/test_livepatch.ml index 31b0eef8bfc..b87b657c8fe 100644 --- a/ocaml/tests/test_livepatch.ml +++ b/ocaml/tests/test_livepatch.ml @@ -52,6 +52,38 @@ lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8 | CHECKED |} , None ) + ; ( {| + ID | status +----------------------------------------+------------ +lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8| CHECKED +lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8| APPLIED + |} + , Some ("4.13.4", "10.22.xs8", "4.13.4", "10.23.xs8") + ) + ; ( {| + ID | status +----------------------------------------+------------ + lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8|CHECKED + lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8|APPLIED + |} + , Some ("4.13.4", "10.22.xs8", "4.13.4", "10.23.xs8") + ) + ; ( {| + ID | status +----------------------------------------+------------ +p_4.13.4-10.22.xs8-4.13.4-10.23.xs8 | CHECKED +p_4.13.4-10.22.xs8-4.13.4-10.23.xs8 | APPLIED + |} + , None + ) + ; ( {| + ID | status | metadata +----------------------------------------+------------+--------------- +lp_4.17.3-3.11.gf717213.xs8-4.17.3-3.12.xs8| CHECKED | +lp_4.17.3-3.11.gf717213.xs8-4.17.3-3.13.xs8| APPLIED | + |} + , Some ("4.17.3", "3.11.gf717213.xs8", "4.17.3", "3.13.xs8") + ) ] end) diff --git a/ocaml/xapi/livepatch.ml b/ocaml/xapi/livepatch.ml index 0dc3869338e..63afa9a2c82 100644 --- a/ocaml/xapi/livepatch.ml +++ b/ocaml/xapi/livepatch.ml @@ -187,29 +187,34 @@ module KernelLivePatch = struct end module XenLivePatch = struct - let get_regexp status = - Re.Posix.compile_pat - (Printf.sprintf {|^[ ]*lp_([^- ]+)-([^- ]+)-([^- ]+)-([^- ]+).+%s.*$|} - status - ) + let drop x = Astring.Char.Ascii.(is_control x || is_white x) - let get_livepatches pattern s = + let get_livepatches state s = + let pattern = + Re.Posix.compile_pat {|^lp_([^- ]+)-([^- ]+)-([^- ]+)-([^- ]+)$|} + in Astring.String.cuts ~sep:"\n" s |> List.filter_map (fun line -> - match Re.exec_opt pattern line with - | Some groups -> - let base_version = Re.Group.get groups 1 in - let base_release = Re.Group.get groups 2 in - let to_version = Re.Group.get groups 3 in - let to_release = Re.Group.get groups 4 in - Some (base_version, base_release, to_version, to_release) - | None -> + Astring.String.cuts ~sep:"|" line + |> List.map (Astring.String.trim ~drop) + |> function + | name :: state' :: _ when state' = state -> ( + match Re.exec_opt pattern name with + | Some groups -> + let base_version = Re.Group.get groups 1 in + let base_release = Re.Group.get groups 2 in + let to_version = Re.Group.get groups 3 in + let to_release = Re.Group.get groups 4 in + Some (base_version, base_release, to_version, to_release) + | None -> + None + ) + | _ -> None ) let get_running_livepatch' s = - let r = get_regexp "APPLIED" in - get_livepatches r s |> get_latest_livepatch + get_livepatches "APPLIED" s |> get_latest_livepatch let get_running_livepatch () = Helpers.call_script !Xapi_globs.xen_livepatch_cmd ["list"] @@ -217,13 +222,9 @@ module XenLivePatch = struct let get_checked_livepatches () = Helpers.call_script !Xapi_globs.xen_livepatch_cmd ["list"] - |> get_livepatches (get_regexp "CHECKED") + |> get_livepatches "CHECKED" let get_base_build_id () = - let drop x = - let open Astring.Char.Ascii in - is_control x || is_blank x || is_white x - in Helpers.call_script !Xapi_globs.xl_cmd ["info"; "build_id"] |> Astring.String.trim ~drop |> function From d50b592d1b98ef2837f7bd5a4b9bf7df4da3aedf Mon Sep 17 00:00:00 2001 From: Lunfan Zhang Date: Tue, 12 Mar 2024 07:13:30 -0400 Subject: [PATCH 94/99] CP-48430 Update the running_domains metrics to count the not paused state domains Signed-off-by: Lunfan Zhang --- ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml b/ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml index 4f696085d6d..3eb708d7a5c 100644 --- a/ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml +++ b/ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml @@ -361,9 +361,11 @@ let dss_loadavg () = ) ] -let count_running_domain domains = +let count_power_state_running_domains domains = List.fold_left - (fun count (dom, _, _) -> if dom.Xenctrl.running then count + 1 else count) + (fun count (dom, _, _) -> + if not dom.Xenctrl.paused then count + 1 else count + ) 0 domains let dss_hostload xc domains = @@ -386,7 +388,7 @@ let dss_hostload xc domains = ) 0 domains in - let running_domains = count_running_domain domains in + let running_domains = count_power_state_running_domains domains in let load_per_cpu = float_of_int load /. float_of_int pcpus in [ From ae318530b9b3dd6af3637d1b1a0c3ea8303e4a46 Mon Sep 17 00:00:00 2001 From: Luca Zhang Date: Thu, 14 Mar 2024 15:29:24 +0800 Subject: [PATCH 95/99] fix typos: priviledges -> privileges Signed-off-by: Luca Zhang --- ocaml/idl/datamodel.ml | 2 +- ocaml/xapi/xapi_session.ml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index 8f452b47069..c8fa2614150 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -136,7 +136,7 @@ module Session = struct call ~flags:[`Session] ~name:"change_password" ~doc: "Change the account password; if your session is authenticated with \ - root priviledges then the old_pwd is validated and the new_pwd is set \ + root privileges then the old_pwd is validated and the new_pwd is set \ regardless" ~params: [ diff --git a/ocaml/xapi/xapi_session.ml b/ocaml/xapi/xapi_session.ml index 455dcef9c55..221498d6f1b 100644 --- a/ocaml/xapi/xapi_session.ml +++ b/ocaml/xapi/xapi_session.ml @@ -1149,8 +1149,8 @@ let change_password ~__context ~old_pwd ~new_pwd = try (* CP-696: only change password if session has is_local_superuser bit set *) (* - CA-13567: If you have root priviledges then we do not authenticate old_pwd; right now, since we only - ever have root priviledges we just comment this out. + CA-13567: If you have root privileges then we do not authenticate old_pwd; right now, since we only + ever have root privileges we just comment this out. begin try From 683a7b99aab8f527b0cc0aa54193fc23ac36b11c Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Sun, 17 Mar 2024 12:14:44 +0000 Subject: [PATCH 96/99] Remove Log_reopen forkexec RPC Not used. It was never implemented. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/lib/fe.ml | 1 - 1 file changed, 1 deletion(-) diff --git a/ocaml/forkexecd/lib/fe.ml b/ocaml/forkexecd/lib/fe.ml index bbad59dbfd2..c18f234803a 100644 --- a/ocaml/forkexecd/lib/fe.ml +++ b/ocaml/forkexecd/lib/fe.ml @@ -24,7 +24,6 @@ type ferpc = | Exec | Execed of int | Finished of process_result - | Log_reopen | Dontwaitpid [@@deriving rpc] From 4f9513ed1398c0200797b7ce28f38687aa08a647 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Sun, 17 Mar 2024 12:18:10 +0000 Subject: [PATCH 97/99] Remove Cancel forkexec RPC Not used. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/lib/fe.ml | 1 - ocaml/forkexecd/src/child.ml | 2 -- 2 files changed, 3 deletions(-) diff --git a/ocaml/forkexecd/lib/fe.ml b/ocaml/forkexecd/lib/fe.ml index c18f234803a..1a176a62baa 100644 --- a/ocaml/forkexecd/lib/fe.ml +++ b/ocaml/forkexecd/lib/fe.ml @@ -20,7 +20,6 @@ type process_result = WEXITED of int | WSIGNALED of int | WSTOPPED of int type ferpc = | Setup of setup_cmd | Setup_response of setup_response - | Cancel | Exec | Execed of int | Finished of process_result diff --git a/ocaml/forkexecd/src/child.ml b/ocaml/forkexecd/src/child.ml index f6ede6c608d..197f3b91f65 100644 --- a/ocaml/forkexecd/src/child.ml +++ b/ocaml/forkexecd/src/child.ml @@ -49,8 +49,6 @@ let handle_fd_sock fd_sock state = let handle_comms_sock comms_sock state = let call = Fecomms.read_raw_rpc comms_sock in match call with - | Ok Fe.Cancel -> - debug "Cancel" ; raise Cancelled | Ok Fe.Exec -> debug "Exec" ; {state with finished= true} From fbdf7331aa7747a9779930bcdac7968c6cc4b62a Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 14 Mar 2024 14:27:50 +0000 Subject: [PATCH 98/99] CA-390109: Use `$PROFILE` path to store and read known cert list Before these changes, the `SaveCertificates` method relied on machines having a `SpecialFolder.MyDocuments` folder. This is true in Windows and GUI versions of some Linux distros, but it's an assumption that caused the save method to fail if the folder didn't exist. With this commit, we're storing and reading from the path where `$PROFILE` is stored, which is platform agnostic from the point of view of the SDK. Signed-off-by: Danilo Del Busso --- .../autogen/Initialize-Environment.ps1 | 2 ++ .../autogen/src/CommonCmdletFunctions.cs | 29 +++++++++++++------ .../autogen/src/Connect-XenServer.cs | 8 ++--- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/ocaml/sdk-gen/powershell/autogen/Initialize-Environment.ps1 b/ocaml/sdk-gen/powershell/autogen/Initialize-Environment.ps1 index d418745ee39..c0d7b30dce3 100644 --- a/ocaml/sdk-gen/powershell/autogen/Initialize-Environment.ps1 +++ b/ocaml/sdk-gen/powershell/autogen/Initialize-Environment.ps1 @@ -46,4 +46,6 @@ if (Test-Path $perUserXsProfile) { Remove-Item variable:systemWideXsProfile Remove-Item variable:perUserXsProfile +$global:KnownServerCertificatesFilePath = Join-Path -Path (Split-Path $PROFILE) -ChildPath "XenServer_Known_Certificates.xml" + $XenServer_Environment_Initialized = $true diff --git a/ocaml/sdk-gen/powershell/autogen/src/CommonCmdletFunctions.cs b/ocaml/sdk-gen/powershell/autogen/src/CommonCmdletFunctions.cs index d01a03098cb..8f29ecde1f5 100644 --- a/ocaml/sdk-gen/powershell/autogen/src/CommonCmdletFunctions.cs +++ b/ocaml/sdk-gen/powershell/autogen/src/CommonCmdletFunctions.cs @@ -42,8 +42,10 @@ namespace Citrix.XenServer class CommonCmdletFunctions { private const string SessionsVariable = "global:Citrix.XenServer.Sessions"; + private const string DefaultSessionVariable = "global:XenServer_Default_Session"; - private static string CertificatePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"WindowsPowerShell\XenServer_Known_Certificates.xml"); + + private const string KnownServerCertificatesFilePathVariable = "global:KnownServerCertificatesFilePath"; static CommonCmdletFunctions() { @@ -68,8 +70,7 @@ internal static void SetAllSessions(PSCmdlet cmdlet, Dictionary internal static Session GetDefaultXenSession(PSCmdlet cmdlet) { - object obj = cmdlet.SessionState.PSVariable.GetValue(DefaultSessionVariable); - return obj as Session; + return cmdlet.SessionState.PSVariable.GetValue(DefaultSessionVariable) as Session; } internal static void SetDefaultXenSession(PSCmdlet cmdlet, Session session) @@ -77,19 +78,28 @@ internal static void SetDefaultXenSession(PSCmdlet cmdlet, Session session) cmdlet.SessionState.PSVariable.Set(DefaultSessionVariable, session); } + internal static string GetKnownServerCertificatesFilePathVariable(PSCmdlet cmdlet) + { + var knownCertificatesFilePathObject = cmdlet.SessionState.PSVariable.GetValue(KnownServerCertificatesFilePathVariable); + if (knownCertificatesFilePathObject is PSObject psObject) + return psObject.BaseObject as string; + return knownCertificatesFilePathObject?.ToString() ?? string.Empty; + } + internal static string GetUrl(string hostname, int port) { return string.Format("{0}://{1}:{2}", port == 80 ? "http" : "https", hostname, port); } - public static Dictionary LoadCertificates() + public static Dictionary LoadCertificates(PSCmdlet cmdlet) { Dictionary certificates = new Dictionary(); + var knownServerCertificatesFilePath = GetKnownServerCertificatesFilePathVariable(cmdlet); - if (File.Exists(CertificatePath)) + if (File.Exists(knownServerCertificatesFilePath)) { XmlDocument doc = new XmlDocument(); - doc.Load(CertificatePath); + doc.Load(knownServerCertificatesFilePath); foreach (XmlNode node in doc.GetElementsByTagName("certificate")) { @@ -104,9 +114,10 @@ public static Dictionary LoadCertificates() return certificates; } - public static void SaveCertificates(Dictionary certificates) + public static void SaveCertificates(PSCmdlet cmdlet, Dictionary certificates) { - string dirName = Path.GetDirectoryName(CertificatePath); + var knownServerCertificatesFilePath = GetKnownServerCertificatesFilePathVariable(cmdlet); + string dirName = Path.GetDirectoryName(knownServerCertificatesFilePath); if (!Directory.Exists(dirName)) Directory.CreateDirectory(dirName); @@ -129,7 +140,7 @@ public static void SaveCertificates(Dictionary certificates) } doc.AppendChild(node); - doc.Save(CertificatePath); + doc.Save(knownServerCertificatesFilePath); } public static string FingerprintPrettyString(string fingerprint) diff --git a/ocaml/sdk-gen/powershell/autogen/src/Connect-XenServer.cs b/ocaml/sdk-gen/powershell/autogen/src/Connect-XenServer.cs index 0ec80444a85..c1155f7a2a3 100644 --- a/ocaml/sdk-gen/powershell/autogen/src/Connect-XenServer.cs +++ b/ocaml/sdk-gen/powershell/autogen/src/Connect-XenServer.cs @@ -253,9 +253,9 @@ protected override void ProcessRecord() private void AddCertificate(string hostname, string fingerprint) { - var certificates = CommonCmdletFunctions.LoadCertificates(); + var certificates = CommonCmdletFunctions.LoadCertificates(this); certificates[hostname] = fingerprint; - CommonCmdletFunctions.SaveCertificates(certificates); + CommonCmdletFunctions.SaveCertificates(this, certificates); } private bool ValidateServerCertificate(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors) @@ -274,7 +274,7 @@ private bool ValidateServerCertificate(object sender, X509Certificate certificat bool trusted = VerifyInAllStores(new X509Certificate2(certificate)); - var certificates = CommonCmdletFunctions.LoadCertificates(); + var certificates = CommonCmdletFunctions.LoadCertificates(this); if (certificates.ContainsKey(hostname)) { @@ -292,7 +292,7 @@ private bool ValidateServerCertificate(object sender, X509Certificate certificat } certificates[hostname] = fingerprint; - CommonCmdletFunctions.SaveCertificates(certificates); + CommonCmdletFunctions.SaveCertificates(this, certificates); return true; } } From 8af3aa54e2dba2ef0711994ed8dbd10e69a85f42 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 14 Mar 2024 14:31:25 +0000 Subject: [PATCH 99/99] Fix typo in `XenServerPowerShell.csproj` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj b/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj index 85ea0dc72b4..23fff01346e 100644 --- a/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj +++ b/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj @@ -6,7 +6,7 @@ True - + true