From 5a2ddefeb7b43726f9dee326280b511d3ed2b49b Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 29 Oct 2024 22:29:38 +0100 Subject: [PATCH 01/15] Initial commit --- Directory.Build.props | 2 +- Remote.sln | 7 + sample/docker-compose.yml | 9 + setup/README.md | 18 - setup/docker/docker-compose.yml | 27 -- setup/docker/dotnet/satellite.sh | 27 -- setup/docker/python/satellite.sh | 34 -- setup/docker/run-user.sh | 39 --- setup/docker/run.sh | 52 --- setup/docker/setup-host.sh | 22 -- setup/docker/setup-main.sh | 19 -- setup/docker/setup-satellite.sh | 22 -- .../PackageReferencesController.cs | 82 +++++ .../Controllers/SourcesController.cs | 59 ++++ src/Nexus.Agent/Core/Agent.cs | 156 +++++++++ src/Nexus.Agent/Core/CustomExtensions.cs | 16 + src/Nexus.Agent/Core/DatabaseService.cs | 53 +++ src/Nexus.Agent/Core/Models_Public_v1.cs | 30 ++ src/Nexus.Agent/Core/Types.cs | 23 ++ src/Nexus.Agent/Program.cs | 47 +++ .../Properties/launchSettings.json | 13 + src/Nexus.Agent/appsettings.Development.json | 8 + src/Nexus.Agent/appsettings.json | 9 + src/Nexus.Agent/nexus-agent.csproj | 29 ++ src/Nexus.Sources.Remote/DataSourceTypes.cs | 7 +- .../RemoteCommunicator.cs | 312 ++++-------------- src/remoting/dotnet-remoting/Remoting.cs | 129 +++++--- 27 files changed, 694 insertions(+), 557 deletions(-) create mode 100644 sample/docker-compose.yml delete mode 100644 setup/README.md delete mode 100644 setup/docker/docker-compose.yml delete mode 100644 setup/docker/dotnet/satellite.sh delete mode 100644 setup/docker/python/satellite.sh delete mode 100644 setup/docker/run-user.sh delete mode 100644 setup/docker/run.sh delete mode 100644 setup/docker/setup-host.sh delete mode 100644 setup/docker/setup-main.sh delete mode 100644 setup/docker/setup-satellite.sh create mode 100644 src/Nexus.Agent/Controllers/PackageReferencesController.cs create mode 100644 src/Nexus.Agent/Controllers/SourcesController.cs create mode 100644 src/Nexus.Agent/Core/Agent.cs create mode 100644 src/Nexus.Agent/Core/CustomExtensions.cs create mode 100644 src/Nexus.Agent/Core/DatabaseService.cs create mode 100644 src/Nexus.Agent/Core/Models_Public_v1.cs create mode 100644 src/Nexus.Agent/Core/Types.cs create mode 100644 src/Nexus.Agent/Program.cs create mode 100644 src/Nexus.Agent/Properties/launchSettings.json create mode 100644 src/Nexus.Agent/appsettings.Development.json create mode 100644 src/Nexus.Agent/appsettings.json create mode 100644 src/Nexus.Agent/nexus-agent.csproj diff --git a/Directory.Build.props b/Directory.Build.props index db678da..eeb7e32 100755 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ true - net8.0 + net9.0 enable enable true diff --git a/Remote.sln b/Remote.sln index b95dc25..5455152 100644 --- a/Remote.sln +++ b/Remote.sln @@ -19,6 +19,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "benchmarks", "benchmarks", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nexus.Benchmarks", "benchmarks\Nexus.Benchmarks\Nexus.Benchmarks.csproj", "{B602750B-5E89-453F-841B-8D7F22DE58EB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "nexus-agent", "src\Nexus.Agent\nexus-agent.csproj", "{71F41120-5BC2-4684-ADB7-C8C97CCE8EAF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -41,6 +43,10 @@ Global {B602750B-5E89-453F-841B-8D7F22DE58EB}.Debug|Any CPU.Build.0 = Debug|Any CPU {B602750B-5E89-453F-841B-8D7F22DE58EB}.Release|Any CPU.ActiveCfg = Release|Any CPU {B602750B-5E89-453F-841B-8D7F22DE58EB}.Release|Any CPU.Build.0 = Release|Any CPU + {71F41120-5BC2-4684-ADB7-C8C97CCE8EAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {71F41120-5BC2-4684-ADB7-C8C97CCE8EAF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {71F41120-5BC2-4684-ADB7-C8C97CCE8EAF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {71F41120-5BC2-4684-ADB7-C8C97CCE8EAF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -51,6 +57,7 @@ Global {59CBCE56-7832-4B55-854F-C9B5E4FBEA86} = {F226553B-2E5B-4910-AF31-0B718872AF18} {FC047115-277B-448E-854F-275B5B8343A5} = {59CBCE56-7832-4B55-854F-C9B5E4FBEA86} {B602750B-5E89-453F-841B-8D7F22DE58EB} = {40C5CF08-E45E-4F25-BE8F-B5BBD4B0EDB3} + {71F41120-5BC2-4684-ADB7-C8C97CCE8EAF} = {F226553B-2E5B-4910-AF31-0B718872AF18} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B035E550-8A68-427B-92F1-9B1A6977BF8F} diff --git a/sample/docker-compose.yml b/sample/docker-compose.yml new file mode 100644 index 0000000..4391526 --- /dev/null +++ b/sample/docker-compose.yml @@ -0,0 +1,9 @@ +version: "3.7" + +services: + + nexus-agent: + container_name: nexus-agent + image: mcr.microsoft.com/dotnet/sdk:9.0 + entrypoint: bash -c "git clone https://github.com/nexus-main/nexus-sources-remote; dotnet run --project src/Nexus.Agent/Nexus.Agent.csproj" + restart: always diff --git a/setup/README.md b/setup/README.md deleted file mode 100644 index 9590aff..0000000 --- a/setup/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# How to run the test: - -1. `cd` into the `setup/docker` folder - -2. Run the follow on the docker host: - -```sh -sudo bash setup-host.sh python -sudo bash setup-host.sh dotnet -sudo docker-compose up -d -``` - -3. Wait a few seconds until the containers are running, then test the connection between both containers: - -```sh -sudo docker exec nexus-main \ - bash -c "cd /root/nexus-sources-remote; dotnet test --filter Nexus.Sources.Tests.SetupDockerTests" -``` \ No newline at end of file diff --git a/setup/docker/docker-compose.yml b/setup/docker/docker-compose.yml deleted file mode 100644 index 1e7b3f9..0000000 --- a/setup/docker/docker-compose.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: "3.7" - -services: - - main: - container_name: nexus-main - image: mcr.microsoft.com/dotnet/sdk:8.0 - volumes: - - /var/lib/nexus/docker/nexus-main/.ssh:/root/.ssh - entrypoint: bash -c "cd; curl -s -O 'https://raw.githubusercontent.com/nexus-main/nexus-sources-remote/master/setup/docker/setup-main.sh'; source 'setup-main.sh'" - restart: always - - satellite-python: - container_name: nexus-python - image: python:3.9.0 - volumes: - - /var/lib/nexus/docker/nexus-python/.ssh:/root/.ssh - entrypoint: bash -c "satellite_id=python;cd; curl -s -O 'https://raw.githubusercontent.com/nexus-main/nexus-sources-remote/master/setup/docker/setup-satellite.sh'; source 'setup-satellite.sh'" - restart: always - - satellite-nexus: - container_name: nexus-dotnet - image: mcr.microsoft.com/dotnet/sdk:8.0 - volumes: - - /var/lib/nexus/docker/nexus-dotnet/.ssh:/root/.ssh - entrypoint: bash -c "satellite_id=dotnet;cd; curl -s -O 'https://raw.githubusercontent.com/nexus-main/nexus-sources-remote/master/setup/docker/setup-satellite.sh'; source 'setup-satellite.sh'" - restart: always diff --git a/setup/docker/dotnet/satellite.sh b/setup/docker/dotnet/satellite.sh deleted file mode 100644 index 5f16811..0000000 --- a/setup/docker/dotnet/satellite.sh +++ /dev/null @@ -1,27 +0,0 @@ -red=$'\e[0;31m' -green=$'\e[0;32m' -orange=$'\e[0;33m' -white=$'\e[0m' - -echo "${green}Welcome to the dotnet satellite.sh script!${white}" -project=$3 - -if [[ -f "../tag_changed" || ! -f "../build_successful" ]]; then - - echo "Build project ${green}${project}${white}" - dotnet build -c Release ${project} - - if [[ $? == 0 ]]; then - touch "../build_successful" - else - echo "${red}Build failed${white}" - rm --force "../build_successful" - fi -fi - -# run user code -shift -shift -shift -echo "Run command ${green}dotnet run -c Release --no-build --project ${project} -- $@${white}" -dotnet run -c Release --no-build --project ${project} -- $@ diff --git a/setup/docker/python/satellite.sh b/setup/docker/python/satellite.sh deleted file mode 100644 index 2bdfdb0..0000000 --- a/setup/docker/python/satellite.sh +++ /dev/null @@ -1,34 +0,0 @@ -green=$'\e[0;32m' -orange=$'\e[0;33m' -white=$'\e[0m' - -echo "${green}Welcome to the python satellite.sh script!${white}" - -# virtual environment (!!DO NOT QUOTE TO ALLOW TILDE EXPANSION!!) -env=~/venv - -if [ ! -d $env ]; then - echo "Create virtual environment ${green}${env}${white}" - python3 -m venv $env -fi - -echo "Activate virtual environment ${green}${env}${white}" -source $env/bin/activate - -# requirements -if [ -f "requirements.txt" ]; then - - if [ -f "../tag_changed" ]; then - echo "${green}Install requirements${white}" - python -m pip install --pre --index-url https://pypi.python.org/simple --extra-index-url https://www.myget.org/F/apollo3zehn-dev/python/ -r "requirements.txt" --disable-pip-version-check - fi - -else - echo "${orange}No requirements.txt found${white}" -fi - -# run user code -shift -shift -echo "Run command ${green}python $@${white}" -python $@ diff --git a/setup/docker/run-user.sh b/setup/docker/run-user.sh deleted file mode 100644 index 0f67103..0000000 --- a/setup/docker/run-user.sh +++ /dev/null @@ -1,39 +0,0 @@ -green=$'\e[0;32m' -orange=$'\e[0;33m' -white=$'\e[0m' - -echo "Continue as: ${green}$(whoami)${white}" - -# check if git project exists -if [[ ! -d 'repository' ]]; then - mkdir -p 'repository' -fi - -cd 'repository' - -( - clone_required=false - - if [ -d '.git' ]; then - current_tag=$(git describe --tags) - echo "Current tag is ${green}${current_tag}${white}" - - if [ "$current_tag" != "$2" ]; then - rm --force -r .* * 2> /dev/null - clone_required=true - fi - else - rm --force -r .* * 2> /dev/null - clone_required=true - fi - - if [[ "$clone_required" = true ]]; then - echo "Clone repository ${orange}$1${white} @ ${orange}$2${white}" - git clone -c advice.detachedHead=false --depth 1 --branch $2 $1 . - touch "../tag_changed" - else - rm --force "../tag_changed" - fi -) 100>"/tmp/run-user-$(whoami).lock" - -source "../satellite.sh" diff --git a/setup/docker/run.sh b/setup/docker/run.sh deleted file mode 100644 index 72cb823..0000000 --- a/setup/docker/run.sh +++ /dev/null @@ -1,52 +0,0 @@ -red=$'\e[0;31m' -green=$'\e[0;32m' -orange=$'\e[0;33m' -white=$'\e[0m' - -locked() -{ - >&2 echo "${red}Unable to acquire the file lock${white}" - exit 1 -} - -if (( $# < 2 )); then - >&2 echo "${red}Illegal number of parameters${white}" - exit 1 -fi - -echo "The git-url is: ${green}$1${white}" - -# derive user id -user_id=$(echo -n $1 | openssl dgst -binary -md5 | openssl base64) -user_id=${user_id//[\/=]/_} -echo "Derived user id: ${green}$user_id${white}" - -( - # get lock (wait max 10 s) - flock -w 10 100 || locked - - # get or add user - if id $user_id &>/dev/null; then - echo "User ${green}exists${white}" - # password=$(<"password-store/$user_id") - else - echo "User ${orange}does not exist${white}" - password=$(cat /dev/urandom | tr -dc a-zA-Z0-9 | fold -w 14 | head -n 1) - mkdir --parents "password-store" - echo $password > "password-store/$user_id" - useradd --password $password $user_id - echo "Created user ${green}$user_id${white}" - fi - - # prepare user folder - mkdir --parents "/home/$user_id" - cp --force "run-user.sh" "/home/$user_id/run-user.sh" - cp --force "satellite.sh" "/home/$user_id/satellite.sh" - chown --recursive $user_id:$user_id "/home/$user_id" - -) 100>"/tmp/run-$user_id.lock" - -# continue as $user_id -cd "/home/$user_id" -command="bash run-user.sh $@" -su $user_id -c "$command" \ No newline at end of file diff --git a/setup/docker/setup-host.sh b/setup/docker/setup-host.sh deleted file mode 100644 index 4eec85c..0000000 --- a/setup/docker/setup-host.sh +++ /dev/null @@ -1,22 +0,0 @@ -green=$'\e[0;32m' -white=$'\e[0m' - -satellite_id=$1 - -# Generate key for main container and add config file, but do not override if it already exists -main_folder='/var/lib/nexus/docker/nexus-main' - -if [[ ! -d ${main_folder}/.ssh ]]; then - echo "Generate SSH key for container ${green}nexus-main${white}" - mkdir -p "${main_folder}/.ssh" - ssh-keygen -q -t rsa -N '' -f "${main_folder}/.ssh/id_rsa" <</dev/null 2>&1 - echo "StrictHostKeyChecking no" > "${main_folder}/.ssh/config" - echo "UserKnownHostsFile=/dev/null" >> "${main_folder}/.ssh/config" -fi - -# Generate key for satellite container and add main container key to authorized keys file -echo "Generate SSH key for container ${green}nexus-${satellite_id}${white}" -satellite_folder="/var/lib/nexus/docker/nexus-${satellite_id}" -mkdir -p "${satellite_folder}/.ssh" -ssh-keygen -q -t rsa -N '' -f "${satellite_folder}/.ssh/id_rsa" <</dev/null 2>&1 -cat "${main_folder}/.ssh/id_rsa.pub" > "${satellite_folder}/.ssh/authorized_keys" diff --git a/setup/docker/setup-main.sh b/setup/docker/setup-main.sh deleted file mode 100644 index 37454ac..0000000 --- a/setup/docker/setup-main.sh +++ /dev/null @@ -1,19 +0,0 @@ -set -e - -green=$'\e[0;32m' -white=$'\e[0m' -echo "${green}Welcome to the setup-main.sh script"'!'"${white}" - -echo "${green}Set up SSH client${white}" -apt update -apt install openssh-client -y - -echo "Clone repository ${green}https://github.com/nexus-main/nexus-sources-remote${white}" -git clone https://github.com/nexus-main/nexus-sources-remote -cd nexus-sources-remote - -mkdir -p /var/lib/nexus -touch "/var/lib/nexus/ready" - -trap : TERM INT -sleep infinity & wait diff --git a/setup/docker/setup-satellite.sh b/setup/docker/setup-satellite.sh deleted file mode 100644 index f9c1907..0000000 --- a/setup/docker/setup-satellite.sh +++ /dev/null @@ -1,22 +0,0 @@ -set -e - -green=$'\e[0;32m' -white=$'\e[0m' -echo "${green}Welcome to the setup-${satellite_id}.sh script"'!'"${white}" - -echo "${green}Set up SSH server${white}" -apt update -apt install openssh-server -y -sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' '/etc/ssh/sshd_config' -service ssh start - -echo "${green}Load run.sh, run-user.sh and ${satellite_id}/satellite.sh scripts${white}" -curl -s -O 'https://raw.githubusercontent.com/nexus-main/nexus-sources-remote/master/setup/docker/run.sh' -curl -s -O 'https://raw.githubusercontent.com/nexus-main/nexus-sources-remote/master/setup/docker/run-user.sh' -curl -s -O "https://raw.githubusercontent.com/nexus-main/nexus-sources-remote/master/setup/docker/${satellite_id}/satellite.sh" - -mkdir -p /var/lib/nexus -touch "/var/lib/nexus/ready" - -trap : TERM INT -sleep infinity & wait diff --git a/src/Nexus.Agent/Controllers/PackageReferencesController.cs b/src/Nexus.Agent/Controllers/PackageReferencesController.cs new file mode 100644 index 0000000..685005d --- /dev/null +++ b/src/Nexus.Agent/Controllers/PackageReferencesController.cs @@ -0,0 +1,82 @@ +// MIT License +// Copyright (c) [2024] [nexus-main] + +using Asp.Versioning; +using Microsoft.AspNetCore.Mvc; +using Nexus.Core.V1; +using Nexus.Services; + +namespace Nexus.Controllers; + +/// +/// Provides access to package references. +/// +[ApiController] +[ApiVersion("1.0")] +[Route("api/v{version:apiVersion}/[controller]")] +internal class PackageReferencesController( + IPackageService packageService, + IExtensionHive extensionHive) : ControllerBase +{ + // GET /api/packagereferences + // POST /api/packagereferences + // DELETE /api/packagereferences/{id} + // GET /api/packagereferences/{id}/versions + + private readonly IPackageService _packageService = packageService; + + private readonly IExtensionHive _extensionHive = extensionHive; + + /// + /// Gets the list of package references. + /// + /// + [HttpGet] + public async Task> GetAsync() + { + return await _packageService.GetAllAsync(); + } + + /// + /// Creates a package reference. + /// + /// The package reference to create. + [HttpPost] + public Task CreateAsync( + [FromBody] PackageReference packageReference) + { + return _packageService.PutAsync(packageReference); + } + + /// + /// Deletes a package reference. + /// + /// The ID of the package reference. + [HttpDelete("{id}")] + public Task DeleteAsync( + Guid id) + { + return _packageService.DeleteAsync(id); + } + + /// + /// Gets package versions. + /// + /// The ID of the package reference. + /// A token to cancel the current operation. + [HttpGet("{id}/versions")] + public async Task> GetVersionsAsync( + Guid id, + CancellationToken cancellationToken) + { + var packageReferenceMap = await _packageService.GetAllAsync(); + + if (!packageReferenceMap.TryGetValue(id, out var packageReference)) + return NotFound($"Unable to find package reference with ID {id}."); + + var result = await _extensionHive + .GetVersionsAsync(packageReference, cancellationToken); + + return result; + } +} diff --git a/src/Nexus.Agent/Controllers/SourcesController.cs b/src/Nexus.Agent/Controllers/SourcesController.cs new file mode 100644 index 0000000..adf0fb6 --- /dev/null +++ b/src/Nexus.Agent/Controllers/SourcesController.cs @@ -0,0 +1,59 @@ +// MIT License +// Copyright (c) [2024] [nexus-main] + +using Asp.Versioning; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Nexus.Core.V1; +using Nexus.Extensibility; +using Nexus.Services; +using System.Reflection; + +namespace Nexus.Controllers; + +/// +/// Provides access to extensions. +/// +[Authorize] +[ApiController] +[ApiVersion("1.0")] +[Route("api/v{version:apiVersion}/[controller]")] +internal class SourcesController( + IExtensionHive extensionHive +) : ControllerBase +{ + // GET /api/sources/descriptions + + private readonly IExtensionHive _extensionHive = extensionHive; + + /// + /// Gets the list of source descriptions. + /// + [HttpGet("descriptions")] + public List GetDescriptions() + { + var result = GetExtensionDescriptions(_extensionHive.GetExtensions()); + return result; + } + + private static List GetExtensionDescriptions( + IEnumerable extensions) + { + return extensions.Select(type => + { + var version = type.Assembly + .GetCustomAttribute()! + .InformationalVersion; + + var attribute = type + .GetCustomAttribute(inherit: false); + + if (attribute is null) + return new ExtensionDescription(type.FullName!, version, default, default, default, default); + + else + return new ExtensionDescription(type.FullName!, version, attribute.Description, attribute.ProjectUrl, attribute.RepositoryUrl, default); + }) + .ToList(); + } +} diff --git a/src/Nexus.Agent/Core/Agent.cs b/src/Nexus.Agent/Core/Agent.cs new file mode 100644 index 0000000..d186c20 --- /dev/null +++ b/src/Nexus.Agent/Core/Agent.cs @@ -0,0 +1,156 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Nexus.Core; +using Nexus.Extensibility; +using Nexus.Remoting; +using Nexus.Services; + +namespace Nexus.Agent; + +public class Agent +{ + private readonly ConcurrentDictionary _tcpClientPairs = new(); + + public async Task RunAsync() + { + var extensionHive = await LoadPackagesAsync(); + await AcceptClientsAsync(extensionHive); + } + + private async Task LoadPackagesAsync() + { + var pathsOptions = Options.Create(new PathsOptions()); + var loggerFactory = new LoggerFactory(); + + var databaseService = new DatabaseService(pathsOptions); + var packageService = new PackageService(databaseService); + + var extensionHive = new ExtensionHive( + pathsOptions, + NullLogger.Instance, loggerFactory + ); + + var packageReferenceMap = await packageService.GetAllAsync(); + var progress = new Progress(); + + await extensionHive.LoadPackagesAsync( + packageReferenceMap: packageReferenceMap, + progress, + CancellationToken.None + ); + + return extensionHive; + } + + private Task AcceptClientsAsync(IExtensionHive extensionHive) + { + var tcpListenerComm = new TcpListener(IPAddress.Any, 56145); + + return Task.Run(async () => + { + while (true) + { + var client = await tcpListenerComm.AcceptTcpClientAsync(); + + _ = Task.Run(async () => + { + var streamReadCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + using var stream = client.GetStream(); + + // get connection id + var buffer1 = new byte[36]; + await stream.ReadExactlyAsync(buffer1, streamReadCts.Token); + var idString = Encoding.UTF8.GetString(buffer1); + + // get connection type + var buffer2 = new byte[4]; + await stream.ReadExactlyAsync(buffer2, streamReadCts.Token); + var typeString = Encoding.UTF8.GetString(buffer2); + + if (Guid.TryParse(Encoding.UTF8.GetString(buffer1), out var id)) + { + var tcpPairCts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Handle the timeout event + tcpPairCts.Token.Register(() => + { + // If TCP client pair can be found ... + if (_tcpClientPairs.TryGetValue(id, out var pair)) + { + // and if TCP client pair is not yet complete ... + if (pair.Comm is null || pair.Data is null) + { + // then dispose and remove the clients and the pair + pair.Comm?.Dispose(); + pair.Data?.Dispose(); + + _tcpClientPairs.Remove(id, out _); + } + } + }); + + // We got a "comm" tcp connection + if (typeString == "comm") + { + _tcpClientPairs.AddOrUpdate( + id, + addValueFactory: id => new TcpClientPair { Comm = client }, + updateValueFactory: (id, pair) => + { + pair.Comm?.Dispose(); + pair.Comm = client; + return pair; + } + ); + } + + // We got a "data" tcp connection + else if (typeString == "data") + { + _tcpClientPairs.AddOrUpdate( + id, + addValueFactory: id => new TcpClientPair { Data = client }, + updateValueFactory: (id, pair) => + { + pair.Data?.Dispose(); + pair.Data = client; + return pair; + } + ); + } + + // Something went wrong, dispose the client + else + { + client.Dispose(); + } + + var pair = _tcpClientPairs[id]; + + if (pair.Comm is not null && pair.Data is not null) + { + pair.RemoteCommunicator = new RemoteCommunicator( + pair.Comm, + pair.Data, + getDataSource: type => extensionHive.GetInstance(type) + ); + } + } + }); + } + }); + } +} + +public class TcpClientPair +{ + public TcpClient? Comm { get; set; } + + public TcpClient? Data { get; set; } + + public RemoteCommunicator? RemoteCommunicator { get; set; } +} \ No newline at end of file diff --git a/src/Nexus.Agent/Core/CustomExtensions.cs b/src/Nexus.Agent/Core/CustomExtensions.cs new file mode 100644 index 0000000..4a75256 --- /dev/null +++ b/src/Nexus.Agent/Core/CustomExtensions.cs @@ -0,0 +1,16 @@ +// MIT License +// Copyright (c) [2024] [nexus-main] + +using System.Security.Cryptography; +using System.Text; + +namespace Nexus.Core; + +internal static class CustomExtensions +{ + public static byte[] Hash(this string value) + { + var hash = MD5.HashData(Encoding.UTF8.GetBytes(value)); + return hash; + } +} diff --git a/src/Nexus.Agent/Core/DatabaseService.cs b/src/Nexus.Agent/Core/DatabaseService.cs new file mode 100644 index 0000000..49d06e3 --- /dev/null +++ b/src/Nexus.Agent/Core/DatabaseService.cs @@ -0,0 +1,53 @@ +// MIT License +// Copyright (c) [2024] [nexus-main] + +using Microsoft.Extensions.Options; +using Nexus.Core; +using System.Diagnostics.CodeAnalysis; + +namespace Nexus.Services; + +internal interface IDatabaseService +{ + /* /config/packages.json */ + bool TryReadPackageReferenceMap([NotNullWhen(true)] out string? packageReferenceMap); + + Stream WritePackageReferenceMap(); +} + +internal class DatabaseService(IOptions pathsOptions) + : IDatabaseService +{ + private readonly PathsOptions _pathsOptions = pathsOptions.Value; + + private const string FILE_EXTENSION = ".json"; + + private const string PACKAGES = "packages"; + + /* /config/packages.json */ + public bool TryReadPackageReferenceMap([NotNullWhen(true)] out string? packageReferenceMap) + { + var folderPath = _pathsOptions.Config; + var packageReferencesFilePath = Path.Combine(folderPath, PACKAGES + FILE_EXTENSION); + + packageReferenceMap = default; + + if (File.Exists(packageReferencesFilePath)) + { + packageReferenceMap = File.ReadAllText(packageReferencesFilePath); + return true; + } + + return false; + } + + public Stream WritePackageReferenceMap() + { + var folderPath = _pathsOptions.Config; + var packageReferencesFilePath = Path.Combine(folderPath, PACKAGES + FILE_EXTENSION); + + Directory.CreateDirectory(folderPath); + + return File.Open(packageReferencesFilePath, FileMode.Create, FileAccess.Write); + } +} \ No newline at end of file diff --git a/src/Nexus.Agent/Core/Models_Public_v1.cs b/src/Nexus.Agent/Core/Models_Public_v1.cs new file mode 100644 index 0000000..835a77b --- /dev/null +++ b/src/Nexus.Agent/Core/Models_Public_v1.cs @@ -0,0 +1,30 @@ +using System.Text.Json; + +namespace Nexus.Core.V1; + +/// +/// A package reference. +/// +/// The provider which loads the package. +/// The configuration of the package reference. +public record PackageReference( + string Provider, + Dictionary Configuration +); + +/// +/// An extension description. +/// +/// The extension type. +/// The extension version. +/// A nullable description. +/// A nullable project website URL. +/// A nullable source repository URL. +/// Additional information about the extension. +public record ExtensionDescription( + string Type, + string Version, + string? Description, + string? ProjectUrl, + string? RepositoryUrl, + IReadOnlyDictionary? AdditionalInformation); \ No newline at end of file diff --git a/src/Nexus.Agent/Core/Types.cs b/src/Nexus.Agent/Core/Types.cs new file mode 100644 index 0000000..7e9cda6 --- /dev/null +++ b/src/Nexus.Agent/Core/Types.cs @@ -0,0 +1,23 @@ +using System.Runtime.InteropServices; +using System.Text.Json; + +namespace Nexus.Core; + +internal record PathsOptions +{ + public const string Section = "Paths"; + + public string Config { get; set; } = Path.Combine(PlatformSpecificRoot, "config"); + + public string Packages { get; set; } = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nexus", "packages"); + // GetGlobalPackagesFolder: https://github.com/NuGet/NuGet.Client/blob/0fc58e13683565e7bdf30e706d49e58fc497bbed/src/NuGet.Core/NuGet.Configuration/Utility/SettingsUtility.cs#L225-L254 + // GetFolderPath: https://github.com/NuGet/NuGet.Client/blob/1d75910076b2ecfbe5f142227cfb4fb45c093a1e/src/NuGet.Core/NuGet.Common/PathUtil/NuGetEnvironment.cs#L54-L57 + + #region Support + + private static string PlatformSpecificRoot { get; } = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Nexus.Agent") + : "/var/lib/nexus-agent"; + + #endregion +} \ No newline at end of file diff --git a/src/Nexus.Agent/Program.cs b/src/Nexus.Agent/Program.cs new file mode 100644 index 0000000..3aff902 --- /dev/null +++ b/src/Nexus.Agent/Program.cs @@ -0,0 +1,47 @@ +using Asp.Versioning; +using Nexus.Agent; +using Nexus.Core; +using Nexus.Services; +using Scalar.AspNetCore; + +var builder = WebApplication.CreateBuilder(); + +builder.Services + + .AddOpenApi() + // .AddOpenApi("v2") + + .AddApiVersioning(config => + { + config.ReportApiVersions = true; + config.ApiVersionReader = new UrlSegmentApiVersionReader(); + }) + + .AddApiExplorer(config => + { + config.GroupNameFormat = "'v'VVV"; + config.SubstituteApiVersionInUrl = true; + }); + +builder.Services + .AddControllers() + .ConfigureApplicationPartManager( + manager => manager.FeatureProviders.Add(new InternalControllerFeatureProvider()) + ); + +builder.Services + .AddSingleton() + .AddSingleton() + .AddSingleton(); + +var app = builder.Build(); + +app.MapGet("/", () => Results.Redirect("/scalar/v1")).ExcludeFromDescription(); +app.MapOpenApi(); +app.MapScalarApiReference(); +app.MapControllers(); + +var agent = new Agent(); +_ = agent.RunAsync(); + +app.Run(); diff --git a/src/Nexus.Agent/Properties/launchSettings.json b/src/Nexus.Agent/Properties/launchSettings.json new file mode 100644 index 0000000..24838d3 --- /dev/null +++ b/src/Nexus.Agent/Properties/launchSettings.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "applicationUrl": "http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/Nexus.Agent/appsettings.Development.json b/src/Nexus.Agent/appsettings.Development.json new file mode 100644 index 0000000..ff66ba6 --- /dev/null +++ b/src/Nexus.Agent/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/src/Nexus.Agent/appsettings.json b/src/Nexus.Agent/appsettings.json new file mode 100644 index 0000000..4d56694 --- /dev/null +++ b/src/Nexus.Agent/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/src/Nexus.Agent/nexus-agent.csproj b/src/Nexus.Agent/nexus-agent.csproj new file mode 100644 index 0000000..9fed2d8 --- /dev/null +++ b/src/Nexus.Agent/nexus-agent.csproj @@ -0,0 +1,29 @@ + + + + $(TargetFrameworkVersion) + false + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Nexus.Sources.Remote/DataSourceTypes.cs b/src/Nexus.Sources.Remote/DataSourceTypes.cs index 51a7a8a..b11f4ce 100644 --- a/src/Nexus.Sources.Remote/DataSourceTypes.cs +++ b/src/Nexus.Sources.Remote/DataSourceTypes.cs @@ -54,7 +54,12 @@ public override bool CanConvert(Type objectType) return canConvert; } - public override object? ReadJson(Newtonsoft.Json.JsonReader reader, Type objectType, object? existingValue, Newtonsoft.Json.JsonSerializer serializer) + public override object? ReadJson( + Newtonsoft.Json.JsonReader reader, + Type objectType, + object? existingValue, + Newtonsoft.Json.JsonSerializer serializer + ) { if (reader.TokenType == Newtonsoft.Json.JsonToken.Null) return default; diff --git a/src/Nexus.Sources.Remote/RemoteCommunicator.cs b/src/Nexus.Sources.Remote/RemoteCommunicator.cs index af146e5..c662d01 100644 --- a/src/Nexus.Sources.Remote/RemoteCommunicator.cs +++ b/src/Nexus.Sources.Remote/RemoteCommunicator.cs @@ -1,9 +1,6 @@ -using System.Diagnostics; -using System.Net; -using System.Net.NetworkInformation; +using System.Net; using System.Net.Sockets; using System.Text; -using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Serialization; @@ -11,204 +8,111 @@ namespace Nexus.Sources; -internal partial class RemoteCommunicator +internal class RemoteCommunicator { - #region Fields + private readonly IPEndPoint _endPoint; - private static readonly object _lock = new(); - private static int _nextMin = -1; - private readonly TcpListener _tcpListener; - private Stream _commStream = default!; - private Stream _dataStream = default!; - private IJsonRpcServer _rpcServer = default!; + private readonly TcpClient _comm = new(); - private readonly ILogger _logger; - private readonly Func _readData; + private readonly TcpClient _data = new(); - private readonly string _command; - private readonly string _arguments; - private readonly Dictionary _environmentVariables; + private NetworkStream? _commStream; - private Process _process = default!; + private NetworkStream? _dataStream; - #endregion + private IJsonRpcServer _rpcServer = default!; - #region Constructors + private readonly ILogger _logger; + + private readonly Func _readData; public RemoteCommunicator( - string command, - Dictionary environmentVariables, - IPAddress listenAddress, - int listenPortMin, - int listenPortMax, + IPEndPoint endPoint, Func readData, - ILogger logger) + ILogger logger + ) { - _environmentVariables = environmentVariables; + _endPoint = endPoint; _readData = readData; _logger = logger; - - var listenPort = GetNextUnusedPort(listenPortMin, listenPortMax); - - command = CommandRegex().Replace(command, listenPort.ToString()); - var commandParts = command.Split(" ", count: 2); - _command = commandParts[0]; - - _arguments = commandParts.Length == 2 - ? commandParts[1] - : ""; - - _tcpListener = new TcpListener(listenAddress, listenPort); - _tcpListener.Start(); } - #endregion - - #region Methods - public async Task ConnectAsync(CancellationToken cancellationToken) { - try - { - cancellationToken.Register(_tcpListener.Stop); - - // start process - _logger.LogDebug("Start process."); - - var psi = new ProcessStartInfo(_command) - { - Arguments = _arguments, - }; - - foreach (var variable in _environmentVariables) - { - psi.EnvironmentVariables[variable.Key] = variable.Value; - } - - psi.RedirectStandardOutput = true; - psi.RedirectStandardError = true; + var id = Guid.NewGuid().ToString(); - _process = new Process() { StartInfo = psi }; - _process.Start(); + // comm connection + await _comm.ConnectAsync(_endPoint, cancellationToken); + _commStream = _comm.GetStream(); - _process.OutputDataReceived += (sender, e) => - { - if (!string.IsNullOrWhiteSpace(e.Data)) - _logger.LogDebug("{Message}", e.Data); - }; - - _process.ErrorDataReceived += (sender, e) => - { - if (!string.IsNullOrWhiteSpace(e.Data)) - _logger.LogWarning("{Message}", e.Data); - }; - - _process.BeginOutputReadLine(); - _process.BeginErrorReadLine(); - - // wait for clients to connect - _logger.LogDebug("Wait for clients to connect."); - - var filters = new string[] { "comm", "data" }; + await _commStream.WriteAsync(Encoding.UTF8.GetBytes(id), cancellationToken); + await _commStream.WriteAsync(Encoding.UTF8.GetBytes("comm"), cancellationToken); + await _commStream.FlushAsync(cancellationToken); - Stream? commStream = default; - Stream? dataStream = default; - - for (int i = 0; i < 2; i++) - { - var (identifier, client) = await GetTcpClientAsync(filters, cancellationToken); - - if (commStream is null && identifier == "comm") - commStream = client.GetStream(); - - else if (dataStream is null && identifier == "data") - dataStream = client.GetStream(); - } + // data connection + await _data.ConnectAsync(_endPoint, cancellationToken); + _dataStream = _data.GetStream(); + + await _dataStream.WriteAsync(Encoding.UTF8.GetBytes(id), cancellationToken); + await _dataStream.WriteAsync(Encoding.UTF8.GetBytes("data"), cancellationToken); + await _dataStream.FlushAsync(cancellationToken); - if (commStream is null || dataStream is null) - throw new Exception("The RPC server did not connect properly via communication and a data stream. This may indicate that other TCP clients have tried to connect."); - - _commStream = commStream; - _dataStream = dataStream; - - var formatter = new JsonMessageFormatter() - { - JsonSerializer = { - ContractResolver = new DefaultContractResolver - { - NamingStrategy = new CamelCaseNamingStrategy() - } + var formatter = new JsonMessageFormatter() + { + JsonSerializer = { + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new CamelCaseNamingStrategy() } - }; + } + }; - formatter.JsonSerializer.Converters.Add(new JsonElementConverter()); - formatter.JsonSerializer.Converters.Add(new StringEnumConverter()); + formatter.JsonSerializer.Converters.Add(new JsonElementConverter()); + formatter.JsonSerializer.Converters.Add(new StringEnumConverter()); - var messageHandler = new LengthHeaderMessageHandler(commStream, commStream, formatter); - var jsonRpc = new JsonRpc(messageHandler); + var messageHandler = new LengthHeaderMessageHandler(_commStream, _commStream, formatter); + var jsonRpc = new JsonRpc(messageHandler); - jsonRpc.AddLocalRpcMethod("log", new Action((logLevel, message) => - { - _logger.Log(logLevel, "{Message}", message); - })); + jsonRpc.AddLocalRpcMethod("log", new Action((logLevel, message) => + { + _logger.Log(logLevel, "{Message}", message); + })); - jsonRpc.AddLocalRpcMethod("readData", _readData); - jsonRpc.StartListening(); + jsonRpc.AddLocalRpcMethod("readData", _readData); + jsonRpc.StartListening(); - _rpcServer = jsonRpc.Attach(new JsonRpcProxyOptions() - { - MethodNameTransform = pascalCaseAsyncName => - { - return char.ToLower(pascalCaseAsyncName[0]) + pascalCaseAsyncName[1..].Replace("Async", string.Empty); - } - }); - - return _rpcServer; - } - catch + _rpcServer = jsonRpc.Attach(new JsonRpcProxyOptions() { - try + MethodNameTransform = pascalCaseAsyncName => { - _process?.Kill(); - } - catch - { - // + return char.ToLower(pascalCaseAsyncName[0]) + pascalCaseAsyncName[1..].Replace("Async", string.Empty); } + }); - throw; - } - finally - { - _tcpListener.Stop(); - } + return _rpcServer; } - public Task ReadRawAsync(Memory buffer, CancellationToken cancellationToken) + public ValueTask ReadRawAsync(Memory buffer, CancellationToken cancellationToken) { - return InternalReadRawAsync(buffer, _dataStream, cancellationToken); - } + if (_dataStream is null) + throw new Exception("You need to connect before read any data"); - private static async Task InternalReadRawAsync(Memory buffer, Stream source, CancellationToken cancellationToken) - { - while (buffer.Length > 0) - { - cancellationToken.ThrowIfCancellationRequested(); - var readCount = await source.ReadAsync(buffer, cancellationToken); - - if (readCount == 0) - throw new Exception("The TCP connection closed early."); - - buffer = buffer[readCount..]; - } + return _dataStream.ReadExactlyAsync(buffer, cancellationToken); } public Task WriteRawAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken) { + if (_dataStream is null) + throw new Exception("You need to connect before write any data"); + return InternalWriteRawAsync(buffer, _dataStream, cancellationToken); } - private static async Task InternalWriteRawAsync(ReadOnlyMemory buffer, Stream target, CancellationToken cancellationToken) + private static async Task InternalWriteRawAsync( + ReadOnlyMemory buffer, + Stream target, + CancellationToken cancellationToken + ) { var length = BitConverter.GetBytes(buffer.Length).Reverse().ToArray(); @@ -217,58 +121,7 @@ private static async Task InternalWriteRawAsync(ReadOnlyMemory buffer, Str await target.FlushAsync(cancellationToken); } - private static int GetNextUnusedPort(int min, int max) - { - lock (_lock) - { - min = Math.Max(_nextMin, min); - - if (max <= min) - throw new ArgumentException("Max port cannot be less than or equal to min."); - - var ipProperties = IPGlobalProperties.GetIPGlobalProperties(); - - var usedPorts = - ipProperties.GetActiveTcpConnections() - .Where(connection => connection.State != TcpState.Closed) - .Select(connection => connection.LocalEndPoint) - .Concat(ipProperties.GetActiveTcpListeners()) - .Select(endpoint => endpoint.Port) - .ToArray(); - - var firstUnused = - Enumerable.Range(min, max - min) - .Where(port => !usedPorts.Contains(port)) - .Select(port => new int?(port)) - .FirstOrDefault(); - - if (!firstUnused.HasValue) - throw new Exception($"All TCP ports in the range of {min}..{max} are currently in use."); - - _nextMin = (firstUnused.Value + 1) % max; - return firstUnused.Value; - } - } - - private async Task<(string Identifier, TcpClient Client)> GetTcpClientAsync(string[] filters, CancellationToken cancellationToken) - { - var buffer = new byte[4]; - var client = await _tcpListener.AcceptTcpClientAsync(cancellationToken); - - await InternalReadRawAsync(buffer, client.GetStream(), cancellationToken); - - foreach (var filter in filters) - { - if (buffer.SequenceEqual(Encoding.UTF8.GetBytes(filter))) - return (filter, client); - } - - throw new Exception("Invalid stream identifier received."); - } - - #endregion - - #region IDisposable +#region IDisposable private bool _disposedValue; @@ -278,29 +131,11 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - try - { - var disposable = _rpcServer as IDisposable; - disposable?.Dispose(); - - _commStream?.Dispose(); - _dataStream?.Dispose(); - - try - { - _process?.Kill(); - } - catch - { - // - } - } - catch (Exception) - { - // - } + var disposable = _rpcServer as IDisposable; + disposable?.Dispose(); - //_process?.WaitForExitAsync(); + _commStream?.Dispose(); + _dataStream?.Dispose(); } _disposedValue = true; @@ -312,9 +147,6 @@ public void Dispose() Dispose(disposing: true); GC.SuppressFinalize(this); } - - [GeneratedRegex("{remote-port}")] - private static partial Regex CommandRegex(); - - #endregion -} + +#endregion +} \ No newline at end of file diff --git a/src/remoting/dotnet-remoting/Remoting.cs b/src/remoting/dotnet-remoting/Remoting.cs index 0288ee4..aabb9c1 100644 --- a/src/remoting/dotnet-remoting/Remoting.cs +++ b/src/remoting/dotnet-remoting/Remoting.cs @@ -5,18 +5,17 @@ using System.Globalization; using System.Net.Sockets; using System.Runtime.InteropServices; -using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace Nexus.Remoting; -internal class Logger(NetworkStream tcpCommSocketStream) : ILogger +internal class Logger(NetworkStream commStream) : ILogger { - private readonly NetworkStream _tcpCommSocketStream = tcpCommSocketStream; + private readonly NetworkStream _commStream = commStream; - public IDisposable BeginScope(TState state) + public IDisposable? BeginScope(TState state) where TState : notnull { throw new NotImplementedException("Scopes are not supported on this logger."); } @@ -26,7 +25,13 @@ public bool IsEnabled(LogLevel logLevel) return true; } - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter + ) { var notification = new JsonObject() { @@ -35,7 +40,7 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except ["params"] = new JsonArray(logLevel.ToString(), formatter(state, exception)) }; - _ = Utilities.SendToServerAsync(notification, _tcpCommSocketStream); + _ = Utilities.SendToServerAsync(notification, _commStream); } } @@ -44,33 +49,37 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except /// public class RemoteCommunicator { - private readonly string _address; - private readonly int _port; - private readonly TcpClient _tcpCommSocket; - private readonly TcpClient _tcpDataSocket; - private NetworkStream _tcpCommSocketStream = default!; - private NetworkStream _tcpDataSocketStream = default!; - private readonly IDataSource _dataSource; + private readonly TcpClient _comm; + + private readonly TcpClient _data; + + private readonly NetworkStream _commStream; + + private readonly NetworkStream _dataStream; + + private readonly Func _getDataSource; + private ILogger _logger = default!; /// - /// Initializes a new instance of the RemoteCommunicator. + /// Initializes a new instance of the . /// - /// The data source - /// The address to connect to - /// The port to connect to - public RemoteCommunicator(IDataSource dataSource, string address, int port) + /// The TCP client for communications. + /// The TCP client for data. + /// A func to get a new data source instance by its type name. + public RemoteCommunicator( + TcpClient comm, + TcpClient data, + Func getDataSource + ) { - _address = address; - _port = port; - - _tcpCommSocket = new TcpClient(); - _tcpDataSocket = new TcpClient(); + _comm = comm; + _commStream = comm.GetStream(); - if (!(0 < port && port < 65536)) - throw new Exception($"The port {port} is not a valid port number."); + _data = data; + _dataStream = data.GetStream(); - _dataSource = dataSource; + _getDataSource = getDataSource; } /// @@ -85,30 +94,18 @@ static JsonElement Read(Span jsonRequest) return JsonSerializer.Deserialize(ref reader, Utilities.Options); } - // comm connection - await _tcpCommSocket.ConnectAsync(_address, _port); - _tcpCommSocketStream = _tcpCommSocket.GetStream(); - await _tcpCommSocketStream.WriteAsync(Encoding.UTF8.GetBytes("comm")); - await _tcpCommSocketStream.FlushAsync(); - - // data connection - await _tcpDataSocket.ConnectAsync(_address, _port); - _tcpDataSocketStream = _tcpDataSocket.GetStream(); - await _tcpDataSocketStream.WriteAsync(Encoding.UTF8.GetBytes("data")); - await _tcpDataSocketStream.FlushAsync(); - // loop while (true) { // https://www.jsonrpc.org/specification // get request message - var size = ReadSize(_tcpCommSocketStream); + var size = ReadSize(_commStream); using var memoryOwner = MemoryPool.Shared.Rent(size); var messageMemory = memoryOwner.Memory[..size]; - _tcpCommSocketStream.ReadExactly(messageMemory.Span, _logger); + _commStream.ReadExactly(messageMemory.Span, _logger); var request = Read(messageMemory.Span); // process message @@ -166,14 +163,14 @@ static JsonElement Read(Span jsonRequest) response.Add("id", id); // send response - await Utilities.SendToServerAsync(response, _tcpCommSocketStream); + await Utilities.SendToServerAsync(response, _commStream); // send data if (!data.Equals(default) && !status.Equals(default)) { - await _tcpDataSocketStream.WriteAsync(data); - await _tcpDataSocketStream.WriteAsync(status); - await _tcpDataSocketStream.FlushAsync(); + await _dataStream.WriteAsync(data); + await _dataStream.WriteAsync(status); + await _dataStream.FlushAsync(); } } } @@ -185,6 +182,7 @@ static JsonElement Read(Span jsonRequest) JsonObject? result = default; Memory data = default; Memory status = default; + IDataSource? dataSource = default; var methodName = request.GetProperty("method").GetString(); var @params = request.GetProperty("params"); @@ -202,6 +200,12 @@ static JsonElement Read(Span jsonRequest) var rawContext = @params[0]; var resourceLocator = default(Uri?); + if (rawContext.TryGetProperty("type", out var type)) + dataSource = _getDataSource(type.ToString()); + + else + throw new Exception("The type property is required"); + if (rawContext.TryGetProperty("resourceLocator", out var value)) resourceLocator = new Uri(value.GetString()!); @@ -223,7 +227,7 @@ static JsonElement Read(Span jsonRequest) if (rawContext.TryGetProperty("requestConfiguration", out var requestConfigurationElement)) requestConfiguration = JsonSerializer.Deserialize?>(requestConfigurationElement); - _logger = new Logger(_tcpCommSocketStream); + _logger = new Logger(_commStream); var context = new DataSourceContext( resourceLocator, @@ -232,13 +236,16 @@ static JsonElement Read(Span jsonRequest) requestConfiguration ); - await _dataSource.SetContextAsync(context, _logger, CancellationToken.None); + await dataSource.SetContextAsync(context, _logger, CancellationToken.None); } else if (methodName == "getCatalogRegistrations") { + if (dataSource is null) + throw new Exception("The data source context must be set before invoking other methods."); + var path = @params[0].GetString()!; - var registrations = await _dataSource.GetCatalogRegistrationsAsync(path, CancellationToken.None); + var registrations = await dataSource.GetCatalogRegistrationsAsync(path, CancellationToken.None); result = new JsonObject() { @@ -248,8 +255,11 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "getCatalog") { + if (dataSource is null) + throw new Exception("The data source context must be set before invoking other methods."); + var catalogId = @params[0].GetString()!; - var catalog = await _dataSource.GetCatalogAsync(catalogId, CancellationToken.None); + var catalog = await dataSource.GetCatalogAsync(catalogId, CancellationToken.None); result = new JsonObject() { @@ -259,8 +269,11 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "getTimeRange") { + if (dataSource is null) + throw new Exception("The data source context must be set before invoking other methods."); + var catalogId = @params[0].GetString()!; - var (begin, end) = await _dataSource.GetTimeRangeAsync(catalogId, CancellationToken.None); + var (begin, end) = await dataSource.GetTimeRangeAsync(catalogId, CancellationToken.None); result = new JsonObject() { @@ -271,6 +284,9 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "getAvailability") { + if (dataSource is null) + throw new Exception("The data source context must be set before invoking other methods."); + var catalogId = @params[0].GetString()!; var beginString = @params[1].GetString()!; @@ -279,7 +295,7 @@ static JsonElement Read(Span jsonRequest) var endString = @params[2].GetString()!; var end = DateTime.ParseExact(endString, "yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture); - var availability = await _dataSource.GetAvailabilityAsync(catalogId, begin, end, CancellationToken.None); + var availability = await dataSource.GetAvailabilityAsync(catalogId, begin, end, CancellationToken.None); result = new JsonObject() { @@ -289,6 +305,9 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "readSingle") { + if (dataSource is null) + throw new Exception("The data source context must be set before invoking other methods."); + var beginString = @params[0].GetString()!; var begin = DateTime.ParseExact(beginString, "yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture).ToUniversalTime(); @@ -299,7 +318,7 @@ static JsonElement Read(Span jsonRequest) (data, status) = ExtensibilityUtilities.CreateBuffers(catalogItem.Representation, begin, end); var readRequest = new ReadRequest(catalogItem, data, status); - await _dataSource.ReadAsync( + await dataSource.ReadAsync( begin, end, [readRequest], @@ -349,16 +368,16 @@ private async Task HandleReadDataAsync( _logger.LogDebug("Read resource path {ResourcePath} from Nexus", resourcePath); - await Utilities.SendToServerAsync(readDataRequest, _tcpCommSocketStream); + await Utilities.SendToServerAsync(readDataRequest, _commStream); - var size = ReadSize(_tcpDataSocketStream); + var size = ReadSize(_dataStream); if (size != buffer.Length * sizeof(double)) throw new Exception("Data returned by Nexus have an unexpected length"); _logger.LogTrace("Try to read {ByteCount} bytes from Nexus", size); - _tcpDataSocketStream.ReadExactly(MemoryMarshal.AsBytes(buffer.Span), _logger); + _dataStream.ReadExactly(MemoryMarshal.AsBytes(buffer.Span), _logger); } private int ReadSize(NetworkStream currentStream) @@ -390,7 +409,7 @@ static Utilities() public static async Task SendToServerAsync(JsonNode response, NetworkStream currentStream) { - var encodedResponse = JsonSerializer.SerializeToUtf8Bytes(response, Utilities.Options); + var encodedResponse = JsonSerializer.SerializeToUtf8Bytes(response, Options); var messageLength = BitConverter.GetBytes(encodedResponse.Length); Array.Reverse(messageLength); From 7ca98ad37a044dee08b2fa33bacd29e5ef8c589e Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Wed, 30 Oct 2024 22:28:28 +0100 Subject: [PATCH 02/15] Prepare for first real test --- .vscode/launch.json | 18 ++++ .vscode/tasks.json | 18 ++-- Remote.sln | 2 +- src/Nexus.Agent/Core/Agent.cs | 33 +++++-- ...{nexus-agent.csproj => Nexus.Agent.csproj} | 2 +- src/Nexus.Sources.Remote/DataSourceTypes.cs | 6 +- .../Nexus.Sources.Remote.csproj | 2 +- src/Nexus.Sources.Remote/Remote.cs | 86 +++++-------------- .../RemoteCommunicator.cs | 10 +-- src/remoting/dotnet-remoting/Remoting.cs | 21 +++-- .../dotnet-remoting/dotnet-remoting.csproj | 2 +- .../nexus_remoting/_remoting.py | 9 +- .../Nexus.Sources.Remote.Tests/AgentTests.cs | 35 ++++++++ .../Nexus.Sources.Remote.Tests.csproj | 2 +- .../Nexus.Sources.Remote.Tests/RemoteTests.cs | 16 ++-- .../SetupDockerTests.cs | 4 +- .../Nexus.Sources.Remote.Tests/bash/remote.sh | 2 +- .../dotnet/remote.csproj | 2 +- 18 files changed, 148 insertions(+), 122 deletions(-) create mode 100644 .vscode/launch.json rename src/Nexus.Agent/{nexus-agent.csproj => Nexus.Agent.csproj} (97%) create mode 100644 tests/Nexus.Sources.Remote.Tests/AgentTests.cs diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..33e9751 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Nexus.Agent", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build-nexus-agent", + "program": "${workspaceFolder}/artifacts/bin/Nexus.Agent/debug/Nexus.Agent.dll", + "args": [], + "cwd": "${workspaceFolder}", + "stopAtEntry": false, + "env": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 1b88bd7..532fe2b 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -2,23 +2,23 @@ "version": "2.0.0", "tasks": [ { - "label": "build", + "label": "build-nexus-agent", "command": "dotnet", "type": "process", "args": [ "build", - "${workspaceFolder}/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj", + "${workspaceFolder}/src/Nexus.Agent/Nexus.Agent.csproj", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" ], "problemMatcher": "$msCompile" }, { - "label": "publish", + "label": "build", "command": "dotnet", "type": "process", "args": [ - "publish", + "build", "${workspaceFolder}/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj", "/property:GenerateFullPaths=true", "/consoleloggerparameters:NoSummary" @@ -26,14 +26,14 @@ "problemMatcher": "$msCompile" }, { - "label": "watch", + "label": "publish", "command": "dotnet", "type": "process", "args": [ - "watch", - "run", - "--project", - "${workspaceFolder}/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj" + "publish", + "${workspaceFolder}/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" ], "problemMatcher": "$msCompile" } diff --git a/Remote.sln b/Remote.sln index 5455152..7d46acc 100644 --- a/Remote.sln +++ b/Remote.sln @@ -19,7 +19,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "benchmarks", "benchmarks", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nexus.Benchmarks", "benchmarks\Nexus.Benchmarks\Nexus.Benchmarks.csproj", "{B602750B-5E89-453F-841B-8D7F22DE58EB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "nexus-agent", "src\Nexus.Agent\nexus-agent.csproj", "{71F41120-5BC2-4684-ADB7-C8C97CCE8EAF}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nexus.Agent", "src\Nexus.Agent\Nexus.Agent.csproj", "{71F41120-5BC2-4684-ADB7-C8C97CCE8EAF}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/src/Nexus.Agent/Core/Agent.cs b/src/Nexus.Agent/Core/Agent.cs index d186c20..f17e06d 100644 --- a/src/Nexus.Agent/Core/Agent.cs +++ b/src/Nexus.Agent/Core/Agent.cs @@ -13,6 +13,8 @@ namespace Nexus.Agent; public class Agent { + private Lock _lock = new(); + private readonly ConcurrentDictionary _tcpClientPairs = new(); public async Task RunAsync() @@ -48,13 +50,14 @@ await extensionHive.LoadPackagesAsync( private Task AcceptClientsAsync(IExtensionHive extensionHive) { - var tcpListenerComm = new TcpListener(IPAddress.Any, 56145); + var tcpListener = new TcpListener(IPAddress.Any, 56145); + tcpListener.Start(); return Task.Run(async () => { while (true) { - var client = await tcpListenerComm.AcceptTcpClientAsync(); + var client = await tcpListener.AcceptTcpClientAsync(); _ = Task.Run(async () => { @@ -127,17 +130,31 @@ private Task AcceptClientsAsync(IExtensionHive extensionHive) else { client.Dispose(); + return; } var pair = _tcpClientPairs[id]; - if (pair.Comm is not null && pair.Data is not null) + lock (_lock) { - pair.RemoteCommunicator = new RemoteCommunicator( - pair.Comm, - pair.Data, - getDataSource: type => extensionHive.GetInstance(type) - ); + if (pair.Comm is not null && pair.Data is not null && pair.RemoteCommunicator is null) + { + try + { + pair.RemoteCommunicator = new RemoteCommunicator( + pair.Comm, + pair.Data, + getDataSource: type => extensionHive.GetInstance(type) + ); + + _ = pair.RemoteCommunicator.RunAsync(); + } + catch + { + pair.Comm?.Dispose(); + pair.Data?.Dispose(); + } + } } } }); diff --git a/src/Nexus.Agent/nexus-agent.csproj b/src/Nexus.Agent/Nexus.Agent.csproj similarity index 97% rename from src/Nexus.Agent/nexus-agent.csproj rename to src/Nexus.Agent/Nexus.Agent.csproj index 9fed2d8..4ffc1c7 100644 --- a/src/Nexus.Agent/nexus-agent.csproj +++ b/src/Nexus.Agent/Nexus.Agent.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Nexus.Sources.Remote/DataSourceTypes.cs b/src/Nexus.Sources.Remote/DataSourceTypes.cs index b11f4ce..8957741 100644 --- a/src/Nexus.Sources.Remote/DataSourceTypes.cs +++ b/src/Nexus.Sources.Remote/DataSourceTypes.cs @@ -12,13 +12,13 @@ public Task GetApiVersionAsync(CancellationToken cancellationToken); public Task - SetContextAsync(DataSourceContext context, CancellationToken cancellationToken); + SetContextAsync(string type, DataSourceContext context, CancellationToken cancellationToken); public Task GetCatalogRegistrationsAsync(string path, CancellationToken cancellationToken); public Task - GetCatalogAsync(string catalogId, CancellationToken cancellationToken); + EnrichCatalogAsync(ResourceCatalog catalog, CancellationToken cancellationToken); public Task GetTimeRangeAsync(string catalogId, CancellationToken cancellationToken); @@ -27,7 +27,7 @@ public Task GetAvailabilityAsync(string catalogId, DateTime begin, DateTime end, CancellationToken cancellationToken); public Task - ReadSingleAsync(DateTime begin, DateTime end, CatalogItem catalogItem, CancellationToken cancellationToken); + ReadSingleAsync(DateTime begin, DateTime end, string OriginalResourceName, CatalogItem catalogItem, CancellationToken cancellationToken); } internal record ApiVersionResponse(int ApiVersion); diff --git a/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj b/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj index f3b3844..42c1214 100644 --- a/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj +++ b/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj @@ -11,7 +11,7 @@ - + runtime;native diff --git a/src/Nexus.Sources.Remote/Remote.cs b/src/Nexus.Sources.Remote/Remote.cs index 9ea341f..c061ac4 100644 --- a/src/Nexus.Sources.Remote/Remote.cs +++ b/src/Nexus.Sources.Remote/Remote.cs @@ -4,7 +4,6 @@ using System.Buffers; using System.Net; using System.Reflection; -using System.Text.Json; using System.Text.RegularExpressions; namespace Nexus.Sources; @@ -12,7 +11,7 @@ namespace Nexus.Sources; [ExtensionDescription( "Provides access to remote databases", "https://github.com/nexus-main/nexus-sources-remote", - "https://github.com/nexus-main/nexus-sources-remote")] + "https://github.com/nexus-main/nexus-sources-remote")] public partial class Remote : IDataSource, IDisposable { #region Fields @@ -62,55 +61,24 @@ public async Task SetContextAsync( if (mode != "tcp") throw new NotSupportedException($"The mode {mode} is not supported."); - // listen-address - var listenAddressString = Context.SourceConfiguration?.GetStringValue("listen-address") ?? "0.0.0.0"; + // endpoint + var endpointString = Context.SourceConfiguration?.GetStringValue("endpoint") ?? "127.0.0.1:56145"; - if (!IPAddress.TryParse(listenAddressString, out var listenAddress)) - throw new ArgumentException("The listen-address parameter is not a valid IP-Address."); + if (!IPEndPoint.TryParse(endpointString, out var endpoint)) + throw new ArgumentException("The endpoint parameter is not a valid IP Endpoint."); - // listen-port - var listenPortMin = Context.SourceConfiguration?.GetIntValue("listen-port-min") ?? 49152; - - if (!(1 <= listenPortMin && listenPortMin < 65536)) - throw new ArgumentException("The listen-port-min parameter is invalid."); - - var listenPortMax = Context.SourceConfiguration?.GetIntValue("listen-port-max") ?? 65536; - - if (!(1 <= listenPortMin && listenPortMin < 65536)) - throw new ArgumentException("The listen-port-max parameter is invalid."); - - // template - var templateId = (Context.SourceConfiguration?.GetStringValue("template")) ?? throw new KeyNotFoundException("The template parameter must be provided."); - - // environment variables - var requestConfiguration = Context.SourceConfiguration!; - var environmentVariables = new Dictionary(); - - if (requestConfiguration.TryGetValue("environment-variables", out var propertyValue) && - propertyValue.ValueKind == JsonValueKind.Object) - { - var environmentVariablesRaw = propertyValue.Deserialize>(); - - if (environmentVariablesRaw is not null) - environmentVariables = environmentVariablesRaw - .Where(entry => entry.Value.ValueKind == JsonValueKind.String) - .ToDictionary(entry => entry.Key, entry => entry.Value.GetString() ?? ""); - } - - // Build command - var actualCommand = BuildCommand(templateId); + // type + var type = Context.SourceConfiguration?.GetStringValue("type") ?? throw new Exception("The data source type is missing."); // - var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); - _communicator = new RemoteCommunicator( - actualCommand, - environmentVariables, - listenAddress, - listenPortMin, - listenPortMax, + endpoint, HandleReadDataAsync, - logger); + logger + ); + + var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); + cancellationToken.Register(() => timeoutTokenSource.Cancel()); _rpcServer = await _communicator.ConnectAsync(timeoutTokenSource.Token); @@ -122,7 +90,7 @@ public async Task SetContextAsync( logger.LogTrace("Set context to remote client"); await _rpcServer - .SetContextAsync(context, timeoutTokenSource.Token); + .SetContextAsync(type, context, timeoutTokenSource.Token); logger.LogDebug("Done preparing remote client"); } @@ -140,15 +108,15 @@ public async Task GetCatalogRegistrationsAsync( return response.Registrations; } - public async Task GetCatalogAsync( - string catalogId, + public async Task EnrichCatalogAsync( + ResourceCatalog catalog, CancellationToken cancellationToken) { var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); cancellationToken.Register(() => timeoutTokenSource.Cancel()); var response = await _rpcServer - .GetCatalogAsync(catalogId, timeoutTokenSource.Token); + .EnrichCatalogAsync(catalog, timeoutTokenSource.Token); return response.Catalog; } @@ -198,17 +166,17 @@ public async Task ReadAsync( { var counter = 0.0; - foreach (var (catalogItem, data, status) in requests) + foreach (var (originalResourceName, catalogItem, data, status) in requests) { cancellationToken.ThrowIfCancellationRequested(); var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); - cancellationToken.Register(() => timeoutTokenSource.Cancel()); + cancellationToken.Register(timeoutTokenSource.Cancel); var elementCount = data.Length / catalogItem.Representation.ElementSize; await _rpcServer - .ReadSingleAsync(begin, end, catalogItem, timeoutTokenSource.Token); + .ReadSingleAsync(begin, end, originalResourceName, catalogItem, timeoutTokenSource.Token); await _communicator.ReadRawAsync(data, timeoutTokenSource.Token); await _communicator.ReadRawAsync(status, timeoutTokenSource.Token); @@ -222,20 +190,6 @@ await _rpcServer } } - private string BuildCommand(string templateId) - { - var template = (Context.SystemConfiguration? - .GetStringValue($"{typeof(Remote).FullName}/templates/{templateId}")) ?? throw new Exception($"The template {templateId} does not exist."); - var command = CommandRegex().Replace(template, match => - { - var parameterKey = match.Groups[1].Value; - var parameterValue = (Context.SourceConfiguration?.GetStringValue(parameterKey)) ?? throw new Exception($"The {parameterKey} parameter must be provided."); - return parameterValue; - }); - - return command; - } - // copy from Nexus -> DataModelUtilities private static readonly Regex _resourcePathEvaluator = MyRegex(); diff --git a/src/Nexus.Sources.Remote/RemoteCommunicator.cs b/src/Nexus.Sources.Remote/RemoteCommunicator.cs index c662d01..7e7d97f 100644 --- a/src/Nexus.Sources.Remote/RemoteCommunicator.cs +++ b/src/Nexus.Sources.Remote/RemoteCommunicator.cs @@ -10,7 +10,7 @@ namespace Nexus.Sources; internal class RemoteCommunicator { - private readonly IPEndPoint _endPoint; + private readonly IPEndPoint _endpoint; private readonly TcpClient _comm = new(); @@ -27,12 +27,12 @@ internal class RemoteCommunicator private readonly Func _readData; public RemoteCommunicator( - IPEndPoint endPoint, + IPEndPoint endpoint, Func readData, ILogger logger ) { - _endPoint = endPoint; + _endpoint = endpoint; _readData = readData; _logger = logger; } @@ -42,7 +42,7 @@ public async Task ConnectAsync(CancellationToken cancellationTok var id = Guid.NewGuid().ToString(); // comm connection - await _comm.ConnectAsync(_endPoint, cancellationToken); + await _comm.ConnectAsync(_endpoint, cancellationToken); _commStream = _comm.GetStream(); await _commStream.WriteAsync(Encoding.UTF8.GetBytes(id), cancellationToken); @@ -50,7 +50,7 @@ public async Task ConnectAsync(CancellationToken cancellationTok await _commStream.FlushAsync(cancellationToken); // data connection - await _data.ConnectAsync(_endPoint, cancellationToken); + await _data.ConnectAsync(_endpoint, cancellationToken); _dataStream = _data.GetStream(); await _dataStream.WriteAsync(Encoding.UTF8.GetBytes(id), cancellationToken); diff --git a/src/remoting/dotnet-remoting/Remoting.cs b/src/remoting/dotnet-remoting/Remoting.cs index aabb9c1..db5b978 100644 --- a/src/remoting/dotnet-remoting/Remoting.cs +++ b/src/remoting/dotnet-remoting/Remoting.cs @@ -197,14 +197,11 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "setContext") { - var rawContext = @params[0]; + var type = @params[0].ToString(); + var rawContext = @params[1]; var resourceLocator = default(Uri?); - if (rawContext.TryGetProperty("type", out var type)) - dataSource = _getDataSource(type.ToString()); - - else - throw new Exception("The type property is required"); + dataSource = _getDataSource(type); if (rawContext.TryGetProperty("resourceLocator", out var value)) resourceLocator = new Uri(value.GetString()!); @@ -253,13 +250,13 @@ static JsonElement Read(Span jsonRequest) }; } - else if (methodName == "getCatalog") + else if (methodName == "enrichCatalog") { if (dataSource is null) throw new Exception("The data source context must be set before invoking other methods."); - var catalogId = @params[0].GetString()!; - var catalog = await dataSource.GetCatalogAsync(catalogId, CancellationToken.None); + var originalCatalog = JsonSerializer.Deserialize(@params[0])!; + var catalog = await dataSource.EnrichCatalogAsync(originalCatalog, CancellationToken.None); result = new JsonObject() { @@ -314,9 +311,11 @@ static JsonElement Read(Span jsonRequest) var endString = @params[1].GetString()!; var end = DateTime.ParseExact(endString, "yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture).ToUniversalTime(); - var catalogItem = JsonSerializer.Deserialize(@params[2], Utilities.Options)!; + var originalResourceName = @params[2].GetString()!; + + var catalogItem = JsonSerializer.Deserialize(@params[3], Utilities.Options)!; (data, status) = ExtensibilityUtilities.CreateBuffers(catalogItem.Representation, begin, end); - var readRequest = new ReadRequest(catalogItem, data, status); + var readRequest = new ReadRequest(originalResourceName, catalogItem, data, status); await dataSource.ReadAsync( begin, diff --git a/src/remoting/dotnet-remoting/dotnet-remoting.csproj b/src/remoting/dotnet-remoting/dotnet-remoting.csproj index 702a3cb..273a687 100644 --- a/src/remoting/dotnet-remoting/dotnet-remoting.csproj +++ b/src/remoting/dotnet-remoting/dotnet-remoting.csproj @@ -26,7 +26,7 @@ - + diff --git a/src/remoting/python-remoting/nexus_remoting/_remoting.py b/src/remoting/python-remoting/nexus_remoting/_remoting.py index cce7431..87a00b3 100644 --- a/src/remoting/python-remoting/nexus_remoting/_remoting.py +++ b/src/remoting/python-remoting/nexus_remoting/_remoting.py @@ -149,7 +149,9 @@ async def _process_invocation(self, request: dict[str, Any]) \ elif method_name == "setContext": - raw_context = params[0] + # TODO: make use of the type (see C# implementation) + type = params[0] + raw_context = params[1] resource_locator_string = cast(str, raw_context["resourceLocator"]) if "resourceLocator" in raw_context else None resource_locator = None if resource_locator_string is None else urlparse(resource_locator_string) @@ -215,9 +217,10 @@ async def _process_invocation(self, request: dict[str, Any]) \ begin = datetime.strptime(params[0], "%Y-%m-%dT%H:%M:%SZ") end = datetime.strptime(params[1], "%Y-%m-%dT%H:%M:%SZ") - catalog_item = JsonEncoder.decode(CatalogItem, params[2], _json_encoder_options) + original_resource_name = params[2] + catalog_item = JsonEncoder.decode(CatalogItem, params[3], _json_encoder_options) (data, status) = ExtensibilityUtilities.create_buffers(catalog_item.representation, begin, end) - read_request = ReadRequest(catalog_item, data, status) + read_request = ReadRequest(original_resource_name, catalog_item, data, status) await self._data_source.read( begin, diff --git a/tests/Nexus.Sources.Remote.Tests/AgentTests.cs b/tests/Nexus.Sources.Remote.Tests/AgentTests.cs new file mode 100644 index 0000000..556537b --- /dev/null +++ b/tests/Nexus.Sources.Remote.Tests/AgentTests.cs @@ -0,0 +1,35 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Nexus.DataModel; +using Nexus.Extensibility; +using Xunit; + +namespace Nexus.Sources.Tests; + +public class AgentTests +{ + [Fact] + public async Task CanProvideCatalog() + { + // Arrange + var dataSource = new Remote() as IDataSource; + + var context = new DataSourceContext( + ResourceLocator: new Uri("file:///" + Path.Combine(Directory.GetCurrentDirectory(), "TESTDATA")), + SystemConfiguration: default, + SourceConfiguration: new Dictionary() + { + ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.Famos") + }, + RequestConfiguration: default + ); + + await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); + + // Act + var actual = await dataSource.EnrichCatalogAsync(new ResourceCatalog("/A/B/C"), CancellationToken.None); + + // Assert + var b = 1; + } +} \ No newline at end of file diff --git a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj index 390ee5d..dd62e4d 100644 --- a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj +++ b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj @@ -15,7 +15,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs index 5403e81..52315c2 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs @@ -21,16 +21,16 @@ public class RemoteTests #endif public async Task ProvidesCatalog(string command) { - // arrange + // Arrange var dataSource = new Remote() as IDataSource; var context = CreateContext(command); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); - // act - var actual = await dataSource.GetCatalogAsync("/A/B/C", CancellationToken.None); + // Act + var actual = await dataSource.EnrichCatalogAsync(new ResourceCatalog("/A/B/C"), CancellationToken.None); - // assert + // Assert var actualProperties1 = actual.Properties; var actualIds = actual.Resources!.Select(resource => resource.Id).ToList(); var actualUnits = actual.Resources!.Select(resource => resource.Properties?.GetStringValue("unit")).ToList(); @@ -105,7 +105,7 @@ public async Task CanReadFullDay(string command, bool complexData) await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); - var catalog = await dataSource.GetCatalogAsync("/A/B/C", CancellationToken.None); + var catalog = await dataSource.EnrichCatalogAsync(new ResourceCatalog("/A/B/C"), CancellationToken.None); var resource = catalog.Resources![0]; var representation = resource.Representations![0]; @@ -148,7 +148,7 @@ void GenerateData(DateTimeOffset dateTime) expectedStatus.AsSpan().Fill((byte)'s'); } - var request = new ReadRequest(catalogItem, data, status); + var request = new ReadRequest(resource.Id, catalogItem, data, status); await dataSource.ReadAsync(begin, end, [request], default!, new Progress(), CancellationToken.None); var longData = new CastMemoryManager(data).Memory; @@ -192,7 +192,7 @@ public async Task CanReadDataHandler(string command) await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); - var catalog = await dataSource.GetCatalogAsync("/D/E/F", CancellationToken.None); + var catalog = await dataSource.EnrichCatalogAsync(new ResourceCatalog("/D/E/F"), CancellationToken.None); var resource = catalog.Resources![0]; var representation = resource.Representations![0]; @@ -230,7 +230,7 @@ Task HandleReadDataAsync(string resourcePath, DateTime begin, DateTime end, Memo return Task.CompletedTask; } - var request = new ReadRequest(catalogItem, data, status); + var request = new ReadRequest(resource.Id, catalogItem, data, status); await dataSource.ReadAsync(begin, end, [request], HandleReadDataAsync, new Progress(), CancellationToken.None); var doubleData = new CastMemoryManager(data).Memory; diff --git a/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.cs b/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.cs index 928aba1..5b5a7d9 100644 --- a/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.cs @@ -22,7 +22,7 @@ public async Task CanReadFullDay(string satelliteId, string command, string vers await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); - var catalog = await dataSource.GetCatalogAsync("/A/B/C", CancellationToken.None); + var catalog = await dataSource.EnrichCatalogAsync(new ResourceCatalog("/A/B/C"), CancellationToken.None); var resource = catalog.Resources![0]; var representation = resource.Representations![0]; @@ -59,7 +59,7 @@ Task ReadData(string resourcePath, DateTime begin, DateTime end, Memory return Task.CompletedTask; } - var request = new ReadRequest(catalogItem, data, status); + var request = new ReadRequest(resource.Id, catalogItem, data, status); await dataSource.ReadAsync( begin, diff --git a/tests/Nexus.Sources.Remote.Tests/bash/remote.sh b/tests/Nexus.Sources.Remote.Tests/bash/remote.sh index ad9ae17..8e23396 100644 --- a/tests/Nexus.Sources.Remote.Tests/bash/remote.sh +++ b/tests/Nexus.Sources.Remote.Tests/bash/remote.sh @@ -75,7 +75,7 @@ listen() { elif [ "$method" = "getCatalogIds" ]; then response='{ "jsonrpc": "2.0", "id": '$id', "result": { "CatalogIds": [ "/A/B/C" ] } }' - elif [ "$method" = "getCatalog" ]; then + elif [ "$method" = "enrichCatalog" ]; then catalog=$( - + From bfcfa6f87d2d376adbf19aa95453d612b4a66cb4 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Thu, 31 Oct 2024 22:05:44 +0100 Subject: [PATCH 03/15] Improve Nexus Agent --- .nexus-agent/config/packages.json | 10 ++++ .vscode/launch.json | 3 +- src/Nexus.Agent/Core/Agent.cs | 17 ++++--- src/Nexus.Agent/Core/{Types.cs => Options.cs} | 25 +++++++++- src/Nexus.Agent/Program.cs | 14 +++++- .../Nexus.Sources.Remote.Tests/AgentTests.cs | 2 +- .../dotnet/{ => v1}/remote.cs | 49 ++++--------------- .../dotnet/{ => v1}/remote.csproj | 9 ++-- 8 files changed, 71 insertions(+), 58 deletions(-) create mode 100644 .nexus-agent/config/packages.json rename src/Nexus.Agent/Core/{Types.cs => Options.cs} (60%) rename tests/Nexus.Sources.Remote.Tests/dotnet/{ => v1}/remote.cs (87%) rename tests/Nexus.Sources.Remote.Tests/dotnet/{ => v1}/remote.csproj (58%) diff --git a/.nexus-agent/config/packages.json b/.nexus-agent/config/packages.json new file mode 100644 index 0000000..bb259c8 --- /dev/null +++ b/.nexus-agent/config/packages.json @@ -0,0 +1,10 @@ +{ + "c05b592f-e198-472d-9902-3f60cf0a6332": { + "Provider": "local", + "Configuration": { + "path": "tests/Nexus.Sources.Remote.Tests/dotnet", + "version": "v1", + "csproj": "remote.csproj" + } + } +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 33e9751..99a222d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,8 @@ "cwd": "${workspaceFolder}", "stopAtEntry": false, "env": { - "ASPNETCORE_ENVIRONMENT": "Development" + "ASPNETCORE_ENVIRONMENT": "Development", + "NEXUSAGENT_Paths__Config": ".nexus-agent/config" } } ] diff --git a/src/Nexus.Agent/Core/Agent.cs b/src/Nexus.Agent/Core/Agent.cs index f17e06d..ce7f395 100644 --- a/src/Nexus.Agent/Core/Agent.cs +++ b/src/Nexus.Agent/Core/Agent.cs @@ -11,21 +11,22 @@ namespace Nexus.Agent; -public class Agent +internal class Agent { - private Lock _lock = new(); + private readonly Lock _lock = new(); private readonly ConcurrentDictionary _tcpClientPairs = new(); - public async Task RunAsync() + private readonly PathsOptions _pathsOptions; + + public Agent(PathsOptions pathsOptions) { - var extensionHive = await LoadPackagesAsync(); - await AcceptClientsAsync(extensionHive); + _pathsOptions = pathsOptions; } - private async Task LoadPackagesAsync() + public async Task LoadPackagesAsync() { - var pathsOptions = Options.Create(new PathsOptions()); + var pathsOptions = Options.Create(_pathsOptions); var loggerFactory = new LoggerFactory(); var databaseService = new DatabaseService(pathsOptions); @@ -48,7 +49,7 @@ await extensionHive.LoadPackagesAsync( return extensionHive; } - private Task AcceptClientsAsync(IExtensionHive extensionHive) + public Task AcceptClientsAsync(IExtensionHive extensionHive) { var tcpListener = new TcpListener(IPAddress.Any, 56145); tcpListener.Start(); diff --git a/src/Nexus.Agent/Core/Types.cs b/src/Nexus.Agent/Core/Options.cs similarity index 60% rename from src/Nexus.Agent/Core/Types.cs rename to src/Nexus.Agent/Core/Options.cs index 7e9cda6..8325ed2 100644 --- a/src/Nexus.Agent/Core/Types.cs +++ b/src/Nexus.Agent/Core/Options.cs @@ -1,8 +1,31 @@ using System.Runtime.InteropServices; -using System.Text.Json; namespace Nexus.Core; +internal abstract record NexusOptionsBase() +{ + // for testing only + public string? BlindSample { get; set; } + + internal static IConfiguration BuildConfiguration() + { + var environmentName = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); + + var builder = new ConfigurationBuilder() + .AddJsonFile("appsettings.json"); + + if (!string.IsNullOrWhiteSpace(environmentName)) + { + builder + .AddJsonFile($"appsettings.{environmentName}.json", optional: true, reloadOnChange: true); + } + + builder.AddEnvironmentVariables(prefix: "NEXUSAGENT_"); + + return builder.Build(); + } +} + internal record PathsOptions { public const string Section = "Paths"; diff --git a/src/Nexus.Agent/Program.cs b/src/Nexus.Agent/Program.cs index 3aff902..b26263a 100644 --- a/src/Nexus.Agent/Program.cs +++ b/src/Nexus.Agent/Program.cs @@ -6,6 +6,9 @@ var builder = WebApplication.CreateBuilder(); +var configuration = NexusOptionsBase.BuildConfiguration(); +builder.Configuration.AddConfiguration(configuration); + builder.Services .AddOpenApi() @@ -34,6 +37,8 @@ .AddSingleton() .AddSingleton(); +builder.Services.Configure(configuration.GetSection(PathsOptions.Section)); + var app = builder.Build(); app.MapGet("/", () => Results.Redirect("/scalar/v1")).ExcludeFromDescription(); @@ -41,7 +46,12 @@ app.MapScalarApiReference(); app.MapControllers(); -var agent = new Agent(); -_ = agent.RunAsync(); +var pathsOptions = configuration + .GetRequiredSection(PathsOptions.Section) + .Get() ?? throw new Exception("Unable to instantiate path options"); + +var agent = new Agent(pathsOptions); +var extensionHive = await agent.LoadPackagesAsync(); +_ = agent.AcceptClientsAsync(extensionHive); app.Run(); diff --git a/tests/Nexus.Sources.Remote.Tests/AgentTests.cs b/tests/Nexus.Sources.Remote.Tests/AgentTests.cs index 556537b..2d5e261 100644 --- a/tests/Nexus.Sources.Remote.Tests/AgentTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/AgentTests.cs @@ -19,7 +19,7 @@ public async Task CanProvideCatalog() SystemConfiguration: default, SourceConfiguration: new Dictionary() { - ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.Famos") + ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.Remote") }, RequestConfiguration: default ); diff --git a/tests/Nexus.Sources.Remote.Tests/dotnet/remote.cs b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.cs similarity index 87% rename from tests/Nexus.Sources.Remote.Tests/dotnet/remote.cs rename to tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.cs index d3b47cc..10d6868 100644 --- a/tests/Nexus.Sources.Remote.Tests/dotnet/remote.cs +++ b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.cs @@ -4,39 +4,12 @@ using Microsoft.Extensions.Logging; using Nexus.DataModel; using Nexus.Extensibility; -using Nexus.Remoting; -namespace Nexus.Remote; +namespace Nexus.Sources; -#warning Inherit from StructuredFileDataSource would be possible but collides with ReadAndModifyNexusData method" - -public static class Program -{ - public static async Task Main(string[] args) - { - // args - if (args.Length < 2) - throw new Exception("No argument for address and/or port was specified."); - - // get address - var address = args[0]; - - // get port - int port; - - try - { - port = int.Parse(args[1]); - } - catch (Exception ex) - { - throw new Exception("The second command line argument must be a valid port number.", ex); - } - - var communicator = new RemoteCommunicator(new DotnetDataSource(), address, port); - await communicator.RunAsync(); - } -} +/* Note: Inherit from StructuredFileDataSource would be possible + * but collides with ReadAndModifyNexusData method + */ public class DotnetDataSource : IDataSource { @@ -69,11 +42,9 @@ public Task GetCatalogRegistrationsAsync(string path, Can return Task.FromResult(new CatalogRegistration[0]); } - public Task GetCatalogAsync(string catalogId, CancellationToken cancellationToken) + public Task EnrichCatalogAsync(ResourceCatalog catalog, CancellationToken cancellationToken) { - ResourceCatalog catalog; - - if (catalogId == "/A/B/C") + if (catalog.Id == "/A/B/C") { var representation1 = new Representation(NexusDataType.INT64, TimeSpan.FromSeconds(1)); @@ -97,7 +68,7 @@ public Task GetCatalogAsync(string catalogId, CancellationToken .AddResources(resource1, resource2) .Build(); } - else if (catalogId == "/D/E/F") + else if (catalog.Id == "/D/E/F") { var representation = new Representation(NexusDataType.FLOAT64, TimeSpan.FromSeconds(1)); @@ -123,7 +94,7 @@ public Task GetCatalogAsync(string catalogId, CancellationToken if (catalogId != "/A/B/C") throw new Exception("Unknown catalog identifier."); - var filePaths = Directory.GetFiles(_context.ResourceLocator.ToPath(), "*.dat", SearchOption.AllDirectories); + var filePaths = Directory.GetFiles(_context.ResourceLocator!.ToPath(), "*.dat", SearchOption.AllDirectories); var fileNames = filePaths.Select(filePath => Path.GetFileName(filePath)); var dateTimes = fileNames @@ -146,7 +117,7 @@ public Task GetAvailabilityAsync(string catalogId, DateTime begin, DateT var periodPerFile = TimeSpan.FromMinutes(10); var maxFileCount = (end - begin).Ticks / periodPerFile.Ticks; - var filePaths = Directory.GetFiles(_context.ResourceLocator.ToPath(), "*.dat", SearchOption.AllDirectories); + var filePaths = Directory.GetFiles(_context.ResourceLocator!.ToPath(), "*.dat", SearchOption.AllDirectories); var fileNames = filePaths.Select(filePath => Path.GetFileName(filePath)); var actualFileCount = fileNames @@ -212,7 +183,7 @@ CancellationToken cancellationToken while (currentBegin < end) { // find files - var searchPath = Path.Combine(_context.ResourceLocator.ToPath(), currentBegin.ToString("yyyy-MM"), currentBegin.ToString("yyyy-MM-dd")); + var searchPath = Path.Combine(_context.ResourceLocator!.ToPath(), currentBegin.ToString("yyyy-MM"), currentBegin.ToString("yyyy-MM-dd")); var filePaths = Directory.GetFiles(searchPath, "*.dat", SearchOption.AllDirectories); foreach (var filePath in filePaths) diff --git a/tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj similarity index 58% rename from tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj rename to tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj index 7ac8abd..fd76d38 100644 --- a/tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj +++ b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj @@ -2,15 +2,12 @@ $(TargetFrameworkVersion) - Exe - - - - - + + runtime;native + From 58320afb27216ccadbc85f4837af683331092daf Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Fri, 1 Nov 2024 15:31:25 +0100 Subject: [PATCH 04/15] Tests work again --- .nexus-agent/config/packages.json | 2 +- .vscode/launch.json | 4 +- benchmarks/Nexus.Benchmarks/PipeVsTcp.cs | 2 +- .../Core/{Agent.cs => AgentService.cs} | 76 +++++--- src/Nexus.Agent/Program.cs | 25 ++- src/Nexus.Agent/appsettings.json | 8 +- src/Nexus.Sources.Remote/DataSourceTypes.cs | 2 +- src/Nexus.Sources.Remote/Remote.cs | 14 +- src/remoting/dotnet-remoting/Remoting.cs | 176 +++++++++--------- .../AgentFixture.cs | 82 ++++++++ .../Nexus.Sources.Remote.Tests/AgentTests.cs | 35 ---- .../Nexus.Sources.Remote.Tests/RemoteTests.cs | 124 ++++++------ .../SetupDockerTests.cs | 103 ---------- .../SetupDockerTests.sh | 19 -- 14 files changed, 327 insertions(+), 345 deletions(-) rename src/Nexus.Agent/Core/{Agent.cs => AgentService.cs} (72%) create mode 100644 tests/Nexus.Sources.Remote.Tests/AgentFixture.cs delete mode 100644 tests/Nexus.Sources.Remote.Tests/AgentTests.cs delete mode 100644 tests/Nexus.Sources.Remote.Tests/SetupDockerTests.cs delete mode 100644 tests/Nexus.Sources.Remote.Tests/SetupDockerTests.sh diff --git a/.nexus-agent/config/packages.json b/.nexus-agent/config/packages.json index bb259c8..b3fc53e 100644 --- a/.nexus-agent/config/packages.json +++ b/.nexus-agent/config/packages.json @@ -2,7 +2,7 @@ "c05b592f-e198-472d-9902-3f60cf0a6332": { "Provider": "local", "Configuration": { - "path": "tests/Nexus.Sources.Remote.Tests/dotnet", + "path": "../../tests/Nexus.Sources.Remote.Tests/dotnet", "version": "v1", "csproj": "remote.csproj" } diff --git a/.vscode/launch.json b/.vscode/launch.json index 99a222d..17d4fc0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,11 +8,11 @@ "preLaunchTask": "build-nexus-agent", "program": "${workspaceFolder}/artifacts/bin/Nexus.Agent/debug/Nexus.Agent.dll", "args": [], - "cwd": "${workspaceFolder}", + "cwd": "${workspaceFolder}/src/Nexus.Agent", "stopAtEntry": false, "env": { "ASPNETCORE_ENVIRONMENT": "Development", - "NEXUSAGENT_Paths__Config": ".nexus-agent/config" + "NEXUSAGENT_Paths__Config": "../../.nexus-agent/config" } } ] diff --git a/benchmarks/Nexus.Benchmarks/PipeVsTcp.cs b/benchmarks/Nexus.Benchmarks/PipeVsTcp.cs index 53689fa..86376d5 100644 --- a/benchmarks/Nexus.Benchmarks/PipeVsTcp.cs +++ b/benchmarks/Nexus.Benchmarks/PipeVsTcp.cs @@ -26,7 +26,7 @@ public void GlobalSetup() { Arguments = $"{assemblyPath} pipe", UseShellExecute = false, - RedirectStandardInput = true, + RedirectStandardInput = false, RedirectStandardOutput = true, RedirectStandardError = true }; diff --git a/src/Nexus.Agent/Core/Agent.cs b/src/Nexus.Agent/Core/AgentService.cs similarity index 72% rename from src/Nexus.Agent/Core/Agent.cs rename to src/Nexus.Agent/Core/AgentService.cs index ce7f395..54de961 100644 --- a/src/Nexus.Agent/Core/Agent.cs +++ b/src/Nexus.Agent/Core/AgentService.cs @@ -2,7 +2,6 @@ using System.Net; using System.Net.Sockets; using System.Text; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Nexus.Core; using Nexus.Extensibility; @@ -11,7 +10,16 @@ namespace Nexus.Agent; -internal class Agent +public class TcpClientPair +{ + public NetworkStream? Comm { get; set; } + + public NetworkStream? Data { get; set; } + + public RemoteCommunicator? RemoteCommunicator { get; set; } +} + +internal class AgentService { private readonly Lock _lock = new(); @@ -19,13 +27,24 @@ internal class Agent private readonly PathsOptions _pathsOptions; - public Agent(PathsOptions pathsOptions) + private readonly ILogger _agentLogger; + + private readonly ILogger _extensionHiveLogger; + + public AgentService( + IOptions pathsOptions, + ILogger agentLogger, + ILogger extensionHiveLogger) { - _pathsOptions = pathsOptions; + _pathsOptions = pathsOptions.Value; + _agentLogger = agentLogger; + _extensionHiveLogger = extensionHiveLogger; } - public async Task LoadPackagesAsync() + public async Task LoadPackagesAsync(CancellationToken cancellationToken) { + _agentLogger.LogInformation("Load packages"); + var pathsOptions = Options.Create(_pathsOptions); var loggerFactory = new LoggerFactory(); @@ -34,7 +53,8 @@ public async Task LoadPackagesAsync() var extensionHive = new ExtensionHive( pathsOptions, - NullLogger.Instance, loggerFactory + _extensionHiveLogger, + loggerFactory ); var packageReferenceMap = await packageService.GetAllAsync(); @@ -43,40 +63,45 @@ public async Task LoadPackagesAsync() await extensionHive.LoadPackagesAsync( packageReferenceMap: packageReferenceMap, progress, - CancellationToken.None + cancellationToken ); return extensionHive; } - public Task AcceptClientsAsync(IExtensionHive extensionHive) + public Task AcceptClientsAsync(IExtensionHive extensionHive, CancellationToken cancellationToken) { var tcpListener = new TcpListener(IPAddress.Any, 56145); tcpListener.Start(); return Task.Run(async () => { - while (true) + while (!cancellationToken.IsCancellationRequested) { - var client = await tcpListener.AcceptTcpClientAsync(); + var client = await tcpListener.AcceptTcpClientAsync(cancellationToken); _ = Task.Run(async () => { + if (!client.Connected) + throw new Exception("client is not connected"); + var streamReadCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); - using var stream = client.GetStream(); + var networkStream = client.GetStream(); /* no using because it will close the TCP client */ // get connection id var buffer1 = new byte[36]; - await stream.ReadExactlyAsync(buffer1, streamReadCts.Token); + await networkStream.ReadExactlyAsync(buffer1, streamReadCts.Token); var idString = Encoding.UTF8.GetString(buffer1); // get connection type var buffer2 = new byte[4]; - await stream.ReadExactlyAsync(buffer2, streamReadCts.Token); + await networkStream.ReadExactlyAsync(buffer2, streamReadCts.Token); var typeString = Encoding.UTF8.GetString(buffer2); if (Guid.TryParse(Encoding.UTF8.GetString(buffer1), out var id)) { + _agentLogger.LogDebug("Accept TCP client with connection ID {ConnectionId} and communication type {CommunicationType}", idString, typeString); + var tcpPairCts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); // Handle the timeout event @@ -102,11 +127,11 @@ public Task AcceptClientsAsync(IExtensionHive extensionHive) { _tcpClientPairs.AddOrUpdate( id, - addValueFactory: id => new TcpClientPair { Comm = client }, + addValueFactory: id => new TcpClientPair { Comm = networkStream }, updateValueFactory: (id, pair) => { pair.Comm?.Dispose(); - pair.Comm = client; + pair.Comm = networkStream; return pair; } ); @@ -117,20 +142,20 @@ public Task AcceptClientsAsync(IExtensionHive extensionHive) { _tcpClientPairs.AddOrUpdate( id, - addValueFactory: id => new TcpClientPair { Data = client }, + addValueFactory: id => new TcpClientPair { Data = networkStream }, updateValueFactory: (id, pair) => { pair.Data?.Dispose(); - pair.Data = client; + pair.Data = networkStream; return pair; } ); } - // Something went wrong, dispose the client + // Something went wrong, dispose the network stream and return else { - client.Dispose(); + networkStream.Dispose(); return; } @@ -140,6 +165,8 @@ public Task AcceptClientsAsync(IExtensionHive extensionHive) { if (pair.Comm is not null && pair.Data is not null && pair.RemoteCommunicator is null) { + _agentLogger.LogDebug("Accept remoting client with connection ID {ConnectionId}", id); + try { pair.RemoteCommunicator = new RemoteCommunicator( @@ -154,6 +181,8 @@ public Task AcceptClientsAsync(IExtensionHive extensionHive) { pair.Comm?.Dispose(); pair.Data?.Dispose(); + + throw; } } } @@ -162,13 +191,4 @@ public Task AcceptClientsAsync(IExtensionHive extensionHive) } }); } -} - -public class TcpClientPair -{ - public TcpClient? Comm { get; set; } - - public TcpClient? Data { get; set; } - - public RemoteCommunicator? RemoteCommunicator { get; set; } } \ No newline at end of file diff --git a/src/Nexus.Agent/Program.cs b/src/Nexus.Agent/Program.cs index b26263a..d4e96cc 100644 --- a/src/Nexus.Agent/Program.cs +++ b/src/Nexus.Agent/Program.cs @@ -1,3 +1,13 @@ +// TODO +// - cancellation (RemoteCommunicator.RunAsync) +// - client logout / timeout +// - listen to localhost by default, make it configurable +// - ResourceLocator should be used for Nexus.Sources.Remote only, not for remotely connected sources +// - Use ResourceLocator variable in SourceConfiguration to derive totally independent Context. +// - rootless Podman example? +// - "src/Nexus.Agent" -> "src/agent/dotnet-agent/..."? +// - check all code ... especially correctness of namespaces of certain files + using Asp.Versioning; using Nexus.Agent; using Nexus.Core; @@ -35,7 +45,8 @@ builder.Services .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); builder.Services.Configure(configuration.GetSection(PathsOptions.Section)); @@ -48,10 +59,14 @@ var pathsOptions = configuration .GetRequiredSection(PathsOptions.Section) - .Get() ?? throw new Exception("Unable to instantiate path options"); + .Get() ?? throw new Exception("Unable to instantiate paths options"); + +var logger = app.Services.GetRequiredService>(); +logger.LogInformation("Current directory: {CurrentDirectory}", Environment.CurrentDirectory); +logger.LogInformation("Loading configuration from path: {ConfigFolderPath}", pathsOptions.Config); -var agent = new Agent(pathsOptions); -var extensionHive = await agent.LoadPackagesAsync(); -_ = agent.AcceptClientsAsync(extensionHive); +var agent = app.Services.GetRequiredService(); +var extensionHive = await agent.LoadPackagesAsync(CancellationToken.None); +_ = agent.AcceptClientsAsync(extensionHive, CancellationToken.None); app.Run(); diff --git a/src/Nexus.Agent/appsettings.json b/src/Nexus.Agent/appsettings.json index 4d56694..47820ef 100644 --- a/src/Nexus.Agent/appsettings.json +++ b/src/Nexus.Agent/appsettings.json @@ -2,8 +2,12 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Warning", + "Nexus": "Information" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "Paths": { + "BlindSample": "Paths" + } } diff --git a/src/Nexus.Sources.Remote/DataSourceTypes.cs b/src/Nexus.Sources.Remote/DataSourceTypes.cs index 8957741..57eb2d4 100644 --- a/src/Nexus.Sources.Remote/DataSourceTypes.cs +++ b/src/Nexus.Sources.Remote/DataSourceTypes.cs @@ -27,7 +27,7 @@ public Task GetAvailabilityAsync(string catalogId, DateTime begin, DateTime end, CancellationToken cancellationToken); public Task - ReadSingleAsync(DateTime begin, DateTime end, string OriginalResourceName, CatalogItem catalogItem, CancellationToken cancellationToken); + ReadSingleAsync(DateTime begin, DateTime end, string originalResourceName, CatalogItem catalogItem, CancellationToken cancellationToken); } internal record ApiVersionResponse(int ApiVersion); diff --git a/src/Nexus.Sources.Remote/Remote.cs b/src/Nexus.Sources.Remote/Remote.cs index c061ac4..c6ec0ab 100644 --- a/src/Nexus.Sources.Remote/Remote.cs +++ b/src/Nexus.Sources.Remote/Remote.cs @@ -78,7 +78,7 @@ public async Task SetContextAsync( ); var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); - cancellationToken.Register(() => timeoutTokenSource.Cancel()); + cancellationToken.Register(timeoutTokenSource.Cancel); _rpcServer = await _communicator.ConnectAsync(timeoutTokenSource.Token); @@ -100,7 +100,7 @@ public async Task GetCatalogRegistrationsAsync( CancellationToken cancellationToken) { var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); - cancellationToken.Register(() => timeoutTokenSource.Cancel()); + cancellationToken.Register(timeoutTokenSource.Cancel); var response = await _rpcServer .GetCatalogRegistrationsAsync(path, timeoutTokenSource.Token); @@ -113,7 +113,7 @@ public async Task EnrichCatalogAsync( CancellationToken cancellationToken) { var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); - cancellationToken.Register(() => timeoutTokenSource.Cancel()); + cancellationToken.Register(timeoutTokenSource.Cancel); var response = await _rpcServer .EnrichCatalogAsync(catalog, timeoutTokenSource.Token); @@ -126,7 +126,7 @@ public async Task EnrichCatalogAsync( CancellationToken cancellationToken) { var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); - cancellationToken.Register(() => timeoutTokenSource.Cancel()); + cancellationToken.Register(timeoutTokenSource.Cancel); var response = await _rpcServer .GetTimeRangeAsync(catalogId, timeoutTokenSource.Token); @@ -144,7 +144,7 @@ public async Task GetAvailabilityAsync( CancellationToken cancellationToken) { var timeoutTokenSource = GetTimeoutTokenSource(TimeSpan.FromMinutes(1)); - cancellationToken.Register(() => timeoutTokenSource.Cancel()); + cancellationToken.Register(timeoutTokenSource.Cancel); var response = await _rpcServer .GetAvailabilityAsync(catalogId, begin, end, timeoutTokenSource.Token); @@ -210,9 +210,9 @@ private async Task HandleReadDataAsync(string resourcePath, DateTime begin, Date if (!match.Success) throw new Exception("Invalid resource path"); - var samplePeriod = (TimeSpan)_toSamplePeriodMethodInfo.Invoke(null, new object[] { + var samplePeriod = (TimeSpan)_toSamplePeriodMethodInfo.Invoke(null, [ match.Groups["sample_period"].Value - })!; + ])!; // find buffer length and rent buffer var length = (int)((end - begin).Ticks / samplePeriod.Ticks); diff --git a/src/remoting/dotnet-remoting/Remoting.cs b/src/remoting/dotnet-remoting/Remoting.cs index db5b978..76adcfa 100644 --- a/src/remoting/dotnet-remoting/Remoting.cs +++ b/src/remoting/dotnet-remoting/Remoting.cs @@ -49,10 +49,6 @@ public void Log( /// public class RemoteCommunicator { - private readonly TcpClient _comm; - - private readonly TcpClient _data; - private readonly NetworkStream _commStream; private readonly NetworkStream _dataStream; @@ -61,23 +57,22 @@ public class RemoteCommunicator private ILogger _logger = default!; + private IDataSource? _dataSource = default; + /// /// Initializes a new instance of the . /// - /// The TCP client for communications. - /// The TCP client for data. + /// The network stream for communications. + /// The network stream for data. /// A func to get a new data source instance by its type name. public RemoteCommunicator( - TcpClient comm, - TcpClient data, + NetworkStream commStream, + NetworkStream dataStream, Func getDataSource ) { - _comm = comm; - _commStream = comm.GetStream(); - - _data = data; - _dataStream = data.GetStream(); + _commStream = commStream; + _dataStream = dataStream; _getDataSource = getDataSource; } @@ -86,7 +81,7 @@ Func getDataSource /// Starts the remoting operation. /// /// - public async Task RunAsync() + public Task RunAsync() { static JsonElement Read(Span jsonRequest) { @@ -94,85 +89,92 @@ static JsonElement Read(Span jsonRequest) return JsonSerializer.Deserialize(ref reader, Utilities.Options); } - // loop - while (true) + /* Make this method async as early as possible to not block the calling method. + * Otherwise new clients cannot connect because the call to ReadSize may block + * forever, preventing the Lock to be released. + */ + return Task.Run(async () => { - // https://www.jsonrpc.org/specification + // loop + while (true) + { + // https://www.jsonrpc.org/specification - // get request message - var size = ReadSize(_commStream); + // get request message + var size = ReadSize(_commStream); - using var memoryOwner = MemoryPool.Shared.Rent(size); - var messageMemory = memoryOwner.Memory[..size]; + using var memoryOwner = MemoryPool.Shared.Rent(size); + var messageMemory = memoryOwner.Memory[..size]; - _commStream.ReadExactly(messageMemory.Span, _logger); - var request = Read(messageMemory.Span); + _commStream.InternalReadExactly(messageMemory.Span); + var request = Read(messageMemory.Span); - // process message - Memory data = default; - Memory status = default; - JsonObject? response; + // process message + Memory data = default; + Memory status = default; + JsonObject? response; - if (request.TryGetProperty("jsonrpc", out var element) && - element.ValueKind == JsonValueKind.String && - element.GetString() == "2.0") - { - if (request.TryGetProperty("id", out var _)) + if (request.TryGetProperty("jsonrpc", out var element) && + element.ValueKind == JsonValueKind.String && + element.GetString() == "2.0") { - try + if (request.TryGetProperty("id", out var _)) { - (var result, data, status) = await ProcessInvocationAsync(request); + try + { + (var result, data, status) = await ProcessInvocationAsync(request); - response = new JsonObject() + response = new JsonObject() + { + ["result"] = result + }; + } + catch (Exception ex) { - ["result"] = result - }; + response = new JsonObject() + { + ["error"] = new JsonObject() + { + ["code"] = -1, + ["message"] = ex.ToString() + } + }; + } } - catch (Exception ex) + else { - response = new JsonObject() - { - ["error"] = new JsonObject() - { - ["code"] = -1, - ["message"] = ex.ToString() - } - }; + throw new Exception($"JSON-RPC 2.0 notifications are not supported."); } } else { - throw new Exception($"JSON-RPC 2.0 notifications are not supported."); + throw new Exception($"JSON-RPC 2.0 message expected, but got something else."); } - } - else - { - throw new Exception($"JSON-RPC 2.0 message expected, but got something else."); - } - response.Add("jsonrpc", "2.0"); + response.Add("jsonrpc", "2.0"); - string? id; + string? id; - if (request.TryGetProperty("id", out var element2)) - id = element2.ToString(); + if (request.TryGetProperty("id", out var element2)) + id = element2.ToString(); - else - throw new Exception("Unable to read the request message id."); + else + throw new Exception("Unable to read the request message id."); - response.Add("id", id); + response.Add("id", id); - // send response - await Utilities.SendToServerAsync(response, _commStream); + // send response + await Utilities.SendToServerAsync(response, _commStream); - // send data - if (!data.Equals(default) && !status.Equals(default)) - { - await _dataStream.WriteAsync(data); - await _dataStream.WriteAsync(status); - await _dataStream.FlushAsync(); + // send data + if (!data.Equals(default) && !status.Equals(default)) + { + await _dataStream.WriteAsync(data); + await _dataStream.WriteAsync(status); + await _dataStream.FlushAsync(); + } } - } + }); } private async Task<(JsonObject?, Memory, Memory)> ProcessInvocationAsync(JsonElement request) @@ -182,7 +184,6 @@ static JsonElement Read(Span jsonRequest) JsonObject? result = default; Memory data = default; Memory status = default; - IDataSource? dataSource = default; var methodName = request.GetProperty("method").GetString(); var @params = request.GetProperty("params"); @@ -201,7 +202,7 @@ static JsonElement Read(Span jsonRequest) var rawContext = @params[1]; var resourceLocator = default(Uri?); - dataSource = _getDataSource(type); + _dataSource = _getDataSource(type); if (rawContext.TryGetProperty("resourceLocator", out var value)) resourceLocator = new Uri(value.GetString()!); @@ -233,16 +234,16 @@ static JsonElement Read(Span jsonRequest) requestConfiguration ); - await dataSource.SetContextAsync(context, _logger, CancellationToken.None); + await _dataSource.SetContextAsync(context, _logger, CancellationToken.None); } else if (methodName == "getCatalogRegistrations") { - if (dataSource is null) + if (_dataSource is null) throw new Exception("The data source context must be set before invoking other methods."); var path = @params[0].GetString()!; - var registrations = await dataSource.GetCatalogRegistrationsAsync(path, CancellationToken.None); + var registrations = await _dataSource.GetCatalogRegistrationsAsync(path, CancellationToken.None); result = new JsonObject() { @@ -252,11 +253,11 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "enrichCatalog") { - if (dataSource is null) + if (_dataSource is null) throw new Exception("The data source context must be set before invoking other methods."); - var originalCatalog = JsonSerializer.Deserialize(@params[0])!; - var catalog = await dataSource.EnrichCatalogAsync(originalCatalog, CancellationToken.None); + var originalCatalog = JsonSerializer.Deserialize(@params[0], Utilities.Options)!; + var catalog = await _dataSource.EnrichCatalogAsync(originalCatalog, CancellationToken.None); result = new JsonObject() { @@ -266,11 +267,11 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "getTimeRange") { - if (dataSource is null) + if (_dataSource is null) throw new Exception("The data source context must be set before invoking other methods."); var catalogId = @params[0].GetString()!; - var (begin, end) = await dataSource.GetTimeRangeAsync(catalogId, CancellationToken.None); + var (begin, end) = await _dataSource.GetTimeRangeAsync(catalogId, CancellationToken.None); result = new JsonObject() { @@ -281,7 +282,7 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "getAvailability") { - if (dataSource is null) + if (_dataSource is null) throw new Exception("The data source context must be set before invoking other methods."); var catalogId = @params[0].GetString()!; @@ -292,7 +293,7 @@ static JsonElement Read(Span jsonRequest) var endString = @params[2].GetString()!; var end = DateTime.ParseExact(endString, "yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture); - var availability = await dataSource.GetAvailabilityAsync(catalogId, begin, end, CancellationToken.None); + var availability = await _dataSource.GetAvailabilityAsync(catalogId, begin, end, CancellationToken.None); result = new JsonObject() { @@ -302,7 +303,7 @@ static JsonElement Read(Span jsonRequest) else if (methodName == "readSingle") { - if (dataSource is null) + if (_dataSource is null) throw new Exception("The data source context must be set before invoking other methods."); var beginString = @params[0].GetString()!; @@ -317,7 +318,7 @@ static JsonElement Read(Span jsonRequest) (data, status) = ExtensibilityUtilities.CreateBuffers(catalogItem.Representation, begin, end); var readRequest = new ReadRequest(originalResourceName, catalogItem, data, status); - await dataSource.ReadAsync( + await _dataSource.ReadAsync( begin, end, [readRequest], @@ -376,13 +377,13 @@ private async Task HandleReadDataAsync( _logger.LogTrace("Try to read {ByteCount} bytes from Nexus", size); - _dataStream.ReadExactly(MemoryMarshal.AsBytes(buffer.Span), _logger); + _dataStream.InternalReadExactly(MemoryMarshal.AsBytes(buffer.Span)); } private int ReadSize(NetworkStream currentStream) { Span sizeBuffer = stackalloc byte[4]; - currentStream.ReadExactly(sizeBuffer, _logger); + currentStream.InternalReadExactly(sizeBuffer); MemoryExtensions.Reverse(sizeBuffer); var size = BitConverter.ToInt32(sizeBuffer); @@ -429,17 +430,14 @@ public static async Task SendToServerAsync(JsonNode response, NetworkStream curr internal static class StreamExtensions { - public static void ReadExactly(this Stream stream, Span buffer, ILogger logger) + public static void InternalReadExactly(this Stream stream, Span buffer) { while (buffer.Length > 0) { var read = stream.Read(buffer); if (read == 0) - { - logger.LogDebug("No data from Nexus received (exiting)"); - Environment.Exit(0); - } + throw new Exception("The stream has been closed"); buffer = buffer[read..]; } diff --git a/tests/Nexus.Sources.Remote.Tests/AgentFixture.cs b/tests/Nexus.Sources.Remote.Tests/AgentFixture.cs new file mode 100644 index 0000000..7b5b48f --- /dev/null +++ b/tests/Nexus.Sources.Remote.Tests/AgentFixture.cs @@ -0,0 +1,82 @@ +using System.Diagnostics; + +namespace Nexus.Sources.Tests; + +public class AgentFixture : IDisposable +{ + private readonly Process _process; + + private readonly SemaphoreSlim _semaphore = new(0, 1); + + public AgentFixture() + { + Initialize = Task.Run(async () => + { + await _semaphore.WaitAsync(TimeSpan.FromMinutes(1)); + }); + + /* Why not `dotnet run`? Because it spawns a child process for which + * we do not know the process ID and so we cannot kill it. + */ + + // Build Nexus.Agent + var psi_build = new ProcessStartInfo("dotnet") + { + Arguments = $"build ../../../../src/Nexus.Agent/Nexus.Agent.csproj", + UseShellExecute = false + }; + + var process = new Process + { + StartInfo = psi_build + }; + + process.Start(); + process.WaitForExit(); + + if (process.ExitCode != 0) + throw new Exception("Unable to compile Nexus.Agent."); + + // Run Nexus.Agent + var psi_run = new ProcessStartInfo("dotnet") + { + Arguments = $"../../artifacts/bin/Nexus.Agent/debug/Nexus.Agent.dll", + WorkingDirectory="../../../../src/Nexus.Agent", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + psi_run.Environment["NEXUSAGENT_Paths__Config"] = "../../.nexus-agent/config"; + + _process = new Process + { + StartInfo = psi_run, + EnableRaisingEvents = true + }; + + _process.OutputDataReceived += (sender, e) => + { + // File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); + + if (e.Data is not null && e.Data.Contains("Now listening on")) + _semaphore.Release(); + }; + + _process.ErrorDataReceived += (sender, e) => + { + // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + }; + + _process.Start(); + _process.BeginOutputReadLine(); + _process.BeginErrorReadLine(); + } + + public Task Initialize { get; } + + public void Dispose() + { + _process.Kill(); + } +} diff --git a/tests/Nexus.Sources.Remote.Tests/AgentTests.cs b/tests/Nexus.Sources.Remote.Tests/AgentTests.cs deleted file mode 100644 index 2d5e261..0000000 --- a/tests/Nexus.Sources.Remote.Tests/AgentTests.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System.Text.Json; -using Microsoft.Extensions.Logging.Abstractions; -using Nexus.DataModel; -using Nexus.Extensibility; -using Xunit; - -namespace Nexus.Sources.Tests; - -public class AgentTests -{ - [Fact] - public async Task CanProvideCatalog() - { - // Arrange - var dataSource = new Remote() as IDataSource; - - var context = new DataSourceContext( - ResourceLocator: new Uri("file:///" + Path.Combine(Directory.GetCurrentDirectory(), "TESTDATA")), - SystemConfiguration: default, - SourceConfiguration: new Dictionary() - { - ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.Remote") - }, - RequestConfiguration: default - ); - - await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); - - // Act - var actual = await dataSource.EnrichCatalogAsync(new ResourceCatalog("/A/B/C"), CancellationToken.None); - - // Assert - var b = 1; - } -} \ No newline at end of file diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs index 52315c2..d37124f 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs @@ -10,20 +10,25 @@ namespace Nexus.Sources.Tests; -[Trait("TestCategory", "local")] -public class RemoteTests +public class RemoteTests(AgentFixture fixture) + : IClassFixture { - [Theory] - [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] - [InlineData("python python/remote.py localhost {remote-port}")] -#if LINUX - [InlineData("bash bash/remote.sh localhost {remote-port}")] -#endif - public async Task ProvidesCatalog(string command) + private readonly AgentFixture _fixture = fixture; + + [Fact] + // [Theory] + // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] +// [InlineData("python python/remote.py localhost {remote-port}")] +// #if LINUX +// [InlineData("bash bash/remote.sh localhost {remote-port}")] +// #endif + public async Task ProvidesCatalog() { + await _fixture.Initialize; + // Arrange var dataSource = new Remote() as IDataSource; - var context = CreateContext(command); + var context = CreateContext(); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -50,16 +55,19 @@ public async Task ProvidesCatalog(string command) Assert.True(expectedDataTypes.SequenceEqual(actualDataTypes)); } - [Theory] - [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] - [InlineData("python python/remote.py localhost {remote-port}")] -#if LINUX - [InlineData("bash bash/remote.sh localhost {remote-port}")] -#endif - public async Task CanProvideTimeRange(string command) +[Fact] + // [Theory] + // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] +// [InlineData("python python/remote.py localhost {remote-port}")] +// #if LINUX +// [InlineData("bash bash/remote.sh localhost {remote-port}")] +// #endif + public async Task CanProvideTimeRange() { + await _fixture.Initialize; + var dataSource = new Remote() as IDataSource; - var context = CreateContext(command); + var context = CreateContext(); var expectedBegin = new DateTime(2019, 12, 31, 12, 00, 00, DateTimeKind.Utc); var expectedEnd = new DateTime(2020, 01, 02, 09, 50, 00, DateTimeKind.Utc); @@ -72,16 +80,19 @@ public async Task CanProvideTimeRange(string command) Assert.Equal(expectedEnd, end); } - [Theory] - [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] - [InlineData("python python/remote.py localhost {remote-port}")] -#if LINUX - [InlineData("bash bash/remote.sh localhost {remote-port}")] -#endif - public async Task CanProvideAvailability(string command) +[Fact] + // [Theory] + // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] +// [InlineData("python python/remote.py localhost {remote-port}")] +// #if LINUX +// [InlineData("bash bash/remote.sh localhost {remote-port}")] +// #endif + public async Task CanProvideAvailability() { + await _fixture.Initialize; + var dataSource = new Remote() as IDataSource; - var context = CreateContext(command); + var context = CreateContext(); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -92,16 +103,22 @@ public async Task CanProvideAvailability(string command) Assert.Equal(2 / 144.0, actual, precision: 4); } - [Theory] - [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}", true)] - [InlineData("python python/remote.py localhost {remote-port}", true)] -#if LINUX - [InlineData("bash bash/remote.sh localhost {remote-port}", false)] -#endif - public async Task CanReadFullDay(string command, bool complexData) +[Fact] + // [Theory] + // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}", true)] +// [InlineData("python python/remote.py localhost {remote-port}", true)] +// #if LINUX +// [InlineData("bash bash/remote.sh localhost {remote-port}", false)] +// #endif + public async Task CanReadFullDay() { + // TODO fix this + var complexData = true; + + await _fixture.Initialize; + var dataSource = new Remote() as IDataSource; - var context = CreateContext(command); + var context = CreateContext(); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -156,17 +173,20 @@ void GenerateData(DateTimeOffset dateTime) Assert.True(expectedStatus.SequenceEqual(status.ToArray())); } - [Theory] - [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] - [InlineData("python python/remote.py localhost {remote-port}")] -#if LINUX - [InlineData("bash bash/remote.sh localhost {remote-port}")] -#endif - public async Task CanLog(string command) +[Fact] + // [Theory] + // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] +// [InlineData("python python/remote.py localhost {remote-port}")] +// #if LINUX +// [InlineData("bash bash/remote.sh localhost {remote-port}")] +// #endif + public async Task CanLog() { + await _fixture.Initialize; + var loggerMock = new Mock(); var dataSource = new Remote() as IDataSource; - var context = CreateContext(command); + var context = CreateContext(); await dataSource.SetContextAsync(context, loggerMock.Object, CancellationToken.None); @@ -182,13 +202,16 @@ public async Task CanLog(string command) ); } - [Theory] - [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] - [InlineData("python python/remote.py localhost {remote-port}")] - public async Task CanReadDataHandler(string command) +[Fact] + // [Theory] + // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] + // [InlineData("python python/remote.py localhost {remote-port}")] + public async Task CanReadDataHandler() { + await _fixture.Initialize; + var dataSource = new Remote() as IDataSource; - var context = CreateContext(command); + var context = CreateContext(); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -238,7 +261,7 @@ Task HandleReadDataAsync(string resourcePath, DateTime begin, DateTime end, Memo Assert.True(expectedStatus.SequenceEqual(status.ToArray())); } - private static DataSourceContext CreateContext(string command) + private static DataSourceContext CreateContext() { return new DataSourceContext( ResourceLocator: new Uri("file:///" + Path.Combine(Directory.GetCurrentDirectory(), "TESTDATA")), @@ -254,10 +277,7 @@ private static DataSourceContext CreateContext(string command) }, SourceConfiguration: new Dictionary() { - ["listen-address"] = JsonSerializer.SerializeToElement("127.0.0.1"), - ["listen-port-min"] = JsonSerializer.SerializeToElement("63000"), - ["template"] = JsonSerializer.SerializeToElement("local"), - ["command"] = JsonSerializer.SerializeToElement(command), + ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.DotnetDataSource"), ["environment-variables"] = JsonSerializer.SerializeToElement(new JsonObject() { ["PYTHONPATH"] = $"{Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "src", "remoting", "python-remoting")}" diff --git a/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.cs b/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.cs deleted file mode 100644 index 5b5a7d9..0000000 --- a/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.cs +++ /dev/null @@ -1,103 +0,0 @@ -using Microsoft.Extensions.Logging.Abstractions; -using Nexus.DataModel; -using Nexus.Extensibility; -using System.Text.Json; -using System.Text.Json.Nodes; -using Xunit; - -namespace Nexus.Sources.Tests; - -[Trait("TestCategory", "docker")] -public class SetupDockerTests -{ -#if LINUX - [Theory] - [InlineData("python", "main.py nexus-main {remote-port}", "v2.0.0-beta.25")] - [InlineData("dotnet", "nexus-remoting-sample.csproj nexus-main {remote-port}", "v2.0.0-beta.24")] -#endif - public async Task CanReadFullDay(string satelliteId, string command, string version) - { - var dataSource = new Remote() as IDataSource; - var context = CreateContext(satelliteId, command, version); - - await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); - - var catalog = await dataSource.EnrichCatalogAsync(new ResourceCatalog("/A/B/C"), CancellationToken.None); - var resource = catalog.Resources![0]; - var representation = resource.Representations![0]; - - var catalogItem = new CatalogItem( - catalog with { Resources = default! }, - resource with { Representations = default! }, - representation, - default); - - var begin = new DateTime(2020, 01, 01, 0, 0, 0, DateTimeKind.Utc); - var end = new DateTime(2020, 01, 01, 0, 0, 10, DateTimeKind.Utc); - var (data, status) = ExtensibilityUtilities.CreateBuffers(representation, begin, end); - - var length = 10; - var expectedData = new double[length]; - var expectedStatus = new byte[length]; - - for (int i = 0; i < length; i++) - { - expectedData[i] = i * 2; - } - - expectedStatus.AsSpan().Fill(1); - - Task ReadData(string resourcePath, DateTime begin, DateTime end, Memory buffer, CancellationToken cancellationToken) - { - var spanBuffer = buffer.Span; - - for (int i = 0; i < length; i++) - { - spanBuffer[i] = i; - } - - return Task.CompletedTask; - } - - var request = new ReadRequest(resource.Id, catalogItem, data, status); - - await dataSource.ReadAsync( - begin, - end, - [request], - ReadData, - new Progress(), - CancellationToken.None); - - var doubleData = new CastMemoryManager(data).Memory; - - Assert.True(expectedData.SequenceEqual(doubleData.ToArray())); - Assert.True(expectedStatus.SequenceEqual(status.ToArray())); - } - - private static DataSourceContext CreateContext(string satelliteId, string command, string version) - { - return new DataSourceContext( - ResourceLocator: default, - SystemConfiguration: new Dictionary() - { - [typeof(Remote).FullName!] = JsonSerializer.SerializeToElement(new JsonObject() - { - ["templates"] = new JsonObject() - { - ["docker"] = $"ssh root@nexus-{satelliteId} bash run.sh {{git-url}} {{git-tag}} {{command}}" - } - }) - }, - SourceConfiguration: new Dictionary() - { - ["listen-address"] = JsonSerializer.SerializeToElement("0.0.0.0"), - ["template"] = JsonSerializer.SerializeToElement("docker"), - ["command"] = JsonSerializer.SerializeToElement(command), - ["git-url"] = JsonSerializer.SerializeToElement($"https://github.com/nexus-main/nexus-remoting-template-{satelliteId}"), - ["git-tag"] = JsonSerializer.SerializeToElement(version) - }, - RequestConfiguration: default - ); - } -} diff --git a/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.sh b/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.sh deleted file mode 100644 index 2c770a9..0000000 --- a/tests/Nexus.Sources.Remote.Tests/SetupDockerTests.sh +++ /dev/null @@ -1,19 +0,0 @@ -setup_folder="setup/docker" -satellite_ids="python dotnet" - -for satellite_id in $satellite_ids; do - bash "${setup_folder}/setup-host.sh" $satellite_id -done - -docker-compose --file "${setup_folder}/docker-compose.yml" up -d - -while true; do - docker exec "nexus-main" test -f "/var/lib/nexus/ready" && \ - docker exec "nexus-${satellite_id}" test -f "/var/lib/nexus/ready" && \ - break - - echo "Waiting for Docker containers to become ready ..." - sleep 1; -done - -docker exec nexus-main bash -c "cd /root/nexus-sources-remote; dotnet test --filter TestCategory=docker" \ No newline at end of file From fddeddcc3538457a1c187c659f52fadece62e706 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Fri, 1 Nov 2024 22:40:00 +0100 Subject: [PATCH 05/15] Improve context handling --- src/Nexus.Sources.Remote/Remote.cs | 63 +++++---- .../AgentFixture.cs | 82 ------------ .../Nexus.Sources.Remote.Tests/RemoteTests.cs | 9 +- .../RemoteTestsFixture.cs | 120 ++++++++++++++++++ 4 files changed, 164 insertions(+), 110 deletions(-) delete mode 100644 tests/Nexus.Sources.Remote.Tests/AgentFixture.cs create mode 100644 tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs diff --git a/src/Nexus.Sources.Remote/Remote.cs b/src/Nexus.Sources.Remote/Remote.cs index c6ec0ab..527678f 100644 --- a/src/Nexus.Sources.Remote/Remote.cs +++ b/src/Nexus.Sources.Remote/Remote.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Net; using System.Reflection; +using System.Text.Json; using System.Text.RegularExpressions; namespace Nexus.Sources; @@ -14,17 +15,13 @@ namespace Nexus.Sources; "https://github.com/nexus-main/nexus-sources-remote")] public partial class Remote : IDataSource, IDisposable { - #region Fields + private const int DEFAULT_AGENT_PORT = 56145; private ReadDataHandler? _readData; private static readonly int API_LEVEL = 1; private RemoteCommunicator _communicator = default!; private IJsonRpcServer _rpcServer = default!; - #endregion - - #region Properties - /* Possible features to be implemented for this data source: * * Transports: @@ -44,10 +41,6 @@ public partial class Remote : IDataSource, IDisposable private DataSourceContext Context { get; set; } = default!; - #endregion - - #region Methods - public async Task SetContextAsync( DataSourceContext context, ILogger logger, @@ -55,22 +48,40 @@ public async Task SetContextAsync( { Context = context; - // mode - var mode = Context.SourceConfiguration?.GetStringValue("mode") ?? "tcp"; + // Endpoint + if (context.ResourceLocator is null || context.ResourceLocator.Scheme != "tcp") + throw new ArgumentException("The resource locator parameter URI must be set with the 'tcp' scheme."); - if (mode != "tcp") - throw new NotSupportedException($"The mode {mode} is not supported."); + var endpointString = context.ResourceLocator.ToString().Replace("tcp://", "").Replace("/", ""); + var ipAddress = default(IPAddress); - // endpoint - var endpointString = Context.SourceConfiguration?.GetStringValue("endpoint") ?? "127.0.0.1:56145"; + if (!IPEndPoint.TryParse(endpointString, out var endpoint) && !IPAddress.TryParse(endpointString, out ipAddress)) + throw new ArgumentException("The resource locator parameter is not a valid IP endpoint."); - if (!IPEndPoint.TryParse(endpointString, out var endpoint)) - throw new ArgumentException("The endpoint parameter is not a valid IP Endpoint."); + if (endpoint is null) + endpoint = new IPEndPoint(ipAddress!, DEFAULT_AGENT_PORT); - // type + // Type var type = Context.SourceConfiguration?.GetStringValue("type") ?? throw new Exception("The data source type is missing."); - // + // Resource locator + var resourceLocatorString = Context.SourceConfiguration?.GetStringValue("resourceLocator"); + + if (!Uri.TryCreate(resourceLocatorString, UriKind.Absolute, out var resourceLocator)) + throw new ArgumentException("The resource locator parameter is not a valid URI."); + + // Source configuration + var sourceConfigurationJsonElement = Context.SourceConfiguration?.GetValueOrDefault("sourceConfiguration"); + + var sourceConfiguration = sourceConfigurationJsonElement.HasValue && sourceConfigurationJsonElement.Value.ValueKind == JsonValueKind.Object + + ? JsonSerializer + .Deserialize>(sourceConfigurationJsonElement.Value) + + : JsonSerializer + .Deserialize>("{}"); + + // Remote communicator _communicator = new RemoteCommunicator( endpoint, HandleReadDataAsync, @@ -87,12 +98,18 @@ public async Task SetContextAsync( if (apiVersion < 1 || apiVersion > API_LEVEL) throw new Exception($"The API level '{apiVersion}' is not supported."); + // Set context logger.LogTrace("Set context to remote client"); - await _rpcServer - .SetContextAsync(type, context, timeoutTokenSource.Token); + var subContext = new DataSourceContext( + resourceLocator, + context.SystemConfiguration, + sourceConfiguration, + context.RequestConfiguration + ); - logger.LogDebug("Done preparing remote client"); + await _rpcServer + .SetContextAsync(type, subContext, timeoutTokenSource.Token); } public async Task GetCatalogRegistrationsAsync( @@ -236,8 +253,6 @@ private static CancellationTokenSource GetTimeoutTokenSource(TimeSpan timeout) return timeoutToken; } - #endregion - #region IDisposable private bool _disposedValue; diff --git a/tests/Nexus.Sources.Remote.Tests/AgentFixture.cs b/tests/Nexus.Sources.Remote.Tests/AgentFixture.cs deleted file mode 100644 index 7b5b48f..0000000 --- a/tests/Nexus.Sources.Remote.Tests/AgentFixture.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.Diagnostics; - -namespace Nexus.Sources.Tests; - -public class AgentFixture : IDisposable -{ - private readonly Process _process; - - private readonly SemaphoreSlim _semaphore = new(0, 1); - - public AgentFixture() - { - Initialize = Task.Run(async () => - { - await _semaphore.WaitAsync(TimeSpan.FromMinutes(1)); - }); - - /* Why not `dotnet run`? Because it spawns a child process for which - * we do not know the process ID and so we cannot kill it. - */ - - // Build Nexus.Agent - var psi_build = new ProcessStartInfo("dotnet") - { - Arguments = $"build ../../../../src/Nexus.Agent/Nexus.Agent.csproj", - UseShellExecute = false - }; - - var process = new Process - { - StartInfo = psi_build - }; - - process.Start(); - process.WaitForExit(); - - if (process.ExitCode != 0) - throw new Exception("Unable to compile Nexus.Agent."); - - // Run Nexus.Agent - var psi_run = new ProcessStartInfo("dotnet") - { - Arguments = $"../../artifacts/bin/Nexus.Agent/debug/Nexus.Agent.dll", - WorkingDirectory="../../../../src/Nexus.Agent", - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - psi_run.Environment["NEXUSAGENT_Paths__Config"] = "../../.nexus-agent/config"; - - _process = new Process - { - StartInfo = psi_run, - EnableRaisingEvents = true - }; - - _process.OutputDataReceived += (sender, e) => - { - // File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); - - if (e.Data is not null && e.Data.Contains("Now listening on")) - _semaphore.Release(); - }; - - _process.ErrorDataReceived += (sender, e) => - { - // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); - }; - - _process.Start(); - _process.BeginOutputReadLine(); - _process.BeginErrorReadLine(); - } - - public Task Initialize { get; } - - public void Dispose() - { - _process.Kill(); - } -} diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs index d37124f..d251934 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs @@ -10,10 +10,10 @@ namespace Nexus.Sources.Tests; -public class RemoteTests(AgentFixture fixture) - : IClassFixture +public class RemoteTests(RemoteTestsFixture fixture) + : IClassFixture { - private readonly AgentFixture _fixture = fixture; + private readonly RemoteTestsFixture _fixture = fixture; [Fact] // [Theory] @@ -264,7 +264,7 @@ Task HandleReadDataAsync(string resourcePath, DateTime begin, DateTime end, Memo private static DataSourceContext CreateContext() { return new DataSourceContext( - ResourceLocator: new Uri("file:///" + Path.Combine(Directory.GetCurrentDirectory(), "TESTDATA")), + ResourceLocator: new Uri("tcp://127.0.0.1:56145"), SystemConfiguration: new Dictionary() { [typeof(Remote).FullName!] = JsonSerializer.SerializeToElement(new JsonObject() @@ -278,6 +278,7 @@ private static DataSourceContext CreateContext() SourceConfiguration: new Dictionary() { ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.DotnetDataSource"), + ["resourceLocator"] = JsonSerializer.SerializeToElement("file:///" + Path.Combine(Directory.GetCurrentDirectory(), "TESTDATA")), ["environment-variables"] = JsonSerializer.SerializeToElement(new JsonObject() { ["PYTHONPATH"] = $"{Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "src", "remoting", "python-remoting")}" diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs new file mode 100644 index 0000000..9e380bc --- /dev/null +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs @@ -0,0 +1,120 @@ +using System.Diagnostics; + +namespace Nexus.Sources.Tests; + +public class RemoteTestsFixture : IDisposable +{ + private Process? _buildProcess; + + private Process? _runProcess; + + private readonly SemaphoreSlim _semaphoreBuild = new(0, 1); + + private readonly SemaphoreSlim _semaphoreRun = new(0, 1); + + private bool _success; + + public RemoteTestsFixture() + { + Initialize = Task.Run(async () => + { + /* Why not `dotnet run`? Because it spawns a child process for which + * we do not know the process ID and so we cannot kill it. + */ + + // Build Nexus.Agent + var psi_build = new ProcessStartInfo("bash") + { + /* Why `sleep infinity`? Because the test debugger seems to stop whenever a child process stops */ + Arguments = "-c \"dotnet build ../../../../src/Nexus.Agent/Nexus.Agent.csproj && sleep infinity\"", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + _buildProcess = new Process + { + StartInfo = psi_build, + EnableRaisingEvents = true + }; + + _buildProcess.OutputDataReceived += (sender, e) => + { + if (e.Data is not null && e.Data.Contains("Build succeeded")) + { + _success = true; + _semaphoreBuild.Release(); + } + }; + + _buildProcess.ErrorDataReceived += (sender, e) => + { + _success = false; + _semaphoreBuild.Release(); + }; + + _buildProcess.Start(); + _buildProcess.BeginOutputReadLine(); + _buildProcess.BeginErrorReadLine(); + + await _semaphoreBuild.WaitAsync(TimeSpan.FromMinutes(1)); + + if (!_success) + throw new Exception("Unable to build Nexus.Agent."); + + // Run Nexus.Agent + var psi_run = new ProcessStartInfo("dotnet") + { + Arguments = $"../../artifacts/bin/Nexus.Agent/debug/Nexus.Agent.dll", + WorkingDirectory="../../../../src/Nexus.Agent", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + psi_run.Environment["NEXUSAGENT_Paths__Config"] = "../../.nexus-agent/config"; + + _runProcess = new Process + { + StartInfo = psi_run, + EnableRaisingEvents = true + }; + + _runProcess.OutputDataReceived += (sender, e) => + { + // File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); + + if (e.Data is not null && e.Data.Contains("Now listening on")) + { + _success = true; + _semaphoreRun.Release(); + } + }; + + _runProcess.ErrorDataReceived += (sender, e) => + { + // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + + _success = false; + _semaphoreRun.Release(); + }; + + _runProcess.Start(); + _runProcess.BeginOutputReadLine(); + _runProcess.BeginErrorReadLine(); + + await _semaphoreRun.WaitAsync(TimeSpan.FromMinutes(1)); + + if (!_success) + throw new Exception("Unable to launch Nexus.Agent."); + }); + } + + public Task Initialize { get; } + + public void Dispose() + { + _buildProcess?.Kill(); + _runProcess?.Kill(); + } +} From b22561e0544c915c96681a8e758538bd4d403fa1 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Mon, 11 Nov 2024 21:37:21 +0100 Subject: [PATCH 06/15] Make Nexus.Sources.Remote independent of Nexus --- .../PackageReferencesController.cs | 8 +-- .../Controllers/SourcesController.cs | 59 ------------------- src/Nexus.Agent/Core/AgentService.cs | 38 +++++------- src/Nexus.Agent/Core/CustomExtensions.cs | 16 ----- src/Nexus.Agent/Core/DatabaseService.cs | 53 ----------------- .../Core/InternalControllerFeatureProvider.cs | 49 +++++++++++++++ src/Nexus.Agent/Core/Models_Public_v1.cs | 30 ---------- src/Nexus.Agent/Core/Options.cs | 2 +- src/Nexus.Agent/Nexus.Agent.csproj | 10 +--- src/Nexus.Agent/Program.cs | 15 +++-- .../Nexus.Sources.Remote.Tests.csproj | 4 +- 11 files changed, 80 insertions(+), 204 deletions(-) delete mode 100644 src/Nexus.Agent/Controllers/SourcesController.cs delete mode 100644 src/Nexus.Agent/Core/CustomExtensions.cs delete mode 100644 src/Nexus.Agent/Core/DatabaseService.cs create mode 100644 src/Nexus.Agent/Core/InternalControllerFeatureProvider.cs delete mode 100644 src/Nexus.Agent/Core/Models_Public_v1.cs diff --git a/src/Nexus.Agent/Controllers/PackageReferencesController.cs b/src/Nexus.Agent/Controllers/PackageReferencesController.cs index 685005d..1637619 100644 --- a/src/Nexus.Agent/Controllers/PackageReferencesController.cs +++ b/src/Nexus.Agent/Controllers/PackageReferencesController.cs @@ -3,8 +3,8 @@ using Asp.Versioning; using Microsoft.AspNetCore.Mvc; -using Nexus.Core.V1; -using Nexus.Services; +using Nexus.PackageManagement.Services; +using Nexus.PackageManagement; namespace Nexus.Controllers; @@ -32,9 +32,9 @@ internal class PackageReferencesController( /// /// [HttpGet] - public async Task> GetAsync() + public Task> GetAsync() { - return await _packageService.GetAllAsync(); + return _packageService.GetAllAsync(); } /// diff --git a/src/Nexus.Agent/Controllers/SourcesController.cs b/src/Nexus.Agent/Controllers/SourcesController.cs deleted file mode 100644 index adf0fb6..0000000 --- a/src/Nexus.Agent/Controllers/SourcesController.cs +++ /dev/null @@ -1,59 +0,0 @@ -// MIT License -// Copyright (c) [2024] [nexus-main] - -using Asp.Versioning; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Nexus.Core.V1; -using Nexus.Extensibility; -using Nexus.Services; -using System.Reflection; - -namespace Nexus.Controllers; - -/// -/// Provides access to extensions. -/// -[Authorize] -[ApiController] -[ApiVersion("1.0")] -[Route("api/v{version:apiVersion}/[controller]")] -internal class SourcesController( - IExtensionHive extensionHive -) : ControllerBase -{ - // GET /api/sources/descriptions - - private readonly IExtensionHive _extensionHive = extensionHive; - - /// - /// Gets the list of source descriptions. - /// - [HttpGet("descriptions")] - public List GetDescriptions() - { - var result = GetExtensionDescriptions(_extensionHive.GetExtensions()); - return result; - } - - private static List GetExtensionDescriptions( - IEnumerable extensions) - { - return extensions.Select(type => - { - var version = type.Assembly - .GetCustomAttribute()! - .InformationalVersion; - - var attribute = type - .GetCustomAttribute(inherit: false); - - if (attribute is null) - return new ExtensionDescription(type.FullName!, version, default, default, default, default); - - else - return new ExtensionDescription(type.FullName!, version, attribute.Description, attribute.ProjectUrl, attribute.RepositoryUrl, default); - }) - .ToList(); - } -} diff --git a/src/Nexus.Agent/Core/AgentService.cs b/src/Nexus.Agent/Core/AgentService.cs index 54de961..2d9ca88 100644 --- a/src/Nexus.Agent/Core/AgentService.cs +++ b/src/Nexus.Agent/Core/AgentService.cs @@ -5,8 +5,8 @@ using Microsoft.Extensions.Options; using Nexus.Core; using Nexus.Extensibility; +using Nexus.PackageManagement.Services; using Nexus.Remoting; -using Nexus.Services; namespace Nexus.Agent; @@ -25,51 +25,45 @@ internal class AgentService private readonly ConcurrentDictionary _tcpClientPairs = new(); + private readonly IExtensionHive _extensionHive; + + private readonly IPackageService _packageService; + private readonly PathsOptions _pathsOptions; private readonly ILogger _agentLogger; - private readonly ILogger _extensionHiveLogger; + private readonly ILogger _extensionHiveLogger; public AgentService( + IExtensionHive extensionHive, + IPackageService packageService, IOptions pathsOptions, ILogger agentLogger, - ILogger extensionHiveLogger) + ILogger extensionHiveLogger) { + _extensionHive = extensionHive; + _packageService = packageService; _pathsOptions = pathsOptions.Value; _agentLogger = agentLogger; _extensionHiveLogger = extensionHiveLogger; } - public async Task LoadPackagesAsync(CancellationToken cancellationToken) + public async Task LoadPackagesAsync(CancellationToken cancellationToken) { _agentLogger.LogInformation("Load packages"); - var pathsOptions = Options.Create(_pathsOptions); - var loggerFactory = new LoggerFactory(); - - var databaseService = new DatabaseService(pathsOptions); - var packageService = new PackageService(databaseService); - - var extensionHive = new ExtensionHive( - pathsOptions, - _extensionHiveLogger, - loggerFactory - ); - - var packageReferenceMap = await packageService.GetAllAsync(); + var packageReferenceMap = await _packageService.GetAllAsync(); var progress = new Progress(); - await extensionHive.LoadPackagesAsync( + await _extensionHive.LoadPackagesAsync( packageReferenceMap: packageReferenceMap, progress, cancellationToken ); - - return extensionHive; } - public Task AcceptClientsAsync(IExtensionHive extensionHive, CancellationToken cancellationToken) + public Task AcceptClientsAsync(CancellationToken cancellationToken) { var tcpListener = new TcpListener(IPAddress.Any, 56145); tcpListener.Start(); @@ -172,7 +166,7 @@ public Task AcceptClientsAsync(IExtensionHive extensionHive, CancellationToken c pair.RemoteCommunicator = new RemoteCommunicator( pair.Comm, pair.Data, - getDataSource: type => extensionHive.GetInstance(type) + getDataSource: type => _extensionHive.GetInstance(type) ); _ = pair.RemoteCommunicator.RunAsync(); diff --git a/src/Nexus.Agent/Core/CustomExtensions.cs b/src/Nexus.Agent/Core/CustomExtensions.cs deleted file mode 100644 index 4a75256..0000000 --- a/src/Nexus.Agent/Core/CustomExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -// MIT License -// Copyright (c) [2024] [nexus-main] - -using System.Security.Cryptography; -using System.Text; - -namespace Nexus.Core; - -internal static class CustomExtensions -{ - public static byte[] Hash(this string value) - { - var hash = MD5.HashData(Encoding.UTF8.GetBytes(value)); - return hash; - } -} diff --git a/src/Nexus.Agent/Core/DatabaseService.cs b/src/Nexus.Agent/Core/DatabaseService.cs deleted file mode 100644 index 49d06e3..0000000 --- a/src/Nexus.Agent/Core/DatabaseService.cs +++ /dev/null @@ -1,53 +0,0 @@ -// MIT License -// Copyright (c) [2024] [nexus-main] - -using Microsoft.Extensions.Options; -using Nexus.Core; -using System.Diagnostics.CodeAnalysis; - -namespace Nexus.Services; - -internal interface IDatabaseService -{ - /* /config/packages.json */ - bool TryReadPackageReferenceMap([NotNullWhen(true)] out string? packageReferenceMap); - - Stream WritePackageReferenceMap(); -} - -internal class DatabaseService(IOptions pathsOptions) - : IDatabaseService -{ - private readonly PathsOptions _pathsOptions = pathsOptions.Value; - - private const string FILE_EXTENSION = ".json"; - - private const string PACKAGES = "packages"; - - /* /config/packages.json */ - public bool TryReadPackageReferenceMap([NotNullWhen(true)] out string? packageReferenceMap) - { - var folderPath = _pathsOptions.Config; - var packageReferencesFilePath = Path.Combine(folderPath, PACKAGES + FILE_EXTENSION); - - packageReferenceMap = default; - - if (File.Exists(packageReferencesFilePath)) - { - packageReferenceMap = File.ReadAllText(packageReferencesFilePath); - return true; - } - - return false; - } - - public Stream WritePackageReferenceMap() - { - var folderPath = _pathsOptions.Config; - var packageReferencesFilePath = Path.Combine(folderPath, PACKAGES + FILE_EXTENSION); - - Directory.CreateDirectory(folderPath); - - return File.Open(packageReferencesFilePath, FileMode.Create, FileAccess.Write); - } -} \ No newline at end of file diff --git a/src/Nexus.Agent/Core/InternalControllerFeatureProvider.cs b/src/Nexus.Agent/Core/InternalControllerFeatureProvider.cs new file mode 100644 index 0000000..2bc8aa8 --- /dev/null +++ b/src/Nexus.Agent/Core/InternalControllerFeatureProvider.cs @@ -0,0 +1,49 @@ +// MIT License +// Copyright (c) [2024] [nexus-main] + +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.ApplicationParts; +using Microsoft.AspNetCore.Mvc.Controllers; +using System.Reflection; + +namespace Nexus.PackageManagement.Core; + +internal class InternalControllerFeatureProvider : IApplicationFeatureProvider +{ + private const string ControllerTypeNameSuffix = "Controller"; + + public void PopulateFeature(IEnumerable parts, ControllerFeature feature) + { + foreach (var part in parts.OfType()) + { + foreach (var type in part.Types) + { + if (IsController(type) && !feature.Controllers.Contains(type)) + { + feature.Controllers.Add(type); + } + } + } + } + + protected virtual bool IsController(TypeInfo typeInfo) + { + if (!typeInfo.IsClass) + return false; + + if (typeInfo.IsAbstract) + return false; + + if (typeInfo.ContainsGenericParameters) + return false; + + if (typeInfo.IsDefined(typeof(NonControllerAttribute))) + return false; + + if (!typeInfo.Name.EndsWith(ControllerTypeNameSuffix, StringComparison.OrdinalIgnoreCase) && + !typeInfo.IsDefined(typeof(ControllerAttribute))) + return false; + + return true; + } +} diff --git a/src/Nexus.Agent/Core/Models_Public_v1.cs b/src/Nexus.Agent/Core/Models_Public_v1.cs deleted file mode 100644 index 835a77b..0000000 --- a/src/Nexus.Agent/Core/Models_Public_v1.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System.Text.Json; - -namespace Nexus.Core.V1; - -/// -/// A package reference. -/// -/// The provider which loads the package. -/// The configuration of the package reference. -public record PackageReference( - string Provider, - Dictionary Configuration -); - -/// -/// An extension description. -/// -/// The extension type. -/// The extension version. -/// A nullable description. -/// A nullable project website URL. -/// A nullable source repository URL. -/// Additional information about the extension. -public record ExtensionDescription( - string Type, - string Version, - string? Description, - string? ProjectUrl, - string? RepositoryUrl, - IReadOnlyDictionary? AdditionalInformation); \ No newline at end of file diff --git a/src/Nexus.Agent/Core/Options.cs b/src/Nexus.Agent/Core/Options.cs index 8325ed2..8ae8fa4 100644 --- a/src/Nexus.Agent/Core/Options.cs +++ b/src/Nexus.Agent/Core/Options.cs @@ -26,7 +26,7 @@ internal static IConfiguration BuildConfiguration() } } -internal record PathsOptions +internal record PathsOptions : IPackageManagementPathsOptions { public const string Section = "Paths"; diff --git a/src/Nexus.Agent/Nexus.Agent.csproj b/src/Nexus.Agent/Nexus.Agent.csproj index 4ffc1c7..ebc3562 100644 --- a/src/Nexus.Agent/Nexus.Agent.csproj +++ b/src/Nexus.Agent/Nexus.Agent.csproj @@ -8,7 +8,7 @@ - + @@ -18,12 +18,4 @@ - - - - - - - - diff --git a/src/Nexus.Agent/Program.cs b/src/Nexus.Agent/Program.cs index d4e96cc..a59bac5 100644 --- a/src/Nexus.Agent/Program.cs +++ b/src/Nexus.Agent/Program.cs @@ -2,8 +2,6 @@ // - cancellation (RemoteCommunicator.RunAsync) // - client logout / timeout // - listen to localhost by default, make it configurable -// - ResourceLocator should be used for Nexus.Sources.Remote only, not for remotely connected sources -// - Use ResourceLocator variable in SourceConfiguration to derive totally independent Context. // - rootless Podman example? // - "src/Nexus.Agent" -> "src/agent/dotnet-agent/..."? // - check all code ... especially correctness of namespaces of certain files @@ -11,7 +9,7 @@ using Asp.Versioning; using Nexus.Agent; using Nexus.Core; -using Nexus.Services; +using Nexus.PackageManagement.Core; using Scalar.AspNetCore; var builder = WebApplication.CreateBuilder(); @@ -43,13 +41,14 @@ ); builder.Services - .AddSingleton() - .AddSingleton() - .AddSingleton() .AddSingleton(); builder.Services.Configure(configuration.GetSection(PathsOptions.Section)); +// Package management +builder.Services.AddPackageManagement(); +builder.Services.Configure(x => configuration.GetSection(PathsOptions.Section).Bind(new PathsOptions())); + var app = builder.Build(); app.MapGet("/", () => Results.Redirect("/scalar/v1")).ExcludeFromDescription(); @@ -66,7 +65,7 @@ logger.LogInformation("Loading configuration from path: {ConfigFolderPath}", pathsOptions.Config); var agent = app.Services.GetRequiredService(); -var extensionHive = await agent.LoadPackagesAsync(CancellationToken.None); -_ = agent.AcceptClientsAsync(extensionHive, CancellationToken.None); +await agent.LoadPackagesAsync(CancellationToken.None); +_ = agent.AcceptClientsAsync(CancellationToken.None); app.Run(); diff --git a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj index dd62e4d..f2e7ea9 100644 --- a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj +++ b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj @@ -9,7 +9,7 @@ - + @@ -24,7 +24,7 @@ - + From 4c9756ae9162ca74e44e7709b62c47afe56c2155 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Mon, 11 Nov 2024 22:35:43 +0100 Subject: [PATCH 07/15] Prepare rootless container --- .github/workflows/build-and-publish.yml | 65 ++++--- src/Nexus.Agent/Core/Options.cs | 8 +- src/Nexus.Agent/Dockerfile | 6 + .../Nexus.Sources.Remote.Tests/RemoteTests.cs | 33 ---- .../bash/catalog.json | 41 ----- .../Nexus.Sources.Remote.Tests/bash/remote.sh | 159 ------------------ 6 files changed, 50 insertions(+), 262 deletions(-) create mode 100644 src/Nexus.Agent/Dockerfile delete mode 100644 tests/Nexus.Sources.Remote.Tests/bash/catalog.json delete mode 100644 tests/Nexus.Sources.Remote.Tests/bash/remote.sh diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 4cdad3e..d633192 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -9,6 +9,10 @@ on: tags: - '*' + pull_request: + branches: + - dev + jobs: build: @@ -36,41 +40,54 @@ jobs: if: ${{ env.IS_RELEASE == 'true' }} run: python build/create_tag_body.py - - name: Set up Python - uses: actions/setup-python@v3 + - name: Set up dotnet + uses: actions/setup-dotnet@v4 with: - python-version: '3.9' + dotnet-version: "9.0.x" + dotnet-quality: "preview" - - name: Install - run: | - sudo apt install jq - npm install -g pyright - python -m pip install build wheel pytest pytest-asyncio - python -m pip install --pre --index-url https://www.myget.org/F/apollo3zehn-dev/python/ nexus-extensibility + # - name: Set up Python + # uses: actions/setup-python@v3 + # with: + # python-version: '3.9' - - name: Prepare - run: | - chmod +x tests/Nexus.Sources.Remote.Tests/bash/remote.sh + - name: Create Docker Output Folder + run: mkdir --parent artifacts/images + + # - name: Install + # run: | + # npm install -g pyright + # python -m pip install build wheel pytest pytest-asyncio + # python -m pip install --pre --index-url https://www.myget.org/F/apollo3zehn-dev/python/ nexus-extensibility + + - name: Docker Setup + id: buildx + uses: docker/setup-buildx-action@v1 - name: Build run: | - dotnet build -c Release /p:GeneratePackage=true src/remoting/dotnet-remoting/dotnet-remoting.csproj - python -m build --wheel --outdir artifacts/package --no-isolation src/remoting/python-remoting + dotnet publish -c Release -o app /p:GeneratePackage=true src/Nexus.Agent/Nexus.Agent.csproj + # python -m build --wheel --outdir artifacts/package --no-isolation src/remoting/python-remoting - name: Test run: | - dotnet test -c Release --filter TestCategory=local - pyright - pytest - sudo bash tests/Nexus.Sources.Remote.Tests/SetupDockerTests.sh + dotnet test -c Release + # pyright + # pytest + + - name: Docker Build + run: | + docker build -t nexus-main/nexus-agent:v_next . + docker save --output artifacts/images/nexus_agent_image.tar nexus-main/nexus-agent:v_next - name: Upload Artifacts uses: actions/upload-artifact@v3 with: name: artifacts path: | - artifacts/tag_body.txt + artifacts/*.txt artifacts/package/ + artifacts/images/ outputs: is_release: ${{ env.IS_RELEASE }} @@ -98,11 +115,11 @@ jobs: env: MYGET_API_KEY: ${{ secrets.MYGET_API_KEY }} - # GitHub Package Registry does not support Python packages: https://github.community/t/pypi-compatible-github-package-registry/14615 - - name: Python package (MyGet) - run: 'for filePath in artifacts/package/*.whl; do curl -k -X POST https://www.myget.org/F/apollo3zehn-dev/python/upload -H "Authorization: Bearer ${MYGET_API_KEY}" -F "data=@$filePath"; done' - env: - MYGET_API_KEY: ${{ secrets.MYGET_API_KEY }} + # # GitHub Package Registry does not support Python packages: https://github.community/t/pypi-compatible-github-package-registry/14615 + # - name: Python package (MyGet) + # run: 'for filePath in artifacts/package/*.whl; do curl -k -X POST https://www.myget.org/F/apollo3zehn-dev/python/upload -H "Authorization: Bearer ${MYGET_API_KEY}" -F "data=@$filePath"; done' + # env: + # MYGET_API_KEY: ${{ secrets.MYGET_API_KEY }} publish_release: diff --git a/src/Nexus.Agent/Core/Options.cs b/src/Nexus.Agent/Core/Options.cs index 8ae8fa4..73a83e2 100644 --- a/src/Nexus.Agent/Core/Options.cs +++ b/src/Nexus.Agent/Core/Options.cs @@ -32,15 +32,13 @@ internal record PathsOptions : IPackageManagementPathsOptions public string Config { get; set; } = Path.Combine(PlatformSpecificRoot, "config"); - public string Packages { get; set; } = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nexus", "packages"); - // GetGlobalPackagesFolder: https://github.com/NuGet/NuGet.Client/blob/0fc58e13683565e7bdf30e706d49e58fc497bbed/src/NuGet.Core/NuGet.Configuration/Utility/SettingsUtility.cs#L225-L254 - // GetFolderPath: https://github.com/NuGet/NuGet.Client/blob/1d75910076b2ecfbe5f142227cfb4fb45c093a1e/src/NuGet.Core/NuGet.Common/PathUtil/NuGetEnvironment.cs#L54-L57 + public string Packages { get; set; } = Path.Combine(PlatformSpecificRoot, "packages"); #region Support private static string PlatformSpecificRoot { get; } = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Nexus.Agent") - : "/var/lib/nexus-agent"; + ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "nexus-agent") + : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".local", "share", "nexus-agent"); #endregion } \ No newline at end of file diff --git a/src/Nexus.Agent/Dockerfile b/src/Nexus.Agent/Dockerfile new file mode 100644 index 0000000..c42f6d0 --- /dev/null +++ b/src/Nexus.Agent/Dockerfile @@ -0,0 +1,6 @@ +FROM mcr.microsoft.com/dotnet/sdk:9.0 +WORKDIR /app +COPY app . + +USER app +ENTRYPOINT ["./Nexus.Agent"] \ No newline at end of file diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs index d251934..57c41fa 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs @@ -16,12 +16,6 @@ public class RemoteTests(RemoteTestsFixture fixture) private readonly RemoteTestsFixture _fixture = fixture; [Fact] - // [Theory] - // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] -// [InlineData("python python/remote.py localhost {remote-port}")] -// #if LINUX -// [InlineData("bash bash/remote.sh localhost {remote-port}")] -// #endif public async Task ProvidesCatalog() { await _fixture.Initialize; @@ -56,12 +50,6 @@ public async Task ProvidesCatalog() } [Fact] - // [Theory] - // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] -// [InlineData("python python/remote.py localhost {remote-port}")] -// #if LINUX -// [InlineData("bash bash/remote.sh localhost {remote-port}")] -// #endif public async Task CanProvideTimeRange() { await _fixture.Initialize; @@ -81,12 +69,6 @@ public async Task CanProvideTimeRange() } [Fact] - // [Theory] - // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] -// [InlineData("python python/remote.py localhost {remote-port}")] -// #if LINUX -// [InlineData("bash bash/remote.sh localhost {remote-port}")] -// #endif public async Task CanProvideAvailability() { await _fixture.Initialize; @@ -104,12 +86,6 @@ public async Task CanProvideAvailability() } [Fact] - // [Theory] - // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}", true)] -// [InlineData("python python/remote.py localhost {remote-port}", true)] -// #if LINUX -// [InlineData("bash bash/remote.sh localhost {remote-port}", false)] -// #endif public async Task CanReadFullDay() { // TODO fix this @@ -174,12 +150,6 @@ void GenerateData(DateTimeOffset dateTime) } [Fact] - // [Theory] - // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] -// [InlineData("python python/remote.py localhost {remote-port}")] -// #if LINUX -// [InlineData("bash bash/remote.sh localhost {remote-port}")] -// #endif public async Task CanLog() { await _fixture.Initialize; @@ -203,9 +173,6 @@ public async Task CanLog() } [Fact] - // [Theory] - // [InlineData("dotnet run --project ../../../../tests/Nexus.Sources.Remote.Tests/dotnet/remote.csproj localhost {remote-port}")] - // [InlineData("python python/remote.py localhost {remote-port}")] public async Task CanReadDataHandler() { await _fixture.Initialize; diff --git a/tests/Nexus.Sources.Remote.Tests/bash/catalog.json b/tests/Nexus.Sources.Remote.Tests/bash/catalog.json deleted file mode 100644 index 36f6f0e..0000000 --- a/tests/Nexus.Sources.Remote.Tests/bash/catalog.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "Catalog": { - "Id": "/A/B/C", - "Properties": { - "a": "b", - "c": 1 - }, - "Resources": [ - { - "Id": "resource1", - "Properties": { - "unit": "°C", - "groups": [ - "group1" - ] - }, - "Representations": [ - { - "DataType": "INT64", - "SamplePeriod": "00:00:01" - } - ] - }, - { - "Id": "resource2", - "Properties": { - "unit": "bar", - "groups": [ - "group2" - ] - }, - "Representations": [ - { - "DataType": "FLOAT64", - "SamplePeriod": "00:00:01" - } - ] - } - ] - } -} \ No newline at end of file diff --git a/tests/Nexus.Sources.Remote.Tests/bash/remote.sh b/tests/Nexus.Sources.Remote.Tests/bash/remote.sh deleted file mode 100644 index 8e23396..0000000 --- a/tests/Nexus.Sources.Remote.Tests/bash/remote.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/bash -# test command = dotnet test --filter BashRpcDataSourceTests ./tests/Nexus.Core.Tests/Nexus.Core.Tests.csproj - -# quit on error -set -o errexit - -# check prerequisites -if ! command -v jq &> /dev/null; then - echo "I require jq but it's not installed." 1>&2 - exit -fi - -# main -main() { - - # open sockets (https://admin-ahead.com/forum/general-linux/how-to-open-a-tcpudp-socket-in-a-bash-shell/) - echo "Connecting to $1:$2 ..." - - exec 3<>/dev/tcp/$1/$2 - echo -ne "comm" >&3 - - exec 4>/dev/tcp/$1/$2 - echo -ne "data" >&4 - - echo "Starting to listen for JSON-RPC messages ..." - listen - - read dummy -} - -# process incoming messages -listen() { - - while true; do - - # get json length - read32BE json_length <&3 - - # get json - json=$(dd bs=$json_length count=1 <&3 2> /dev/null) - tmp=$(echo $json | jq --raw-output '. | to_entries | map("[\(.key)]=\(.value)") | reduce .[] as $item ("invocation=("; . + $item + " ") + ")"') - declare -A "$tmp" - - jsonrpc=${invocation[jsonrpc]} - id=${invocation[id]} - method=${invocation[method]} - - # check jsonrpc - if [ "$jsonrpc" != "2.0" ]; then - echo "Only JSON-RPC messages are supported." 1>&2 - exit 1 - fi - - # check id - if [ -z "$id" ]; then - echo "Notifications are not supported." 1>&2 - exit 1 - fi - - # prepare response - echo "Received invocation for method '$method'. Preparing response ..." - - if [ "$method" = "getApiVersion" ]; then - response='{ "jsonrpc": "2.0", "id": '$id', "result": { "ApiVersion": 1 } }' - - elif [ "$method" = "setContext" ]; then - response='{ "jsonrpc": "2.0", "id": '$id', "result": null }' - - echo "Sending log message ..." - log_message='{ "jsonrpc": "2.0", "method": "log", "params": [ "Information", "Logging works!" ] }' - declare -i log_message_length=84 - write $log_message_length 32 "dummy" >&3 - printf "$log_message" >&3 - - elif [ "$method" = "getCatalogIds" ]; then - response='{ "jsonrpc": "2.0", "id": '$id', "result": { "CatalogIds": [ "/A/B/C" ] } }' - - elif [ "$method" = "enrichCatalog" ]; then - catalog=$(&2 - exit 1 - - fi - - # get response length - local_lang=$LANG local_lc_all=$LC_ALL - LANG=C LC_ALL=C - byte_length=${#response} - LANG=$local_lang LC_ALL=$local_lc_all - - # send response - echo "Sending response ($byte_length bytes) ..." - write $byte_length 32 "dummy" >&3 - printf "$response" >&3 - - if [ "$method" = "readSingle" ]; then - - # send data (86400 seconds per day * 3 days * 8 bytes) - echo "Sending data ..." - printf 'd%.0s' {1..2073600} >&4 - - # send status - echo "Sending status ..." - printf 's%.0s' {1..259200} >&4 - - fi - - done -} - -# read and write bytes (https://stackoverflow.com/questions/13889659/read-a-file-by-bytes-in-bash) -read8() { - local _r8_var=${1:-OUTBIN} _r8_car LANG=C IFS= - read -r -d '' -n 1 _r8_car - printf -v $_r8_var %d \'$_r8_car; -} - -read16BE() { - local _r16_var=${1:-OUTBIN} _r16_lb _r16_hb - read8 _r16_hb - read8 _r16_lb - printf -v $_r16_var %d $(( _r16_hb<<8 | _r16_lb )); -} - -read32BE() { - local _r32_var=${1:-OUTBIN} _r32_lw _r32_hw - read16BE _r32_hw - read16BE _r32_lw - printf -v $_r32_var %d $(( _r32_hw<<16| _r32_lw )); -} - -# Usage: write [bits:64|32|16|8] [switchto big endian] -write () { - local i=$((${2:-64}/8)) o= v r - r=$((i-1)) - - for ((;i--;)) { - printf -vv '\%03o' $(( ($1>>8*(0${3+-1}?i:r-i))&255 )) - o+=$v - } - - printf "$o" -} - -# run main -main "$@"; exit \ No newline at end of file From cf5df44aadf391848ab479ea7c7621cc9f613a76 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Mon, 11 Nov 2024 22:37:26 +0100 Subject: [PATCH 08/15] Test to trigger pipeline --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index d633192..7c6c423 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -46,7 +46,7 @@ jobs: dotnet-version: "9.0.x" dotnet-quality: "preview" - # - name: Set up Python + # - name: Set up Python # uses: actions/setup-python@v3 # with: # python-version: '3.9' From d69332c69b974ffe63e723ca3bc1ed7e522e81a6 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Mon, 11 Nov 2024 22:39:35 +0100 Subject: [PATCH 09/15] Fixes --- .github/workflows/build-and-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 7c6c423..a3cc2e5 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -4,14 +4,13 @@ on: push: branches: - master - - dev tags: - '*' pull_request: branches: - - dev + - master jobs: From e51180bb75c8cf2478a8e9a28f28444fec038bad Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 12 Nov 2024 12:18:55 +0100 Subject: [PATCH 10/15] Fix paths options issues --- src/Nexus.Agent/Core/AgentService.cs | 12 +----------- src/Nexus.Agent/Nexus.Agent.csproj | 2 +- src/Nexus.Agent/Program.cs | 12 ++++++------ .../Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs | 2 +- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/Nexus.Agent/Core/AgentService.cs b/src/Nexus.Agent/Core/AgentService.cs index 2d9ca88..6f003d5 100644 --- a/src/Nexus.Agent/Core/AgentService.cs +++ b/src/Nexus.Agent/Core/AgentService.cs @@ -2,8 +2,6 @@ using System.Net; using System.Net.Sockets; using System.Text; -using Microsoft.Extensions.Options; -using Nexus.Core; using Nexus.Extensibility; using Nexus.PackageManagement.Services; using Nexus.Remoting; @@ -29,24 +27,16 @@ internal class AgentService private readonly IPackageService _packageService; - private readonly PathsOptions _pathsOptions; - private readonly ILogger _agentLogger; - private readonly ILogger _extensionHiveLogger; - public AgentService( IExtensionHive extensionHive, IPackageService packageService, - IOptions pathsOptions, - ILogger agentLogger, - ILogger extensionHiveLogger) + ILogger agentLogger) { _extensionHive = extensionHive; _packageService = packageService; - _pathsOptions = pathsOptions.Value; _agentLogger = agentLogger; - _extensionHiveLogger = extensionHiveLogger; } public async Task LoadPackagesAsync(CancellationToken cancellationToken) diff --git a/src/Nexus.Agent/Nexus.Agent.csproj b/src/Nexus.Agent/Nexus.Agent.csproj index ebc3562..b89291b 100644 --- a/src/Nexus.Agent/Nexus.Agent.csproj +++ b/src/Nexus.Agent/Nexus.Agent.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Nexus.Agent/Program.cs b/src/Nexus.Agent/Program.cs index a59bac5..d01e64e 100644 --- a/src/Nexus.Agent/Program.cs +++ b/src/Nexus.Agent/Program.cs @@ -4,9 +4,9 @@ // - listen to localhost by default, make it configurable // - rootless Podman example? // - "src/Nexus.Agent" -> "src/agent/dotnet-agent/..."? -// - check all code ... especially correctness of namespaces of certain files using Asp.Versioning; +using Microsoft.Extensions.Options; using Nexus.Agent; using Nexus.Core; using Nexus.PackageManagement.Core; @@ -47,7 +47,9 @@ // Package management builder.Services.AddPackageManagement(); -builder.Services.Configure(x => configuration.GetSection(PathsOptions.Section).Bind(new PathsOptions())); + +builder.Services.AddSingleton( + serviceProvider => serviceProvider.GetRequiredService>().Value); var app = builder.Build(); @@ -56,13 +58,11 @@ app.MapScalarApiReference(); app.MapControllers(); -var pathsOptions = configuration - .GetRequiredSection(PathsOptions.Section) - .Get() ?? throw new Exception("Unable to instantiate paths options"); +var pathsOptions = app.Services.GetRequiredService>(); var logger = app.Services.GetRequiredService>(); logger.LogInformation("Current directory: {CurrentDirectory}", Environment.CurrentDirectory); -logger.LogInformation("Loading configuration from path: {ConfigFolderPath}", pathsOptions.Config); +logger.LogInformation("Loading configuration from path: {ConfigFolderPath}", pathsOptions.Value.Config); var agent = app.Services.GetRequiredService(); await agent.LoadPackagesAsync(CancellationToken.None); diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs index 9e380bc..f9ced55 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs @@ -93,7 +93,7 @@ public RemoteTestsFixture() _runProcess.ErrorDataReceived += (sender, e) => { - // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); _success = false; _semaphoreRun.Release(); From 63c966116743a2861109e9206e3dff27c3d79c8e Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 12 Nov 2024 12:25:56 +0100 Subject: [PATCH 11/15] Test --- tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs index f9ced55..9e380bc 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs @@ -93,7 +93,7 @@ public RemoteTestsFixture() _runProcess.ErrorDataReceived += (sender, e) => { - File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); _success = false; _semaphoreRun.Release(); From ceaec89f46ed47cc843de43d7add76ab1f351be3 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 12 Nov 2024 12:34:12 +0100 Subject: [PATCH 12/15] Fix Dockerfile path --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index a3cc2e5..553afa8 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -76,7 +76,7 @@ jobs: - name: Docker Build run: | - docker build -t nexus-main/nexus-agent:v_next . + docker build -t nexus-main/nexus-agent:v_next -f src/Nexus.Agent/Dockerfile . docker save --output artifacts/images/nexus_agent_image.tar nexus-main/nexus-agent:v_next - name: Upload Artifacts From 32cd8e9f07f35d40af81622cdacc10b4392ca984 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 12 Nov 2024 12:38:27 +0100 Subject: [PATCH 13/15] Add comment --- src/Nexus.Agent/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nexus.Agent/Program.cs b/src/Nexus.Agent/Program.cs index d01e64e..f59e3f0 100644 --- a/src/Nexus.Agent/Program.cs +++ b/src/Nexus.Agent/Program.cs @@ -4,6 +4,7 @@ // - listen to localhost by default, make it configurable // - rootless Podman example? // - "src/Nexus.Agent" -> "src/agent/dotnet-agent/..."? +// - Podman container tests (or remove container and just provide build script = Dockerfile?) using Asp.Versioning; using Microsoft.Extensions.Options; From ac65ce17d4d0a276db897c5f7e2957d2391fec56 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Fri, 15 Nov 2024 08:46:03 +0100 Subject: [PATCH 14/15] Switch to .NET 9 --- src/Nexus.Agent/Nexus.Agent.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nexus.Agent/Nexus.Agent.csproj b/src/Nexus.Agent/Nexus.Agent.csproj index b89291b..ab8955a 100644 --- a/src/Nexus.Agent/Nexus.Agent.csproj +++ b/src/Nexus.Agent/Nexus.Agent.csproj @@ -9,7 +9,7 @@ - + From 1abcae5712d2591fe8c5f1d215379fcf55f63abe Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Thu, 19 Dec 2024 13:16:34 +0100 Subject: [PATCH 15/15] Bump dependencies --- benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj | 2 +- src/Nexus.Agent/Core/Options.cs | 2 +- src/Nexus.Agent/Nexus.Agent.csproj | 4 ++-- src/Nexus.Agent/Program.cs | 2 +- src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj | 4 ++-- src/remoting/dotnet-remoting/dotnet-remoting.csproj | 2 +- .../Nexus.Sources.Remote.Tests.csproj | 10 +++++----- .../Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj b/benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj index 9857d80..a34fbf9 100644 --- a/benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj +++ b/benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/Nexus.Agent/Core/Options.cs b/src/Nexus.Agent/Core/Options.cs index 73a83e2..e75b624 100644 --- a/src/Nexus.Agent/Core/Options.cs +++ b/src/Nexus.Agent/Core/Options.cs @@ -2,7 +2,7 @@ namespace Nexus.Core; -internal abstract record NexusOptionsBase() +internal abstract record NexusAgentOptions() { // for testing only public string? BlindSample { get; set; } diff --git a/src/Nexus.Agent/Nexus.Agent.csproj b/src/Nexus.Agent/Nexus.Agent.csproj index ab8955a..a0b15d2 100644 --- a/src/Nexus.Agent/Nexus.Agent.csproj +++ b/src/Nexus.Agent/Nexus.Agent.csproj @@ -8,9 +8,9 @@ - + - + diff --git a/src/Nexus.Agent/Program.cs b/src/Nexus.Agent/Program.cs index f59e3f0..a3ea36c 100644 --- a/src/Nexus.Agent/Program.cs +++ b/src/Nexus.Agent/Program.cs @@ -15,7 +15,7 @@ var builder = WebApplication.CreateBuilder(); -var configuration = NexusOptionsBase.BuildConfiguration(); +var configuration = NexusAgentOptions.BuildConfiguration(); builder.Configuration.AddConfiguration(configuration); builder.Services diff --git a/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj b/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj index 42c1214..3387817 100644 --- a/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj +++ b/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj @@ -11,11 +11,11 @@ - + runtime;native - + diff --git a/src/remoting/dotnet-remoting/dotnet-remoting.csproj b/src/remoting/dotnet-remoting/dotnet-remoting.csproj index 273a687..2dc0726 100644 --- a/src/remoting/dotnet-remoting/dotnet-remoting.csproj +++ b/src/remoting/dotnet-remoting/dotnet-remoting.csproj @@ -26,7 +26,7 @@ - + diff --git a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj index f2e7ea9..446368f 100644 --- a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj +++ b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj @@ -13,11 +13,11 @@ - - - - - + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj index fd76d38..96a84c4 100644 --- a/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj +++ b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj @@ -5,7 +5,7 @@ - + runtime;native