From 40beefcd477bf248d831ae36f61391748a2e5116 Mon Sep 17 00:00:00 2001 From: Aday Bujeda Date: Tue, 5 Nov 2024 17:19:53 +0000 Subject: [PATCH] Added metrics widget to OOD demo app to deploy to Dev environments --- Makefile | 5 +- config/demo/.env | 1 + .../demo/app_overrides/ondemand.d/metrics.yml | 7 +++ .../initializers/configuration_extension.rb | 3 +- config/local/app_overrides/jobs/README.md | 6 ++ .../app_overrides/jobs/example_job.sbatch | 18 ++++++ .../local/app_overrides/jobs/submit_jobs.sh | 12 ++++ config/local/{ => dashboard}/.env | 0 config/local/dashboard/config/application.rb | 62 +++++++++++++++++++ dev/metrics/README.md | 41 ++++++++++++ .../metrics}/initializers/slurm_extension.rb | 0 .../lib/slurm_metrics/metrics_helper.rb | 0 .../lib/slurm_metrics/metrics_processor.rb | 0 .../lib/slurm_metrics/metrics_service.rb | 0 .../lib/slurm_metrics/metrics_summary.rb | 0 .../lib/slurm_metrics/slurm_data_converter.rb | 0 dev/metrics/metrics.yml | 7 +++ .../views/widgets/metrics/_fairshare.html.erb | 0 .../widgets/metrics/_jobs_summary.html.erb | 0 .../widgets/metrics/_metric_details.html.erb | 0 .../views/widgets/metrics/_metrics.html.erb | 0 .../widgets/metrics/_metrics_summary.html.erb | 0 docker-compose.yml | 3 +- docs/fasrc_puppet.md | 27 ++++++++ 24 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 config/demo/app_overrides/ondemand.d/metrics.yml create mode 100644 config/local/app_overrides/jobs/README.md create mode 100644 config/local/app_overrides/jobs/example_job.sbatch create mode 100755 config/local/app_overrides/jobs/submit_jobs.sh rename config/local/{ => dashboard}/.env (100%) create mode 100644 config/local/dashboard/config/application.rb create mode 100644 dev/metrics/README.md rename {config/local/app_overrides => dev/metrics}/initializers/slurm_extension.rb (100%) rename {config/local/app_overrides => dev/metrics}/lib/slurm_metrics/metrics_helper.rb (100%) rename {config/local/app_overrides => dev/metrics}/lib/slurm_metrics/metrics_processor.rb (100%) rename {config/local/app_overrides => dev/metrics}/lib/slurm_metrics/metrics_service.rb (100%) rename {config/local/app_overrides => dev/metrics}/lib/slurm_metrics/metrics_summary.rb (100%) rename {config/local/app_overrides => dev/metrics}/lib/slurm_metrics/slurm_data_converter.rb (100%) create mode 100644 dev/metrics/metrics.yml rename {config/local/app_overrides => dev/metrics}/views/widgets/metrics/_fairshare.html.erb (100%) rename {config/local/app_overrides => dev/metrics}/views/widgets/metrics/_jobs_summary.html.erb (100%) rename {config/local/app_overrides => dev/metrics}/views/widgets/metrics/_metric_details.html.erb (100%) rename {config/local/app_overrides => dev/metrics}/views/widgets/metrics/_metrics.html.erb (100%) rename {config/local/app_overrides => dev/metrics}/views/widgets/metrics/_metrics_summary.html.erb (100%) create mode 100644 docs/fasrc_puppet.md diff --git a/Makefile b/Makefile index 21d870a9..0bbe513f 100644 --- a/Makefile +++ b/Makefile @@ -39,11 +39,14 @@ build_latest_ood: # BUILD OOD WITH SID OOD IMAGE #docker run --rm -v $(WORKING_DIR):/usr/local/app -w /usr/local/app $(SID_OOD_IMAGE) ./ood_build.sh # BUILD OOD WITH ROCKY8 IMAGE - cp -f config/local/.env ondemand/apps/dashboard + cp -R config/local/dashboard/. ondemand/apps/dashboard docker run --rm -v $(WORKING_DIR):/usr/local/app -w /usr/local/app -e ROOT_URL=$(ROOT_URL) $(SID_BUILDER_IMAGE) ./ood_build.sh build_system_demo build_user_demo: build_latest_ood # COPY DEMO CONFIGURATION cp -R config/demo/. ondemand/apps/dashboard + # COPY METRICS WIDGET + mkdir -p ondemand/apps/dashboard/plugins + cp -R dev/metrics ondemand/apps/dashboard/plugins/metrics start_ood_installer: docker create --rm --name ood_installer --privileged -p 43000:443 ood_puppet:5.0.1 diff --git a/config/demo/.env b/config/demo/.env index c8c0ec4a..8a89fece 100644 --- a/config/demo/.env +++ b/config/demo/.env @@ -6,5 +6,6 @@ ENABLE_NATIVE_VNC=true OOD_DATAROOT_BAK=~/.fasrcood/data/sys/dashboard OOD_VERSION=latest OOD_CONFIG_D_DIRECTORY=app_overrides/ondemand.d +OOD_PLUGINS_DIRECTORY=plugins OOD_LOAD_EXTERNAL_CONFIG=true OOD_APP_CONFIG_ROOT=app_overrides diff --git a/config/demo/app_overrides/ondemand.d/metrics.yml b/config/demo/app_overrides/ondemand.d/metrics.yml new file mode 100644 index 00000000..2fde83a1 --- /dev/null +++ b/config/demo/app_overrides/ondemand.d/metrics.yml @@ -0,0 +1,7 @@ +custom_pages: + metrics: + rows: + - columns: + - width: 12 + widgets: + - "metrics/metrics" diff --git a/config/local/app_overrides/initializers/configuration_extension.rb b/config/local/app_overrides/initializers/configuration_extension.rb index d6187e3c..33450de9 100644 --- a/config/local/app_overrides/initializers/configuration_extension.rb +++ b/config/local/app_overrides/initializers/configuration_extension.rb @@ -1,4 +1,5 @@ -# SessionsHelper extension to support TurboVNC and KVM +# frozen_string_literal: true +# ConfigurationSingleton extension Rails.application.config.after_initialize do Rails.logger.info 'Executing Configuration extension ...' class ConfigurationSingleton diff --git a/config/local/app_overrides/jobs/README.md b/config/local/app_overrides/jobs/README.md new file mode 100644 index 00000000..7fae4bd4 --- /dev/null +++ b/config/local/app_overrides/jobs/README.md @@ -0,0 +1,6 @@ +## Create Slurm Data For Metrics +Execute the following script to create jobs in Slurm and generate metrics data: + - `/etc/ood/config/apps/ood/jobs/submit_jobs.sh` + +The script will take some time to complete. +Check the status of the jobs in the active jobs page in OOD: https://localhost:33000/pun/sys/ood/activejobs \ No newline at end of file diff --git a/config/local/app_overrides/jobs/example_job.sbatch b/config/local/app_overrides/jobs/example_job.sbatch new file mode 100644 index 00000000..2ac874fb --- /dev/null +++ b/config/local/app_overrides/jobs/example_job.sbatch @@ -0,0 +1,18 @@ +#!/bin/sh +#SBATCH --time=00:05:00 +#SBATCH --mail-type=NONE + +case $MODE in + 'stress') + srun yes > /dev/null + ;; + 'sleep') + srun sleep 120 + ;; + 'timeout') + srun sleep 100000 + ;; + *) + exit 1 + ;; +esac \ No newline at end of file diff --git a/config/local/app_overrides/jobs/submit_jobs.sh b/config/local/app_overrides/jobs/submit_jobs.sh new file mode 100755 index 00000000..d48a4e42 --- /dev/null +++ b/config/local/app_overrides/jobs/submit_jobs.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +users='ood' +opmodes='sleep stress timeout' + +for ((node_count=1; node_count <= 2; node_count++)); do + for user in $users; do + for mode in $opmodes; do + su - ood -c "sbatch -n $node_count --job-name=$user_$script_$node_count --export='ALL,MODE='$mode /etc/ood/config/apps/ood/jobs/example_job.sbatch" + done + done +done \ No newline at end of file diff --git a/config/local/.env b/config/local/dashboard/.env similarity index 100% rename from config/local/.env rename to config/local/dashboard/.env diff --git a/config/local/dashboard/config/application.rb b/config/local/dashboard/config/application.rb new file mode 100644 index 00000000..8d73fbea --- /dev/null +++ b/config/local/dashboard/config/application.rb @@ -0,0 +1,62 @@ +# CUSTOM VERSION OF THE config/application.rb FILE TO LOAD CUSTOMIZATIONS AS PLUGINS +# EASIER TO DEPLOY AND DEVELOP LOCALLY +require_relative 'boot' + +require "rails" +# Pick the frameworks you want: +require "active_model/railtie" +require "active_job/railtie" +# require "active_record/railtie" +# require "active_storage/engine" +require "action_controller/railtie" +require "action_mailer/railtie" +# require "action_mailbox/engine" +# require "action_text/engine" +require "action_view/railtie" +# require "action_cable/engine" +require "sprockets/railtie" +require "rails/test_unit/railtie" + +# Require the gems listed in Gemfile, including any gems +# you've limited to :test, :development, or :production. +Bundler.require(*Rails.groups) + +module Dashboard + class Application < Rails::Application + # Initialize configuration defaults for originally generated Rails version. + config.load_defaults(7.0) + + # Configuration for the application, engines, and railties goes here. + # + # These settings can be overridden in specific environments using the files + # in config/environments, which are processed later. + # + # config.time_zone = "Central Time (US & Canada)" + # config.eager_load_paths << Rails.root.join("extras") + + config.autoload_paths << Rails.root.join('lib') + + # Locales are handled in config/initializers/locales.rb. + + # Custom error pages + config.exceptions_app = self.routes + + if ::Configuration.load_external_config? + # Ensuring OOD initializers run last so that user's cannot override what we + # specify unless we allow the override as well in our own initializers. + config.paths["config/initializers"] << ::Configuration.config_root.join("initializers").to_s + config.autoload_paths << ::Configuration.config_root.join("lib").to_s + config.paths["app/views"].unshift ::Configuration.config_root.join("views").to_s + end + + # Enable installed plugins + plugins_dir = Pathname.new(ENV['OOD_PLUGINS_DIRECTORY'] || '/var/www/ood/apps/plugins') + if plugins_dir.directory? + plugins_dir.children.select(&:directory?).each do |installed_plugin| + config.paths["config/initializers"] << installed_plugin.join("initializers").to_s + config.autoload_paths << installed_plugin.join("lib").to_s + config.paths["app/views"].unshift installed_plugin.join("views").to_s + end + end + end +end diff --git a/dev/metrics/README.md b/dev/metrics/README.md new file mode 100644 index 00000000..7408a1ba --- /dev/null +++ b/dev/metrics/README.md @@ -0,0 +1,41 @@ +## IQSS Metrics Widget For OnDemand +Open OnDemand metrics widget displaying information about fairshare, job status summary, memory, CPU, GPU, and time. Data is retrieved directly from Slurm using the `sacct` and `sshare` commands. + +The metrics widget consist of four panels: +- Fairshare +- CPU Jobs by state +- GPU Jobs by state +- Jobs Stats Summary + +### Implementation Summary +The implementation consist of three components +- OOD Slurm adapter extension +- ./initializers/slurm_extension.rb +- Metrics calculations utility classes +- ./lib/slurm_metrics +- Metrics widget templates +- ./views/widgets/metrics + +### Deployment +Using the customization feature from OnDemand, with the default location under: `/etc/ood/config/apps/dashboard` + - copy `./initializers/slurm_extension.rb` into `/etc/ood/config/apps/dashboard/intializers` + - copy `./lib/slurm_metrics` into `/etc/ood/config/apps/dashboard/lib` + - copy `./views/widgets/metrics` into `/etc/ood/config/apps/dashboard/views/widgets` + +Restart the OnDemand application for the customizations to take effect. + +### Deployment With FASRC Puppet +The widget components need to be deployed using FASRC Puppet control repo. We are already using the OOD Puppet module feature to add files to the OOD dashboard location to add/extend functionality: +`openondemand::apps_config_source:` + +The folder that we are deploying is: `site-modules/profile/files/openondemand/common/apps_config` + +In the Puppet control repo, we need to add the files for the three components to the following folders: +- site-modules/profile/files/openondemand/common/apps_config/dashboard/intializers +- site-modules/profile/files/openondemand/common/apps_config/dashboard/lib +- site-modules/profile/files/openondemand/common/apps_config/dashboard/views/widgets + +To review and test the new widget, we could use a custom page to display it. This is a sample configuration: +https://github.com/hmdc/ondemand_development/blob/main/config/local/ondemand.d/metrics.yml + +This will create a custom page under: `/pun/sys/dashboard/custom/metrics \ No newline at end of file diff --git a/config/local/app_overrides/initializers/slurm_extension.rb b/dev/metrics/initializers/slurm_extension.rb similarity index 100% rename from config/local/app_overrides/initializers/slurm_extension.rb rename to dev/metrics/initializers/slurm_extension.rb diff --git a/config/local/app_overrides/lib/slurm_metrics/metrics_helper.rb b/dev/metrics/lib/slurm_metrics/metrics_helper.rb similarity index 100% rename from config/local/app_overrides/lib/slurm_metrics/metrics_helper.rb rename to dev/metrics/lib/slurm_metrics/metrics_helper.rb diff --git a/config/local/app_overrides/lib/slurm_metrics/metrics_processor.rb b/dev/metrics/lib/slurm_metrics/metrics_processor.rb similarity index 100% rename from config/local/app_overrides/lib/slurm_metrics/metrics_processor.rb rename to dev/metrics/lib/slurm_metrics/metrics_processor.rb diff --git a/config/local/app_overrides/lib/slurm_metrics/metrics_service.rb b/dev/metrics/lib/slurm_metrics/metrics_service.rb similarity index 100% rename from config/local/app_overrides/lib/slurm_metrics/metrics_service.rb rename to dev/metrics/lib/slurm_metrics/metrics_service.rb diff --git a/config/local/app_overrides/lib/slurm_metrics/metrics_summary.rb b/dev/metrics/lib/slurm_metrics/metrics_summary.rb similarity index 100% rename from config/local/app_overrides/lib/slurm_metrics/metrics_summary.rb rename to dev/metrics/lib/slurm_metrics/metrics_summary.rb diff --git a/config/local/app_overrides/lib/slurm_metrics/slurm_data_converter.rb b/dev/metrics/lib/slurm_metrics/slurm_data_converter.rb similarity index 100% rename from config/local/app_overrides/lib/slurm_metrics/slurm_data_converter.rb rename to dev/metrics/lib/slurm_metrics/slurm_data_converter.rb diff --git a/dev/metrics/metrics.yml b/dev/metrics/metrics.yml new file mode 100644 index 00000000..2fde83a1 --- /dev/null +++ b/dev/metrics/metrics.yml @@ -0,0 +1,7 @@ +custom_pages: + metrics: + rows: + - columns: + - width: 12 + widgets: + - "metrics/metrics" diff --git a/config/local/app_overrides/views/widgets/metrics/_fairshare.html.erb b/dev/metrics/views/widgets/metrics/_fairshare.html.erb similarity index 100% rename from config/local/app_overrides/views/widgets/metrics/_fairshare.html.erb rename to dev/metrics/views/widgets/metrics/_fairshare.html.erb diff --git a/config/local/app_overrides/views/widgets/metrics/_jobs_summary.html.erb b/dev/metrics/views/widgets/metrics/_jobs_summary.html.erb similarity index 100% rename from config/local/app_overrides/views/widgets/metrics/_jobs_summary.html.erb rename to dev/metrics/views/widgets/metrics/_jobs_summary.html.erb diff --git a/config/local/app_overrides/views/widgets/metrics/_metric_details.html.erb b/dev/metrics/views/widgets/metrics/_metric_details.html.erb similarity index 100% rename from config/local/app_overrides/views/widgets/metrics/_metric_details.html.erb rename to dev/metrics/views/widgets/metrics/_metric_details.html.erb diff --git a/config/local/app_overrides/views/widgets/metrics/_metrics.html.erb b/dev/metrics/views/widgets/metrics/_metrics.html.erb similarity index 100% rename from config/local/app_overrides/views/widgets/metrics/_metrics.html.erb rename to dev/metrics/views/widgets/metrics/_metrics.html.erb diff --git a/config/local/app_overrides/views/widgets/metrics/_metrics_summary.html.erb b/dev/metrics/views/widgets/metrics/_metrics_summary.html.erb similarity index 100% rename from config/local/app_overrides/views/widgets/metrics/_metrics_summary.html.erb rename to dev/metrics/views/widgets/metrics/_metrics_summary.html.erb diff --git a/docker-compose.yml b/docker-compose.yml index 6b0f0e32..e77ce0e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,10 +34,11 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - ./data:/home/ood/ondemand/data - ./ondemand/apps/dashboard:/var/www/ood/apps/sys/ood - - ./config/local/.env:/var/www/ood/apps/sys/ood/.env + - ./config/local/dashboard/.env:/var/www/ood/apps/sys/ood/.env - ./config/local/public:/var/www/ood/public - ./config/local/ondemand.d:/etc/ood/config/ondemand.d - ./config/local/app_overrides:/etc/ood/config/apps/ood + - ./dev/metrics:/var/www/ood/apps/plugins/metrics ports: - "33000:443" expose: diff --git a/docs/fasrc_puppet.md b/docs/fasrc_puppet.md new file mode 100644 index 00000000..1e04f2a9 --- /dev/null +++ b/docs/fasrc_puppet.md @@ -0,0 +1,27 @@ +### FASRC Puppet YAML Hierarchy +```yaml +hierarchy: +- name: "Secret data (encrypted)" + lookup_key: eyaml_lookup_key + paths: + - "nodes/%{trusted.certname}.yaml" + - "secrets/%{trusted.certname}.yaml" + - "nodes/%{hostname}.yaml" + - "secrets/%{hostname_base}.yaml" + - "nodes/%{hostname_base}.yaml" + - "ib_islands/%{::ib_island}.yaml" + - "vlans/%{vlan}.yaml" + - "vlans/%{vlan}_%{facts.os.family}_%{facts.os.release.major}.yaml" + - "cluster/%{cluster}.yaml" + - "groups/%{group}.yaml" + - "kempner_%{kempner}.yaml" + - "datacenter/%{datacenter}/proxy_%{needs_proxy}.yaml" + - "datacenter/%{datacenter}.yaml" + - "security_level/level%{security_level}/%{facts.os.family}/%{facts.os.release.major}.yaml" + - "os/%{facts.os.family}.yaml" + - "os/%{facts.os.family}/%{facts.os.release.major}.yaml" + - 'common/mounts.yaml' + - "common/is_virtual_%{is_virtual}.yaml" + - 'common/common.yaml' + - "secrets/common.yaml" +``` \ No newline at end of file