(SIMP-3784) Replace the compliance mapper's report generator (#26)

Replace the compliance mapper's report generator parsing code
with the shared library used by the enforcement backend. This
means that the code to read and parse v1 format compliance maps
is now centralized.

Add vendored (maps-in-modules) support to the compiler for v1
format maps. Note: the operation of this is not well defined, and
intended for use in the v2 format of the compliance maps.

Also expand the shared library to a real object tree, with defined
api endpoints, like load() to load a set of compliance maps, and
list_puppet_params() to generate a puppet parameter hash.

Note: this code was tested using the existing spec tests without
modifications to ensure that the change is a drop-in replacement.
The code coverage of the tests may not be complete to do this.

SIMP-3784 #close

.gitignore

@@ -10,6 +10,7 @@ spec/rp_env/
 .rspec_system
 .vagrant/
 .bundle/
+.idea/
 Gemfile.lock
 vendor/
 junit/

.travis.yml

@@ -8,52 +8,92 @@
 # PE 2017.2   4.10  2.1.9  TBD
 ---
 language: ruby
-sudo: false
 cache: bundler
-before_script:
-  - bundle update
+sudo: false
+
 bundler_args: --without development system_tests --path .vendor
-before_install:
-  - rm Gemfile.lock || true
-  - rvm @global do gem uninstall bundler -a -x
-  - rvm @global do gem install bundler -v '~> 1.14.0'
-script:
-  - bundle exec rake compare_latest_tag
-  - bundle exec rake test
+
 notifications:
   email: false
-rvm:
-  - 2.1.9
-env:
-  global:
-    - STRICT_VARIABLES=yes
-  matrix:
-    - PUPPET_VERSION="~> 4.8.2" FORGE_PUBLISH=true
-    - PUPPET_VERSION="~> 4.10.0"
-    - PUPPET_VERSION="~> 4.9.2"
-    - PUPPET_VERSION="~> 4.7.0"
-matrix:
-  fast_finish: true
-
-before_deploy:
-  - 'bundle exec rake metadata_lint'
-  - 'bundle exec rake clobber'
-  - 'bundle exec rake spec_clean'
-  - "export PUPMOD_METADATA_VERSION=`ruby -r json -e \"puts JSON.parse(File.read('metadata.json')).fetch('version')\"`"
-  - '[[ $TRAVIS_TAG =~ ^${PUPMOD_METADATA_VERSION}$ ]]'
-deploy:
-  - provider: puppetforge
-    user: simp
-    password:
-        secure: "vL1MNFrE7aazh/rEsSaHugCYYWXrJmsBuhMIP27CQ/EpBQ6wfWZhukZdkikZrmkIiJHHLi1HaQycKdW2AzcqN932iXOdjYtEA9eE/hO1VtGMYVobZFS4sh2Wtt8saVOg0tMPq45hjFHUe8FmgyrsjHBB+kl/fdfLVr+TiFCmWGXtZkjMlJxqPp+fyZZDrMoKoB7eSm3edOtqX7gONP3MEJ/wcHgCUTyspxI8sSXGl8IPwWNxU4LSgCwbvr/JzmDmKiK2AbSc7Q2+g7NrvZV39CbcV0IH4Uy4A+3fBBli/nPaidm/dvIvcYDacWF0UiXZTMvqd8tL4Z3Sf4vvB6NZ328ml97w233fLSHdLGn8qaIjmkK5x1UN9BRmA/WZbmkSSLWqwr6Cz+c4unik1NeQF2shpNZw8cGyh6IFnpIUbBsXPgxiIbb1HfoQCRFocTgkqpsbH7n8WpJkEdPpVgtJh51xILlbZibsI9U1YsfDjpZWXxUUObsANbAK0Z0Ep0wkzv31+9fSD7FlGOJ//GjqFRjW4inekCeUNyRbXTGIlDMeZyRKs/JZVndKr9AFestwkiM7AWWNFUKfSX5RE0oRNwQDAKAa8HDPtErNhfoAtBCYEcBTB6GoXhnyX/1gwrDb8zeEtbc50PLBnJT6gJIp2M2IiAeXHgGuMLiDbNzgThQ="
-    on:
-      tags: true
-      rvm: 2.1.9
-      condition: '($SKIP_FORGE_PUBLISH != true) && ($FORGE_PUBLISH = true)'
-  - provider: releases
-    api_key:
-        secure: "c/QadLPTz3GBOL8ZPjHlxVMcicavnVVwnaFORMFf1qQYI3zCADLoqSGPaSOWSLXOFgOOKLuz2PTofqtKy/muDK0t8gl8dEFnyrUGRyRMfzvmH88DUihJ5H7Tuj1aqtUEYUgMUvA3552rf93/zyGPmsi2HKx65mg0pM1ubdz3TuDbPmxD3KdWu5DZRpF+WgLwB9yaAi1q44bXfS1QP8G5YcXzmSxXccEg1H+wwR96CnvgxQ/mL6S6H3w3bQOFciprw99J2eXXZXrbTsjlp9Sm3kb3p32mdThZYyTxTmtqVRpyqf/YEM9a9XYFNfAVl4VltiTCjTtTaQqn+4dS7UWuEi4BjEMr6erB2WLtnBISSoOXyhhgYynhtRG006a5xSc+R0iwcEqZbPQgG0rDEgkSXH7P9SxsUcwYvE5ceKphI3HDFf/9r0h0nOQ82hcbAE3hwshMbM+s+dGiv6xlaWo2v9Sr7SCUdGHkEYryggTh2cdtsgGnNPRIrQHTa1HQe303YjSR9MqqdFB2v5lAt8EfHl8iR+y2/do6udj5KsaHDMtxVKnwHXcSf1mNCVSaL4nffMt96h0QBLCGhNibTNjk6zOKoKTkf/+Xuv6WR9tmHHeAULsYzmyFiAoqGBBulBDeBZgsw/zSxkKeXSX9r+JP7p9abEjOd3EoruvvNGr2XS4="
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: '($SKIP_FORGE_PUBLISH != true) && ($FORGE_PUBLISH = true)'
+
+addons:
+  apt:
+    packages:
+      - rpm
+
+before_install:
+  - rm -f Gemfile.lock
+
+jobs:
+  allow_failures:
+    - env: STRICT_VARIABLES=yes TRUSTED_NODE_DATA=yes PUPPET_VERSION="~> 5.0"
+
+  include:
+    - stage: lint
+      rvm: 2.1.9
+      script:
+        - bundle exec rake metadata_lint
+
+    - stage: compare_tag
+      rvm: 2.1.9
+      script:
+        - bundle exec rake compare_latest_tag
+
+    - stage: spec
+      rvm: 2.1.9
+      env: STRICT_VARIABLES=yes TRUSTED_NODE_DATA=yes PUPPET_VERSION="~> 5.0"
+      script:
+        - bundle exec rake spec
+
+    - stage: spec
+      rvm: 2.1.9
+      env: STRICT_VARIABLES=yes TRUSTED_NODE_DATA=yes PUPPET_VERSION="~> 4.10.0"
+      script:
+        - bundle exec rake spec
+
+    - stage: spec
+      rvm: 2.1.9
+      env: STRICT_VARIABLES=yes TRUSTED_NODE_DATA=yes PUPPET_VERSION="~> 4.9.2"
+      script:
+        - bundle exec rake spec
+
+    - stage: spec
+      rvm: 2.1.9
+      env: STRICT_VARIABLES=yes TRUSTED_NODE_DATA=yes PUPPET_VERSION="~> 4.7.0"
+      script:
+        - bundle exec rake spec
+
+    - stage: acceptance
+      sudo: required
+      rvm: 2.1.9
+      services:
+        - docker
+      script:
+        - bundle exec rake beaker:suites[default,docker]
+      env: PUPPET_VERSION"~> 4.8.2"
+
+    # This needs to be last since we have an acceptance test
+    - stage: deploy
+      rvm: 2.1.9
+      env: STRICT_VARIABLES=yes TRUSTED_NODE_DATA=yes PUPPET_VERSION="~> 4.8.2"
+      script:
+        - bundle exec rake spec
+      before_deploy:
+        - "export PUPMOD_METADATA_VERSION=`ruby -r json -e \"puts JSON.parse(File.read('metadata.json')).fetch('version')\"`"
+        - '[[ $TRAVIS_TAG =~ ^simp-${PUPMOD_METADATA_VERSION}$|^${PUPMOD_METADATA_VERSION}$ ]]'
+      deploy:
+        - provider: releases
+          api_key:
+              secure: "c/QadLPTz3GBOL8ZPjHlxVMcicavnVVwnaFORMFf1qQYI3zCADLoqSGPaSOWSLXOFgOOKLuz2PTofqtKy/muDK0t8gl8dEFnyrUGRyRMfzvmH88DUihJ5H7Tuj1aqtUEYUgMUvA3552rf93/zyGPmsi2HKx65mg0pM1ubdz3TuDbPmxD3KdWu5DZRpF+WgLwB9yaAi1q44bXfS1QP8G5YcXzmSxXccEg1H+wwR96CnvgxQ/mL6S6H3w3bQOFciprw99J2eXXZXrbTsjlp9Sm3kb3p32mdThZYyTxTmtqVRpyqf/YEM9a9XYFNfAVl4VltiTCjTtTaQqn+4dS7UWuEi4BjEMr6erB2WLtnBISSoOXyhhgYynhtRG006a5xSc+R0iwcEqZbPQgG0rDEgkSXH7P9SxsUcwYvE5ceKphI3HDFf/9r0h0nOQ82hcbAE3hwshMbM+s+dGiv6xlaWo2v9Sr7SCUdGHkEYryggTh2cdtsgGnNPRIrQHTa1HQe303YjSR9MqqdFB2v5lAt8EfHl8iR+y2/do6udj5KsaHDMtxVKnwHXcSf1mNCVSaL4nffMt96h0QBLCGhNibTNjk6zOKoKTkf/+Xuv6WR9tmHHeAULsYzmyFiAoqGBBulBDeBZgsw/zSxkKeXSX9r+JP7p9abEjOd3EoruvvNGr2XS4="
+          skip_cleanup: true
+          on:
+            tags: true
+            condition: '($SKIP_FORGE_PUBLISH != true)'
+        - provider: puppetforge
+          user: simp
+          password:
+              secure: "vL1MNFrE7aazh/rEsSaHugCYYWXrJmsBuhMIP27CQ/EpBQ6wfWZhukZdkikZrmkIiJHHLi1HaQycKdW2AzcqN932iXOdjYtEA9eE/hO1VtGMYVobZFS4sh2Wtt8saVOg0tMPq45hjFHUe8FmgyrsjHBB+kl/fdfLVr+TiFCmWGXtZkjMlJxqPp+fyZZDrMoKoB7eSm3edOtqX7gONP3MEJ/wcHgCUTyspxI8sSXGl8IPwWNxU4LSgCwbvr/JzmDmKiK2AbSc7Q2+g7NrvZV39CbcV0IH4Uy4A+3fBBli/nPaidm/dvIvcYDacWF0UiXZTMvqd8tL4Z3Sf4vvB6NZ328ml97w233fLSHdLGn8qaIjmkK5x1UN9BRmA/WZbmkSSLWqwr6Cz+c4unik1NeQF2shpNZw8cGyh6IFnpIUbBsXPgxiIbb1HfoQCRFocTgkqpsbH7n8WpJkEdPpVgtJh51xILlbZibsI9U1YsfDjpZWXxUUObsANbAK0Z0Ep0wkzv31+9fSD7FlGOJ//GjqFRjW4inekCeUNyRbXTGIlDMeZyRKs/JZVndKr9AFestwkiM7AWWNFUKfSX5RE0oRNwQDAKAa8HDPtErNhfoAtBCYEcBTB6GoXhnyX/1gwrDb8zeEtbc50PLBnJT6gJIp2M2IiAeXHgGuMLiDbNzgThQ="
+          on:
+            tags: true
+            rvm: 2.1.9
+            condition: '($SKIP_FORGE_PUBLISH != true)'

CHANGELOG

@@ -1,3 +1,7 @@
+* Fri Sep 22 2017 Dylan Cochran <[email protected]> - 2.3.1-0
+- Refactor report generator to use a shared file format parser/compiler.
+- Add vendored 'profiles-in-modules' support
+
 * Tue Sep 19 2017 Liz Nemsick <[email protected]> - 2.3.1-0
 - Remove test link to allow module to be published to PuppetForge
 

Gemfile

@@ -39,5 +39,5 @@ end
 group :system_tests do
   gem 'beaker'
   gem 'beaker-rspec'
-  gem 'simp-beaker-helpers', ENV.fetch('SIMP_BEAKER_HELPERS_VERSION', '~> 1.7')
+  gem 'simp-beaker-helpers', ENV.fetch('SIMP_BEAKER_HELPERS_VERSION', ['>= 1.8.4', '< 2.0'])
 end

lib/puppet/parser/functions/compliance_map.rb

@@ -173,14 +173,38 @@ module Puppet::Parser::Functions
         }
     ENDHEREDOC
 
-#
-# Dynamic per-environment code loader.
-#
-    object = Object.new()
-    myself = __FILE__
-    filename = File.dirname(File.dirname(File.dirname(File.dirname(myself)))) + "/puppetx/simp/compliance_map.rb"
-    object.instance_eval(File.read(filename), filename)
-    object.compliance_map(args, self)
+        #
+        # Dynamic per-environment code loader.
+        #
+        # XXX ToDo
+        # This is persisted into the catalog ONLY to support compliance report
+        # custom entries.
+        #
+        # See the compliance_map.rb source code, but these may not be necessary.
+        # If that functionality is removed, return this logic to being instantiated each time.
+
+        catalog = find_global_scope.catalog
+        begin
+            compliance_report_generator = catalog._compliance_report_generator
+        rescue
+            catalog.instance_eval do
+                def _compliance_report_generator()
+                    @_compliance_report_generator
+                end
+                def _compliance_report_generator=(value)
+                    @_compliance_report_generator = value
+                end
+            end
+            object = Object.new()
+            myself = __FILE__
+            filename = File.dirname(File.dirname(File.dirname(File.dirname(myself)))) + "/puppetx/simp/compliance_map.rb"
+            object.instance_eval(File.read(filename), filename)
+            filename = File.dirname(File.dirname(File.dirname(File.dirname(myself)))) + "/puppetx/simp/compliance_mapper.rb"
+            object.instance_eval(File.read(filename), filename)
+            catalog._compliance_report_generator = object;
+            compliance_report_generator = object;
+        end
+    compliance_report_generator.compliance_map(args, self)
   end
 end
 

lib/puppetx/compliance/compliance_markup/v1/test.yaml

@@ -0,0 +1,8 @@
+---
+version: 1.0.0
+disa:
+        compliance_markup::test::vendoredvariable:
+                value: 'disa'
+nist:
+        compliance_markup::test::vendoredvariable:
+                value: 'nist'