NVMe nil error

.ruby-version

@@ -1 +1 @@
-2.3.1
+2.3.8

.travis.yml

@@ -1,9 +1,8 @@
 language: ruby
 rvm:
-- 2.3.1
+- 2.4.1
 before_install:
-- curl -L https://www.getchef.com/chef/install.sh | sudo bash -s -- -P chefdk -v 2.4.17
-- gem install bundler -v 1.11.2
+- curl -L https://www.getchef.com/chef/install.sh | sudo bash -s -- -P chefdk -v 3.1.0
 install:
 - chef exec bundle install --jobs=3 --retry=3
 before_script: chef exec rake setup_test_environment

Berksfile.lock

@@ -6,9 +6,9 @@ DEPENDENCIES
     path: test/cookbooks/fake
 
 GRAPH
-  ephemeral_lvm (3.0.2)
+  ephemeral_lvm (3.0.3)
     lvm (>= 4.0.6)
     now (>= 0.0.0)
   fake (0.1.1)
-  lvm (4.1.13)
+  lvm (4.1.15)
   now (1.0.0)

CHANGELOG.md

@@ -3,6 +3,14 @@ ephemeral_lvm Cookbook CHANGELOG
 
 This file is used to list changes made in each version of the ephemeral_lvm cookbook.
 
+v3.0.3
+------
+- Fix NVMe devices detection on EC2 instances powered by Nitro hypervisor
+
+v3.0.2
+------
+- Support for explicitly declaring devices using additonal_devices attribute
+
 v3.0.1
 ------
 - Fixed detection for NVMe ephemeral devices in EC2

Gemfile.lock

@@ -1,14 +1,19 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.5.2)
-      public_suffix (>= 2.0.2, < 4.0)
-    backports (3.11.0)
-    builder (3.2.3)
-    chef (12.21.31)
+    activesupport (5.2.6)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    backports (3.21.0)
+    builder (3.2.4)
+    chef (12.22.5)
       addressable
       bundler (>= 1.10)
-      chef-config (= 12.21.31)
+      chef-config (= 12.22.5)
       chef-zero (>= 4.8, < 13)
       diff-lcs (~> 1.2, >= 1.2.4)
       erubis (~> 2.7)
@@ -34,7 +39,7 @@ GEM
       specinfra (~> 2.10)
       syslog-logger (~> 1.6)
       uuidtools (~> 2.1.5)
-    chef-config (12.21.31)
+    chef-config (12.22.5)
       addressable
       fuzzyurl
       mixlib-config (~> 2.0)
@@ -45,48 +50,52 @@ GEM
       mixlib-log (~> 1.3)
       rack (~> 2.0)
       uuidtools (~> 2.1)
-    connection_pool (2.2.1)
-    diff-lcs (1.3)
+    concurrent-ruby (1.1.9)
+    diff-lcs (1.4.4)
     erubis (2.7.0)
-    ethon (0.11.0)
-      ffi (>= 1.3.0)
-    faraday (0.13.1)
+    ethon (0.14.0)
+      ffi (>= 1.15.0)
+    faraday (0.17.4)
       multipart-post (>= 1.2, < 3)
-    faraday_middleware (0.12.2)
+    faraday_middleware (0.14.0)
       faraday (>= 0.7.4, < 1.0)
-    ffi (1.9.18)
-    ffi-yajl (2.3.1)
-      libyajl2 (~> 1.2)
+    ffi (1.15.4)
+    ffi-yajl (2.4.0)
+      libyajl2 (>= 1.2)
     fuzzyurl (0.9.0)
-    gh (0.14.0)
-      addressable
-      backports
+    gh (0.16.0)
+      activesupport (~> 5.0)
+      addressable (~> 2.4)
       faraday (~> 0.8)
+      faraday_middleware (~> 0.14)
       multi_json (~> 1.0)
-      net-http-persistent (>= 2.7)
+      net-http-persistent (~> 2.9)
       net-http-pipeline
-    hashie (3.5.7)
+    hashie (3.6.0)
     highline (1.7.10)
-    iniparse (1.4.4)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    iniparse (1.5.0)
     ipaddress (0.8.3)
     json (2.5.1)
-    launchy (2.4.3)
-      addressable (~> 2.3)
-    libyajl2 (1.2.0)
-    mixlib-archive (0.4.1)
+    launchy (2.5.0)
+      addressable (~> 2.7)
+    libyajl2 (2.1.0)
+    minitest (5.14.4)
+    mixlib-archive (0.4.20)
       mixlib-log
     mixlib-authentication (1.4.2)
     mixlib-cli (1.7.0)
-    mixlib-config (2.2.4)
+    mixlib-config (2.2.18)
+      tomlrb
     mixlib-log (1.7.1)
-    mixlib-shellout (2.3.2)
-    multi_json (1.12.2)
-    multipart-post (2.0.0)
-    net-http-persistent (3.0.0)
-      connection_pool (~> 2.2)
+    mixlib-shellout (2.4.4)
+    multi_json (1.15.0)
+    multipart-post (2.1.1)
+    net-http-persistent (2.9.4)
     net-http-pipeline (1.0.1)
-    net-scp (1.2.1)
-      net-ssh (>= 2.6.5)
+    net-scp (3.0.0)
+      net-ssh (>= 2.6.5, < 7.0.0)
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
@@ -96,7 +105,7 @@ GEM
       net-ssh (>= 2.6.5)
       net-ssh-gateway (>= 1.2.0)
     net-telnet (0.1.1)
-    ohai (8.25.1)
+    ohai (8.26.1)
       chef-config (>= 12.5.0.alpha.1, < 14)
       ffi (~> 1.9)
       ffi-yajl (~> 2.2)
@@ -108,47 +117,49 @@ GEM
       plist (~> 3.1)
       systemu (~> 2.6.4)
       wmi-lite (~> 1.0)
-    plist (3.4.0)
+    plist (3.6.0)
     proxifier (1.0.3)
-    public_suffix (3.0.1)
+    public_suffix (4.0.6)
     pusher-client (0.6.2)
       json
       websocket (~> 1.0)
-    rack (2.0.3)
-    rake (12.3.0)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rack (2.2.3)
+    rake (13.0.6)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-its (1.2.0)
+      rspec-support (~> 3.10.0)
+    rspec-its (1.3.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
-    rspec-mocks (3.7.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
     rspec_junit_formatter (0.2.3)
       builder (< 4)
       rspec-core (>= 2, < 4, != 2.12.0)
-    serverspec (2.41.3)
+    serverspec (2.41.8)
       multi_json
       rspec (~> 3.0)
       rspec-its
       specinfra (~> 2.72)
     sfl (2.3)
-    specinfra (2.73.0)
+    specinfra (2.82.25)
       net-scp
-      net-ssh (>= 2.7, < 5.0)
-      net-telnet
+      net-ssh (>= 2.7)
+      net-telnet (= 0.1.1)
       sfl
     syslog-logger (1.6.8)
     systemu (2.6.5)
-    travis (1.8.8)
+    thread_safe (0.3.6)
+    tomlrb (2.0.1)
+    travis (1.8.13)
       backports
       faraday (~> 0.9)
       faraday_middleware (~> 0.9, >= 0.9.1)
@@ -159,9 +170,11 @@ GEM
       typhoeus (~> 0.6, >= 0.6.8)
     typhoeus (0.8.0)
       ethon (>= 0.8.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
     uuidtools (2.1.5)
-    websocket (1.2.5)
-    wmi-lite (1.0.0)
+    websocket (1.2.9)
+    wmi-lite (1.0.5)
 
 PLATFORMS
   ruby
@@ -172,4 +185,4 @@ DEPENDENCIES
   travis
 
 BUNDLED WITH
-   1.15.1
+   2.2.16

README.md

@@ -29,6 +29,15 @@ Github Repository: [https://github.com/rightscale-cookbooks/ephemeral_lvm](https
 
 Place the `ephemeral_lvm::default` in the runlist and the ephemeral devices will be setup.
 
+## Notes on detection of ephemeral devices on newer AWS EC2 instance types
+With the following instances, EBS volumes are exposed as NVMe block devices: `c5`, `c5d`, `i3.metal`, `m5`, and `m5d`. The device names are `/dev/nvme0n1`, `/dev/nvme1n1`, and so on. The device names that you specify in a block device mapping are renamed using NVMe device names (`/dev/nvme[0-26]n1`). [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html)
+
+This also affects ephemeral SSD devices that are always attached and for which the mapping isn't present in metadata. The current cookbook version tries to find all NVMe devices that aren't mounted.
+
+Note 1) Instance type `i3` is a special case which allows you to map the ephemeral SSDs although there's a naming mismatch `/dev/sdb` vs. `/dev/nvme0n1`. In this case you shouldn't "map" the ephemeral volumes when starting the instance as they are always attached and will only be detected if mapping is not present.
+
+Note 2) To keep things simple if you map additional EBS volumes to the instance types mentioned above the cookbook won't make a distinction between ephemeral devices and EBS volumes and will include all in the logical volume.
+
 # Attributes
 
 * `node['ephemeral_lvm']['filesystem']` - the filesystem to be used on the ephemeral volume. Default: `'ext4'`

libraries/helper.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 #
 # Cookbook Name:: ephemeral_lvm
 # Library:: helper
@@ -30,23 +31,51 @@ def self.gce_ephemeral_devices?(cloud, node)
       # /dev/disk/by-id/google-ephemeral-disk-*. Refer to
       # https://developers.google.com/compute/docs/disks#scratchdisks for more information.
       #
-      ephemeral_devices = node[cloud]['attached_disks']['disks'].map do |disk|
-        if ((disk['type'] == 'EPHEMERAL') || (disk['type'] == 'LOCAL-SSD')) && disk['deviceName'].match(/^local-ssd-\d+$/)
-          "/dev/disk/by-id/google-#{disk['deviceName']}"
+      unless node[cloud]['attached_disks'].nil?
+        ephemeral_devices = node[cloud]['attached_disks']['disks'].map do |disk|
+          if ((disk['type'] == 'EPHEMERAL') || (disk['type'] == 'LOCAL-SSD')) && disk['deviceName'].match(/^local-ssd-\d+$/)
+            "/dev/disk/by-id/google-#{disk['deviceName']}"
+          end
         end
-      end unless node[cloud]['attached_disks'].nil?
+      end
 
-      ephemeral_devices = node[cloud]['instance']['disks'].map do |disk|
-        if disk['type'] == 'LOCAL-SSD' && disk['deviceName'].match(/^local-ssd-\d+$/)
-          "/dev/disk/by-id/google-#{disk['deviceName']}"
+      unless node[cloud]['instance'].nil?
+        ephemeral_devices = node[cloud]['instance']['disks'].map do |disk|
+          if disk['type'] == 'LOCAL-SSD' && disk['deviceName'].match(/^local-ssd-\d+$/)
+            "/dev/disk/by-id/google-#{disk['deviceName']}"
+          end
         end
-      end unless node[cloud]['instance'].nil?
+      end
 
       # Removes nil elements from the ephemeral_devices array if any.
       ephemeral_devices.compact!
       ephemeral_devices
     end
 
+    # @param cloud [String] the name of cloud
+    # @param node [Chef::Node] the Chef node
+    def self.ec2_ephemeral_devices?(_cloud, node)
+      # Find all NVMe devices that are present on newer instance types but aren't listed in metadata
+      unless node['filesystem'].nil? || node['filesystem']['by_device'].nil?
+        nvme_devices = node['filesystem']['by_device'].keys.select { |device| device =~ %r{\/dev\/nvme\d+n\d+$} }
+        Chef::Log.info "Available NVMe devices: #{nvme_devices}"
+        # Find any NVMe devices with mounted partitions - typically root volume on 5th generation of instances
+        nvme_devices_mounted = node['filesystem']['by_pair'].keys.map do |pair|
+          # Split device,mountpoint string into an array
+          pair_array = pair.split(',')
+          # Check if device is NVMe device and has a valid mountpoint
+          if pair_array[0] =~ %r{\/dev\/nvme\d+n\d+} && pair_array.size > 1
+            # Return main device for a mounted partition
+            pair_array[0][%r{\/dev\/nvme\d+n\d+}]
+          end
+        end.compact
+        Chef::Log.info "Mounted NVMe devices: #{nvme_devices_mounted}"
+        ephemeral_devices = nvme_devices - nvme_devices_mounted
+        Chef::Log.info "Usable devices: #{ephemeral_devices}"
+        ephemeral_devices
+      end
+    end
+
     # Identifies the ephemeral devices available on a cloud server based on cloud-specific Ohai data and returns
     # them as an array. This method also does the mapping required for Xen hypervisors (/dev/sdX -> /dev/xvdX).
     #
@@ -56,7 +85,6 @@ def self.gce_ephemeral_devices?(cloud, node)
     #
     def self.get_ephemeral_devices(cloud, node)
       ephemeral_devices = []
-      ephemeral_devices.concat node['ephemeral_lvm']['additonal_devices']
       # Detects the ephemeral disks available on the instance.
       #
       # If the cloud plugin supports block device mapping on the node, obtain the
@@ -72,11 +100,8 @@ def self.get_ephemeral_devices(cloud, node)
         # Removes nil elements from the ephemeral_devices array if any.
         ephemeral_devices.compact!
 
-        # Add all NVMe devices
-        ephemeral_devices.concat Dir.glob('/dev/nvme*n*')
-
         # Servers running on Xen hypervisor require the block device to be in /dev/xvdX instead of /dev/sdX
-        if node.attribute?('virtualization') && node['virtualization']['system'] == 'xen'
+        if !node['virtualization'].nil? && node['virtualization']['system'] == 'xen'
           Chef::Log.info "Mapping for devices: #{ephemeral_devices.inspect}"
           ephemeral_devices = EphemeralLvm::Helper.fix_device_mapping(
             ephemeral_devices,
@@ -90,11 +115,18 @@ def self.get_ephemeral_devices(cloud, node)
         case cloud
         when 'gce'
           ephemeral_devices = gce_ephemeral_devices?(cloud, node)
+        when 'ec2'
+          ephemeral_devices = ec2_ephemeral_devices?(cloud, node)
         else
           Chef::Log.info 'No ephemeral disks found.'
         end
       end
-      ephemeral_devices
+      if !ephemeral_devices.nil?
+        ephemeral_devices.concat(node['ephemeral_lvm']['additonal_devices']).uniq
+      else
+        ephemeral_devices = []
+        ephemeral_devices.concat(node['ephemeral_lvm']['additonal_devices']).uniq
+      end
     end
 
     # Fixes the device mapping on Xen hypervisors. When using Xen hypervisors, the devices are mapped from /dev/sdX to

metadata.rb

@@ -5,7 +5,7 @@
 license          'Apache-2.0'
 description      'Configures available ephemeral devices on a cloud server'
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version          '3.0.2'
+version          '3.0.3'
 issues_url       'https://github.com/rightscale-cookbooks/ephemeral_lvm/issues' if respond_to?(:issues_url)
 source_url       'https://github.com/rightscale-cookbooks/ephemeral_lvm' if respond_to?(:source_url)
 chef_version '>= 12.0' if respond_to?(:chef_version)

recipes/default.rb

@@ -22,7 +22,7 @@
 #
 include_recipe_now 'lvm'
 
-if !node.attribute?('cloud') || !node['cloud'].attribute?('provider') || !node.attribute?(node['cloud']['provider'])
+if node['cloud'].nil? || node['cloud']['provider'].nil?
   log 'Not running on a known cloud, not setting up ephemeral LVM'
 else
   # Obtain the current cloud