Jewel support

.kitchen.yml

@@ -1,7 +1,8 @@
 ---
-driver_plugin: vagrant
-driver_config:
-  vagrantfile_erb: test/integration/Vagrantfile.erb
+driver:
+  name: vagrant
+  vagrantfiles:
+    - test/integration/Vagrantfile.rb
   require_chef_omnibus: true
 
 platforms:
@@ -34,9 +35,9 @@ suites:
       config:
         fsid: ae3f1d03-bacd-4a90-b869-1a4fabb107f2
         mon_initial_members:
-          - "127.0.0.1"
 - name: osd
   run_list:
+    - "role[ceph-mon]"
     - "role[ceph-osd]"
   attributes: *defaults
 - name: mon
@@ -45,10 +46,12 @@ suites:
   attributes: *defaults
 - name: mds
   run_list:
+    - "role[ceph-mon]"
     - "role[ceph-mds]"
   attributes: *defaults
 - name: radosgw
   run_list:
+    - "role[ceph-mon]"
     - "role[ceph-radosgw]"
   attributes: *defaults
 - name: aio

Berksfile

@@ -1,4 +1,4 @@
-site :opscode
+source 'https://supermarket.chef.io'
 
 metadata
 

CHANGELOG.md

@@ -1,6 +1,14 @@
 ceph
 ====
 
+v0.9.0 (2016-07-12)
+-------------------
+
+- Multiple fixes to tests, Berksfile etc.
+- Default ceph version to install is 'jewel'.
+- Run daemons as ceph unprivileged user (from Infernalis onwards).
+- Removed support for the 'ceph_extras' repository, it does not exist anymore.
+
 v0.2.0 (2014-03-03)
 -------------------
 

README.md

@@ -59,7 +59,7 @@ The other set of attributes that this recipe needs is `node['ceph']['osd_devices
 
 * {'device' => '/dev/sdb'} - Use a full disk for the OSD, with a small partition for the journal
 * {'type' => 'directory', 'device' => '/src/node/sdb1/ceph'} - Use a directory, and have a small file for the journal
-* {'device' => '/dev/sde', 'dmcrypt' => true} - Store the data encrypted by passing --dmcrypt to `ceph-disk-prepare`
+* {'device' => '/dev/sde', 'dmcrypt' => true} - Store the data encrypted by passing --dmcrypt to `ceph-disk prepare`
 * {'device' => '/dev/sdc', 'journal' => '/dev/sdd2'} - use a full disk for the OSD with a custom partition for the journal
 
 ### Using a Policy Wrapper Cookbook
@@ -109,7 +109,6 @@ Ceph Rados Gateway nodes should use the ceph-radosgw role
 * `node['ceph']['search_environment']` - a custom Chef environment to search when looking for mon nodes. The cookbook defaults to searching the current environment
 * `node['ceph']['branch']` - selects whether to install the stable, testing, or dev version of Ceph
 * `node['ceph']['version']` - install a version of Ceph that is different than the cookbook default. If this is changed in a wrapper cookbook, some repository urls may also need to be replaced, and they are found in attributes/repo.rb. If the branch attribute is set to dev, this selects the gitbuilder branch to install
-* `node['ceph']['extras_repo']` - whether to install the ceph extras repo. The tgt recipe requires this
 
 * `node['ceph']['config']['fsid']` - the cluster UUID
 * `node['ceph']['config']['global']['public network']` - a CIDR specification of the public network

attributes/default.rb

@@ -1,10 +1,22 @@
+# Major release version to install or gitbuilder branch
+default['ceph']['version'] = 'jewel'
+
 default['ceph']['install_debug'] = false
 default['ceph']['encrypted_data_bags'] = false
 
 default['ceph']['install_repo'] = true
 
 default['ceph']['user_pools'] = []
 
+# starting with Infernalis release, ceph runs as non-root by default
+if node['ceph']['version'] >= 'infernalis'
+  default['ceph']['user'] = 'ceph'
+  default['ceph']['group'] = 'ceph'
+else
+  default['ceph']['user'] = 'root'
+  default['ceph']['group'] = 'root'
+end
+
 case node['platform']
 when 'ubuntu'
   default['ceph']['init_style'] = 'upstart'

attributes/mds.rb

@@ -1,6 +1,7 @@
 include_attribute 'ceph'
 
 default['ceph']['mds']['init_style'] = node['init_style']
+default['ceph']['mds']['fs']['ceph'] = { 'data_pool' => 'data', 'metadata_pool' => 'metadata' }
 
 case node['platform_family']
 when 'debian'

attributes/repo.rb

@@ -1,42 +1,32 @@
 default['ceph']['branch'] = 'stable' # Can be stable, testing or dev.
-# Major release version to install or gitbuilder branch
-default['ceph']['version'] = 'firefly'
 default['ceph']['el_add_epel'] = true
-default['ceph']['repo_url'] = 'http://ceph.com'
-default['ceph']['extras_repo_url'] = 'http://ceph.com/packages/ceph-extras'
-default['ceph']['extras_repo'] = false
+default['ceph']['repo_url'] = 'https://download.ceph.com'
 
 case node['platform_family']
 when 'debian'
   # Debian/Ubuntu default repositories
   default['ceph']['debian']['stable']['repository'] = "#{node['ceph']['repo_url']}/debian-#{node['ceph']['version']}/"
-  default['ceph']['debian']['stable']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
+  default['ceph']['debian']['stable']['repository_key'] = 'https://raw.githubusercontent.com/ceph/ceph/master/keys/release.asc'
   default['ceph']['debian']['testing']['repository'] = "#{node['ceph']['repo_url']}/debian-testing/"
-  default['ceph']['debian']['testing']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
+  default['ceph']['debian']['testing']['repository_key'] = 'https://raw.githubusercontent.com/ceph/ceph/master/keys/release.asc'
   default['ceph']['debian']['dev']['repository'] = "http://gitbuilder.ceph.com/ceph-deb-#{node['lsb']['codename']}-x86_64-basic/ref/#{node['ceph']['version']}"
   default['ceph']['debian']['dev']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc'
-  default['ceph']['debian']['extras']['repository'] = "#{node['ceph']['extras_repo_url']}/debian/"
-  default['ceph']['debian']['extras']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
 when 'rhel'
   # Redhat/CentOS default repositories
   default['ceph']['rhel']['stable']['repository'] = "#{node['ceph']['repo_url']}/rpm-#{node['ceph']['version']}/el6/x86_64/"
-  default['ceph']['rhel']['stable']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
+  default['ceph']['rhel']['stable']['repository_key'] = 'https://raw.githubusercontent.com/ceph/ceph/master/keys/release.asc'
   default['ceph']['rhel']['testing']['repository'] = "#{node['ceph']['repo_url']}/rpm-testing/el6/x86_64/"
-  default['ceph']['rhel']['testing']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
+  default['ceph']['rhel']['testing']['repository_key'] = 'https://raw.githubusercontent.com/ceph/ceph/master/keys/release.asc'
   default['ceph']['rhel']['dev']['repository'] = "http://gitbuilder.ceph.com/ceph-rpm-centos6-x86_64-basic/ref/#{node['ceph']['version']}/x86_64/"
   default['ceph']['rhel']['dev']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc'
-  default['ceph']['rhel']['extras']['repository'] = "#{node['ceph']['extras_repo_url']}/rpm/rhel6/x86_64/"
-  default['ceph']['rhel']['extras']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
 when 'fedora'
   # Fedora default repositories
   default['ceph']['fedora']['stable']['repository'] = "#{node['ceph']['repo_url']}/rpm-#{node['ceph']['version']}/fc#{node['platform_version']}/x86_64/"
-  default['ceph']['fedora']['stable']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
+  default['ceph']['fedora']['stable']['repository_key'] = 'https://raw.githubusercontent.com/ceph/ceph/master/keys/release.asc'
   default['ceph']['fedora']['testing']['repository'] = "#{node['ceph']['repo_url']}/rpm-testing/fc#{node['platform_version']}/x86_64/"
-  default['ceph']['fedora']['testing']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
+  default['ceph']['fedora']['testing']['repository_key'] = 'https://raw.githubusercontent.com/ceph/ceph/master/keys/release.asc'
   default['ceph']['fedora']['dev']['repository'] = "http://gitbuilder.ceph.com/ceph-rpm-fc#{node['platform_version']}-x86_64-basic/ref/#{node['ceph']['version']}/RPMS/x86_64/"
   default['ceph']['fedora']['dev']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc'
-  default['ceph']['fedora']['extras']['repository'] = "#{node['ceph']['extras_repo_url']}/rpm/fedora#{node['platform_version']}/x86_64/"
-  default['ceph']['fedora']['extras']['repository_key'] = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'
 when 'suse'
   # (Open)SuSE default repositories
   # Chef doesn't make a difference between suse and opensuse
@@ -46,7 +36,6 @@
 
   default['ceph']['suse']['stable']['repository'] = "#{node['ceph']['repo_url']}/rpm-#{node['ceph']['version']}/#{suse_version}/x86_64/ceph-release-1-0.#{suse_version}.noarch.rpm"
   default['ceph']['suse']['testing']['repository'] = "#{node['ceph']['repo_url']}/rpm-testing/#{suse_version}/x86_64/ceph-release-1-0.#{suse_version}.noarch.rpm"
-  default['ceph']['suse']['extras']['repository'] = "#{node['ceph']['extras_repo_url']}/rpm/#{suse_version}/x86_64/"
 else
   fail "#{node['platform_family']} is not supported"
 end

infrastructure.yml

@@ -3,4 +3,3 @@ roles:
 - ceph-mon:
 - ceph-osd:
 - ceph-radosgw:
-- ceph-tgt:

metadata.rb

@@ -4,7 +4,7 @@
 license 'Apache 2.0'
 description 'Installs/Configures the Ceph distributed filesystem'
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version '0.8.0'
+version '0.9.0'
 
 depends	'apache2', '>= 1.1.12'
 depends 'apt'

providers/cephfs.rb

@@ -29,6 +29,7 @@ def manage_mount(directory, subdir, use_fuse, action)
       dump 0
       pass 0
       action action
+      not_if "mount | grep \"^ceph-fuse on #{Regexp.escape(directory)}\"" if action == :mount
     end
   else
     mons = mon_addresses.sort.join(',') + ':' + subdir

recipes/apt.rb

@@ -22,12 +22,3 @@
   components ['main']
   key node['ceph']['debian'][branch]['repository_key']
 end
-
-apt_repository 'ceph-extras' do
-  repo_name 'ceph-extras'
-  uri node['ceph']['debian']['extras']['repository']
-  distribution distribution_codename
-  components ['main']
-  key node['ceph']['debian']['extras']['repository_key']
-  only_if { node['ceph']['extras_repo'] }
-end

recipes/conf.rb

@@ -9,7 +9,7 @@
 
 directory '/etc/ceph' do
   owner 'root'
-  group 'root'
+  group node['ceph']['group']
   mode '0755'
   action :create
 end
@@ -22,5 +22,7 @@
       :is_rgw => node['ceph']['is_radosgw']
     }
   }
+  owner 'root'
+  group node['ceph']['group']
   mode '0644'
 end

recipes/default.rb

@@ -1,9 +1,10 @@
 include_recipe 'ceph::repo' if node['ceph']['install_repo']
-include_recipe 'ceph::conf'
 
-# Tools needed by cookbook
+# Tools needed by cookbook (including ceph-common, which creates the ceph user).
 node['ceph']['packages'].each do |pck|
   package pck
 end
 
+include_recipe 'ceph::conf'
+
 chef_gem 'netaddr'

recipes/mds.rb

@@ -22,9 +22,33 @@
 
 cluster = 'ceph'
 
+if node['ceph']['version'] >= 'giant'
+  # doesn't work on non-mon nodes, will need to be moved to lwrp
+  node['ceph']['mds']['fs'].keys.each do |fs|
+    metadata_pool = node['ceph']['mds']['fs'][fs]['metadata_pool']
+    data_pool = node['ceph']['mds']['fs'][fs]['data_pool']
+
+    [metadata_pool, data_pool].each do |pool_name|
+      execute "ensure ceph pool #{pool_name} exists" do
+        command "ceph osd pool create #{pool_name} 32"
+        user node['ceph']['user']
+        group node['ceph']['group']
+        not_if "rados lspools | grep '^#{Regexp.quote(pool_name)}$'"
+      end
+    end
+
+    execute "ensure cephfs #{fs} exists" do
+      command "ceph fs new #{fs} #{metadata_pool} #{data_pool}"
+      user node['ceph']['user']
+      group node['ceph']['group']
+      not_if "ceph fs ls | grep '^#{Regexp.quote(fs)}$'"
+    end
+  end
+end
+
 directory "/var/lib/ceph/mds/#{cluster}-#{node['hostname']}" do
-  owner 'root'
-  group 'root'
+  owner node['ceph']['user']
+  group node['ceph']['group']
   mode 00755
   recursive true
   action :create
@@ -34,6 +58,8 @@
   caps('osd' => 'allow *', 'mon' => 'allow rwx')
   keyname "mds.#{node['hostname']}"
   filename "/var/lib/ceph/mds/#{cluster}-#{node['hostname']}/keyring"
+  owner node['ceph']['user']
+  group node['ceph']['group']
 end
 
 file "/var/lib/ceph/mds/#{cluster}-#{node['hostname']}/done" do

recipes/mon.rb

@@ -22,16 +22,16 @@
 service_type = node['ceph']['mon']['init_style']
 
 directory '/var/run/ceph' do
-  owner 'root'
-  group 'root'
+  owner node['ceph']['user']
+  group node['ceph']['group']
   mode 00755
   recursive true
   action :create
 end
 
 directory "/var/lib/ceph/mon/ceph-#{node['hostname']}" do
-  owner 'root'
-  group 'root'
+  owner node['ceph']['user']
+  group node['ceph']['group']
   mode 00755
   recursive true
   action :create
@@ -40,17 +40,21 @@
 # TODO: cluster name
 cluster = 'ceph'
 
-keyring = "#{Chef::Config[:file_cache_path]}/#{cluster}-#{node['hostname']}.mon.keyring"
+keyring = "/var/run/ceph/#{cluster}-#{node['hostname']}.mon.keyring"
 
 execute 'format mon-secret as keyring' do # ~FC009
   command lazy { "ceph-authtool '#{keyring}' --create-keyring --name=mon. --add-key='#{mon_secret}' --cap mon 'allow *'" }
+  user node['ceph']['user']
+  group node['ceph']['group']
   creates keyring
   only_if { mon_secret }
   sensitive true if Chef::Resource::Execute.method_defined? :sensitive
 end
 
 execute 'generate mon-secret as keyring' do # ~FC009
   command "ceph-authtool '#{keyring}' --create-keyring --name=mon. --gen-key --cap mon 'allow *'"
+  user node['ceph']['user']
+  group node['ceph']['group']
   creates keyring
   not_if { mon_secret }
   notifies :create, 'ruby_block[save mon_secret]', :immediately
@@ -59,12 +63,14 @@
 
 execute 'add bootstrap-osd key to keyring' do
   command lazy { "ceph-authtool '#{keyring}' --name=client.bootstrap-osd --add-key='#{osd_secret}' --cap mon 'allow profile bootstrap-osd'  --cap osd 'allow profile bootstrap-osd'" }
+  user node['ceph']['user']
+  group node['ceph']['group']
   only_if { node['ceph']['encrypted_data_bags'] && osd_secret }
 end
 
 ruby_block 'save mon_secret' do
   block do
-    fetch = Mixlib::ShellOut.new("ceph-authtool '#{keyring}' --print-key --name=mon.")
+    fetch = Mixlib::ShellOut.new("ceph-authtool '#{keyring}' --print-key --name=mon.", user: node['ceph']['user'], group: node['ceph']['group'])
     fetch.run_command
     key = fetch.stdout
     node.set['ceph']['monitor-secret'] = key
@@ -75,6 +81,8 @@
 
 execute 'ceph-mon mkfs' do
   command "ceph-mon --mkfs -i #{node['hostname']} --keyring '#{keyring}'"
+  user node['ceph']['user']
+  group node['ceph']['group']
 end
 
 ruby_block 'finalise' do
@@ -112,6 +120,8 @@
 mon_addresses.each do |addr|
   execute "peer #{addr}" do
     command "ceph --admin-daemon '/var/run/ceph/ceph-mon.#{node['hostname']}.asok' add_bootstrap_peer_hint #{addr}"
+    user node['ceph']['user']
+    group node['ceph']['group']
     ignore_failure true
   end
 end
@@ -123,7 +133,7 @@
     block do
       run_out = ''
       while run_out.empty?
-        run_out = Mixlib::ShellOut.new('ceph auth get-key client.bootstrap-osd').run_command.stdout.strip
+        run_out = Mixlib::ShellOut.new('ceph auth get-key client.bootstrap-osd', user: node['ceph']['user'], group: node['ceph']['group']).run_command.stdout.strip
         sleep 2
       end
       node.set['ceph']['bootstrap_osd_key'] = run_out