diff --git a/attributes/mds.rb b/attributes/mds.rb new file mode 100644 index 0000000..be0c008 --- /dev/null +++ b/attributes/mds.rb @@ -0,0 +1,6 @@ +case node['platform'] +when 'ubuntu' + default["ceph"]["mds"]["init_style"] = "upstart" +else + default["ceph"]["mds"]["init_style"] = "sysvinit" +end diff --git a/attributes/pools.rb b/attributes/pools.rb new file mode 100644 index 0000000..3042938 --- /dev/null +++ b/attributes/pools.rb @@ -0,0 +1,25 @@ +# +# Override the following default to have pools setup automatically by the +# ceph::pools recipe. +# + +default["ceph"]["pools"] = [] + +# +# Below is a sample of how the override can be done. +# Uncomment it to try it out. +# + +#default["ceph"]["pools"] = [ +# { +# "name" => "test1" +# }, +# { "name" => "test2", +# "pg-num" => 10 +# }, +# { +# "name" => "test3", +# "pg-num" => 20, +# "pgp-num" => 15 +# } +#] diff --git a/providers/manage_pool.rb b/providers/manage_pool.rb new file mode 100644 index 0000000..3338206 --- /dev/null +++ b/providers/manage_pool.rb @@ -0,0 +1,48 @@ +# +# Author:: Jesse Pretorius +# Cookbook Name:: ceph +# Provider:: manage_pool +# +# Copyright 2013, Business Connexion (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +action :create do + name = new_resource.name + pg_num = new_resource.pg_num + pgp_num = new_resource.pgp_num + min_size = new_resource.min_size + + if pgp_num.nil? + Log.debug("Setting ceph-pool #{name} pgp_num to #{pg_num} as no value was provided.") + pgp_num = new_resource.pg_num + elsif pgp_num > pg_num + Log.warn("Setting ceph-pool #{name} pgp_num to #{pg_num} as it cannot be a larger value.") + pgp_num = new_resource.pg_num + end + + execute "create ceph pool #{name} with pg_num #{pg_num} and pgp_num #{pgp_num}" do + command "ceph osd pool create #{name} #{pg_num} #{pgp_num}" + end + + if min_size.nil? + Log.debug("Leaving ceph-pool #{name} min_size at the default value.") + else + execute "set ceph pool #{name} to min_size #{min_size}" do + command "ceph osd pool set #{name} min_size #{min_size}" + end + end + + new_resource.updated_by_last_action(true) +end diff --git a/recipes/conf.rb b/recipes/conf.rb index 6480611..3df1b14 100644 --- a/recipes/conf.rb +++ b/recipes/conf.rb @@ -1,5 +1,20 @@ -raise "fsid must be set in config" if node["ceph"]["config"]['fsid'].nil? -raise "mon_initial_members must be set in config" if node["ceph"]["config"]['mon_initial_members'].nil? +if node['ceph']['config']['fsid'].nil? || node['ceph']['config']['mon_initial_members'].nil? + Log.debug("ceph-mds: Trying to retrieve fsid and mon from the ceph-setup role") + search_query = "roles:ceph-setup AND chef_environment:#{node.chef_environment}" + search_result = search(:node, search_query) + if search_result.length < 1 + msg = "ceph-conf: The ceph fsid and mon_initial_members must be set in the config, or the ceph-setup role must be applied to a node." + Chef::Application.fatal! msg + end + + fsid = search_result[0]['ceph']['config']['fsid'] + Log.debug("ceph-mds: Found ceph fsid #{fsid} from the ceph-setup role") + node.set['ceph']['config']['fsid'] = fsid + + mon_initial_members = search_result[0]['ceph']['config']['mon_initial_members'] + Log.debug("ceph-mds: Found ceph mon_initial_members #{mon_initial_members} from the ceph-setup role") + node.set['ceph']['config']['mon_initial_members'] = mon_initial_members +end mon_addresses = get_mon_addresses() diff --git a/recipes/mds.rb b/recipes/mds.rb index 76816ba..2001c30 100644 --- a/recipes/mds.rb +++ b/recipes/mds.rb @@ -1,9 +1,11 @@ # # Author:: Kyle Bader +# Author:: Jesse Pretorius # Cookbook Name:: ceph # Recipe:: mds # # Copyright 2011, DreamHost Web Hosting +# Copyright 2013, Business Connexion (Pty) Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,5 +18,81 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - + include_recipe "ceph::default" +include_recipe "ceph::conf" + +package 'ceph-mds' do + action :install +end + +service_type = node["ceph"]["mds"]["init_style"] + +mons = get_mon_nodes("bootstrap_mds_key:*") + +if mons.empty? then + Log.warn("ceph-mds: No ceph mds bootstrap key found.") +else + mds_bootstrap_directory = "/var/lib/ceph/bootstrap-mds" + + directory "#{mds_bootstrap_directory}" do + owner "root" + group "root" + mode "0755" + end + + # TODO cluster name + cluster = 'ceph' + + execute "create the local keyring file" do + command "ceph-authtool '#{mds_bootstrap_directory}/#{cluster}.keyring' --create-keyring --name=client.bootstrap-mds --add-key='#{mons[0]["ceph"]["bootstrap_mds_key"]}'" + creates "#{mds_bootstrap_directory}/#{cluster}.keyring" + end + + mds_directory = "/var/lib/ceph/mds/#{cluster}-#{node['hostname']}" + + directory "#{mds_directory}" do + owner "root" + group "root" + mode "0755" + recursive true + action :create + end + + unless File.exists?("#{mds_directory}/done") + execute "get or create mds keyring" do + command "ceph --cluster #{cluster} --name client.bootstrap-mds --keyring #{mds_bootstrap_directory}/#{cluster}.keyring auth get-or-create mds.#{node['hostname']} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o #{mds_directory}/keyring" + creates "#{mds_directory}/keyring" + end + ruby_block "finalise" do + block do + ["done", service_type].each do |ack| + File.open("#{mds_directory}/#{ack}", "w").close() + end + end + end + end + + if service_type == "upstart" + service "ceph-mds" do + provider Chef::Provider::Service::Upstart + action :enable + end + service "ceph-mds-all" do + provider Chef::Provider::Service::Upstart + supports :status => true + action [ :enable, :start ] + end + end + + service "ceph_mds" do + if service_type == "upstart" + service_name "ceph-mds-all-starter" + provider Chef::Provider::Service::Upstart + else + service_name "ceph" + end + supports :restart => true, :status => true + action [ :enable, :start ] + end +end diff --git a/recipes/mon.rb b/recipes/mon.rb index 197cd72..9c80c2e 100644 --- a/recipes/mon.rb +++ b/recipes/mon.rb @@ -90,9 +90,8 @@ end end -# The key is going to be automatically -# created, -# We store it when it is created +# The osd and mds keys are going to be automatically created. +# We store on the mon object when it is created ruby_block "get osd-bootstrap keyring" do block do run_out = "" @@ -105,3 +104,15 @@ end not_if { node['ceph']['bootstrap_osd_key'] } end +ruby_block "get mds-bootstrap keyring" do + block do + run_out = "" + while run_out.empty? + run_out = Mixlib::ShellOut.new("ceph auth get-key client.bootstrap-mds").run_command.stdout.strip + sleep 2 + end + node.override['ceph']['bootstrap_mds_key'] = run_out + node.save + end + not_if { node['ceph']['bootstrap_mds_key'] } +end diff --git a/recipes/osd.rb b/recipes/osd.rb index e18613c..d3b0947 100644 --- a/recipes/osd.rb +++ b/recipes/osd.rb @@ -34,10 +34,50 @@ include_recipe "ceph::default" include_recipe "ceph::conf" +if !node["ceph"]["osd_devices"].nil? + node["ceph"]["osd_devices"].each do |osd_device| + Log.debug("ceph-osd: #{osd_device}") + end +elsif node["ceph"]["osd_autoprepare"] + # set node["ceph"]["osd_autoprepare"] to true to enable automated osd disk + # discovery and preparation + osd_devices = Array.new + node['block_device'].select{|device,info| device =~ /^[hvs]d[^a]$/ and info['size'].to_i > 0}.each do |device,info| + Log.debug("ceph-osd: Candidate Device /dev/#{device} found.") + osd_devices << {"device" => "/dev/#{device}"} + end + Log.debug("ceph-osd: New Candidates = #{osd_devices}") + node.set["ceph"]["osd_devices"] = osd_devices + node.save +else + Log.warn('ceph-osd: No ceph osd_devices have been set and ceph osd_autoprepare not enabled.') +end + package 'gdisk' do action :upgrade end +# sometimes there are partitions on the disk that interfere with +# ceph-disk-prepare, so let's make sure there's nothing on each candidate disk +if node["ceph"]["osd_autoprepare"] and !node["ceph"]["osd_devices"].nil? + node["ceph"]["osd_devices"].each do |osd_device| + if osd_device['status'].nil? + ruby_block "ceph-osd: erase #{osd_device['device']} to prepare it as an osd" do + block do + devicewipe = Mixlib::ShellOut.new("sgdisk -oZ #{osd_device['device']}").run_command + if devicewipe.error! + raise "ceph-osd: erase of #{osd_device['device']} failed!" + end + end + end + elsif osd_device['status'] == 'deployed' + Log.debug("ceph-osd: Not erasing #{osd_device['device']} as it has already been deployed.") + else + Log.debug("ceph-osd: Not erasing #{osd_device['device']} as it has an unrecognised status.") + end + end +end + if !search(:node,"hostname:#{node['hostname']} AND dmcrypt:true").empty? package 'cryptsetup' do action :upgrade @@ -45,10 +85,10 @@ end service_type = node["ceph"]["osd"]["init_style"] -mons = get_mon_nodes("ceph_bootstrap_osd_key:*") +mons = get_mon_nodes("bootstrap_osd_key:*") if mons.empty? then - puts "No ceph-mon found." + Log.warn("ceph-osd: No ceph osd bootstrap key found.") else directory "/var/lib/ceph/bootstrap-osd" do @@ -60,7 +100,7 @@ # TODO cluster name cluster = 'ceph' - execute "format as keyring" do + execute "create the local keyring file" do command "ceph-authtool '/var/lib/ceph/bootstrap-osd/#{cluster}.keyring' --create-keyring --name=client.bootstrap-osd --add-key='#{mons[0]["ceph"]["bootstrap_osd_key"]}'" creates "/var/lib/ceph/bootstrap-osd/#{cluster}.keyring" end @@ -107,29 +147,26 @@ unless node["ceph"]["osd_devices"].nil? node["ceph"]["osd_devices"].each_with_index do |osd_device,index| if !osd_device["status"].nil? - Log.info("osd: osd_device #{osd_device} has already been setup.") + Log.debug("ceph-osd: osd_device #{osd_device['device']} has already been prepared.") next end dmcrypt = "" if osd_device["encrypted"] == true dmcrypt = "--dmcrypt" end - execute "Creating Ceph OSD on #{osd_device['device']}" do - command "ceph-disk-prepare #{dmcrypt} #{osd_device['device']} #{osd_device['journal']}" - action :run - notifies :create, "ruby_block[save osd_device status]" - end - # we add this status to the node env - # so that we can implement recreate - # and/or delete functionalities in the - # future. - ruby_block "save osd_device status" do + + ruby_block "ceph-osd: create osd on #{osd_device['device']}" do block do - node.normal["ceph"]["osd_devices"][index]["status"] = "deployed" - node.save + deviceprep = Mixlib::ShellOut.new("ceph-disk-prepare #{dmcrypt} #{osd_device['device']} #{osd_device['journal']}").run_command + if deviceprep.error! + raise "ceph-osd: osd creation on #{osd_device['device']} failed!" + else + node.set["ceph"]["osd_devices"][index]["status"] = "deployed" + node.save + end end - action :nothing end + end service "ceph_osd" do case service_type @@ -142,8 +179,6 @@ action [ :enable, :start ] supports :restart => true end - else - Log.info('node["ceph"]["osd_devices"] empty') end end end diff --git a/recipes/pools.rb b/recipes/pools.rb new file mode 100644 index 0000000..7a2485d --- /dev/null +++ b/recipes/pools.rb @@ -0,0 +1,33 @@ +# +# Cookbook Name:: ceph +# Recipe:: pools +# +# Copyright 2013, Business Connexion (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This recipe uses a LWRP to setup pools defined in the environment + +if !node["ceph"]["pools"].nil? + node["ceph"]["pools"].each do |pool| + Log.debug("ceph-pool: #{pool}") + ceph_manage_pool pool["name"] do + pg_num pool["pg_num"] + pgp_num pool["pgp_num"] + min_size pool["min_size"] + action :create + not_if "ceph osd lspools | grep #{pool["name"]}" + end + end +end diff --git a/recipes/setup.rb b/recipes/setup.rb new file mode 100644 index 0000000..3d700e9 --- /dev/null +++ b/recipes/setup.rb @@ -0,0 +1,50 @@ +# +# Cookbook Name:: ceph +# Recipe:: setup +# +# Copyright 2013, Business Connexion (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# make sure we die early if there is another ceph-setup node +search_query = "roles:ceph-setup NOT name:#{node.name}" +search_result = search(:node, search_query) +if search_result.length > 0 + msg = "You can only have one node with the ceph-setup role." + Chef::Application.fatal! msg +end + +# This recipe creates a new ceph cluster and sets the node up as the +# mon_initial_member for the cluster. This value can be overridden if +# desired. + +if node['ceph']['config']['fsid'].nil? + Log.debug("ceph-setup: ceph fsid not set - generating a new fsid") + require 'securerandom' + fsid = SecureRandom.uuid + node.set['ceph']['config']['fsid'] = fsid + node.save + Log.debug("ceph-setup: ceph fsid has been set to #{fsid}") +else + Log.debug("ceph-setup: ceph fsid is #{node['ceph']['config']['fsid']}") +end + +if node['ceph']['config']['mon_initial_members'].nil? + Log.debug("ceph-setup: mon_initial_members not set - using the ceph-setup node") + node.set['ceph']['config']['mon_initial_members'] = node['hostname'] + node.save + Log.debug("ceph-setup: mon_initial_members has been set to #{node['hostname']}") +else + Log.debug("ceph-setup: mon_initial_members is #{node['ceph']['config']['mon_initial_members']}") +end diff --git a/resources/manage_pool.rb b/resources/manage_pool.rb new file mode 100644 index 0000000..9a33ba8 --- /dev/null +++ b/resources/manage_pool.rb @@ -0,0 +1,26 @@ +# +# Author:: Jesse Pretorius +# Cookbook Name:: ceph +# Resources:: manage_pool +# +# Copyright 2013, Business Connexion (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +actions :create + +attribute :name, :kind_of => String, :name_attribute => true +attribute :pg_num, :kind_of => Integer, :default => 8 +attribute :pgp_num, :kind_of => Integer +attribute :min_size, :kind_of => Integer diff --git a/roles/ceph-setup.rb b/roles/ceph-setup.rb new file mode 100644 index 0000000..549cd1d --- /dev/null +++ b/roles/ceph-setup.rb @@ -0,0 +1,6 @@ +name "ceph-setup" +description "Role for the setup of a Ceph cluster. Includes a mon role." +run_list( + 'recipe[ceph::setup]', + 'role[ceph-mon]' +)