From 6e6a5844fcb7316ff9f019876cc180e23218bc25 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Sat, 9 Jan 2016 14:25:05 -0500 Subject: [PATCH 1/3] Added ceph-restapi and rgw sharding --- attributes/default.rb | 1 + attributes/repo.rb | 5 +++ attributes/restapi.rb | 28 ++++++++++++ libraries/ceph_chef_helper.rb | 47 +++++++++++++++++--- metadata.rb | 7 ++- recipes/conf.rb | 3 +- recipes/default.rb | 7 ++- recipes/mon_start.rb | 8 +++- recipes/mon_stop.rb | 8 +++- recipes/osd_start_all.rb | 8 +++- recipes/osd_stop_all.rb | 8 +++- recipes/pools_create.rb | 11 ++--- recipes/pools_delete.rb | 2 +- recipes/pools_set.rb | 8 ++-- recipes/radosgw_users.rb | 2 + recipes/restapi.rb | 77 +++++++++++++++++++++++++++++++++ templates/default/ceph.conf.erb | 13 ++++++ 17 files changed, 216 insertions(+), 27 deletions(-) create mode 100644 attributes/restapi.rb create mode 100644 recipes/restapi.rb diff --git a/attributes/default.rb b/attributes/default.rb index ccb65a1..47f54b4 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -37,6 +37,7 @@ default['ceph']['mon']['tag'] = 'ceph-mon' default['ceph']['osd']['tag'] = 'ceph-osd' default['ceph']['mds']['tag'] = 'ceph-mds' +default['ceph']['restapi']['tag'] = 'ceph-restapi' default['ceph']['install_repo'] = true default['ceph']['btrfs'] = false diff --git a/attributes/repo.rb b/attributes/repo.rb index 0684451..b4cf9a9 100644 --- a/attributes/repo.rb +++ b/attributes/repo.rb @@ -17,6 +17,11 @@ default['ceph']['branch'] = 'stable' # Can be stable, testing or dev. # Major release version to install or gitbuilder branch default['ceph']['version'] = 'hammer' +# NOTE: If the version is greater than 'hammer' then change owner and group to 'ceph' +default['ceph']['owner'] = 'root' +default['ceph']['group'] = 'root' +default['ceph']['mode'] = 0755 + default['ceph']['el_version'] = 'el7' default['ceph']['el_add_epel'] = true default['ceph']['repo_url'] = 'http://ceph.com' diff --git a/attributes/restapi.rb b/attributes/restapi.rb new file mode 100644 index 0000000..053b908 --- /dev/null +++ b/attributes/restapi.rb @@ -0,0 +1,28 @@ +# +# Cookbook Name:: ceph +# Attributes:: restapi +# +# Copyright 2015, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_attribute 'ceph-chef' + +default['ceph']['restapi']['port'] = 5080 +default['ceph']['restapi']['base_url'] = '/api/v0.1' +default['ceph']['restapi']['log']['level'] = 'warning' + +default['ceph']['restapi']['role'] = 'search-ceph-restapi' + +default['ceph']['restapi']['secret_file'] = '/etc/chef/secrets/ceph_chef_restapi' diff --git a/libraries/ceph_chef_helper.rb b/libraries/ceph_chef_helper.rb index fa44af8..0998c6e 100644 --- a/libraries/ceph_chef_helper.rb +++ b/libraries/ceph_chef_helper.rb @@ -55,6 +55,18 @@ def ceph_chef_is_radosgw_node val end +def ceph_chef_is_restapi_node + val = false + nodes = ceph_chef_restapi_nodes + nodes.each do |n| + if n['hostname'] == node['hostname'] + val = true + break + end + end + val +end + def ceph_chef_is_admin_node val = false nodes = ceph_chef_admin_nodes @@ -202,6 +214,27 @@ def ceph_chef_save_radosgw_secret(secret) secret end +def ceph_chef_restapi_secret + if node['ceph']['encrypted_data_bags'] + secret = Chef::EncryptedDataBagItem.load_secret(node['ceph']['restapi']['secret_file']) + Chef::EncryptedDataBagItem.load('ceph', 'restapi', secret)['secret'] + elsif !ceph_chef_restapi_nodes.empty? + ceph_chef_save_restapi_secret(ceph_chef_restapi_nodes[0]['ceph']['restapi-secret']) + ceph_chef_restapi_nodes[0]['ceph']['restapi-secret'] + elsif node['ceph']['restapi-secret'] + node['ceph']['restapi-secret'] + else + Chef::Log.info('No restapi secret found') + nil + end +end + +def ceph_chef_save_restapi_secret(secret) + node.set['ceph']['restapi-secret'] = secret + node.save + secret +end + # If public_network is specified with one or more networks, we need to # search for a matching monitor IP in the node environment. # 1. For each public network specified: @@ -257,7 +290,6 @@ def ceph_chef_ip_address_to_ceph_chef_address(ip, params) # For this function to work, this cookbook will need to be part of a wrapper or project that implements ceph-mon role # Returns a list of nodes (not hostnames!) def ceph_chef_mon_nodes - # results = search(:node, "role:#{node['ceph']['mon']['role']} AND chef_environment:#{node.chef_environment}") results = search(:node, "tags:#{node['ceph']['mon']['tag']}") results.map! { |x| x['hostname'] == node['hostname'] ? node : x } if !results.include?(node) && node.run_list.roles.include?(node['ceph']['mon']['role']) @@ -267,7 +299,6 @@ def ceph_chef_mon_nodes end def ceph_chef_osd_nodes - # results = search(:node, "role:#{node['ceph']['osd']['role']} AND chef_environment:#{node.chef_environment}") results = search(:node, "tags:#{node['ceph']['osd']['tag']}") results.map! { |x| x['hostname'] == node['hostname'] ? node : x } if !results.include?(node) && node.run_list.roles.include?(node['ceph']['osd']['role']) @@ -277,7 +308,6 @@ def ceph_chef_osd_nodes end def ceph_chef_radosgw_nodes - # results = search(:node, "role:#{node['ceph']['radosgw']['role']} AND chef_environment:#{node.chef_environment}") results = search(:node, "tags:#{node['ceph']['radosgw']['tag']}") results.map! { |x| x['hostname'] == node['hostname'] ? node : x } if !results.include?(node) && node.run_list.roles.include?(node['ceph']['radosgw']['role']) @@ -286,8 +316,16 @@ def ceph_chef_radosgw_nodes results.sort! { |a, b| a['hostname'] <=> b['hostname'] } end +def ceph_chef_restapi_nodes + results = search(:node, "tags:#{node['ceph']['restapi']['tag']}") + results.map! { |x| x['hostname'] == node['hostname'] ? node : x } + if !results.include?(node) && node.run_list.roles.include?(node['ceph']['restapi']['role']) + results.push(node) + end + results.sort! { |a, b| a['hostname'] <=> b['hostname'] } +end + def ceph_chef_admin_nodes - # results = search(:node, "role:#{node['ceph']['admin']['role']} AND chef_environment:#{node.chef_environment}") results = search(:node, "tags:#{node['ceph']['admin']['tag']}") results.map! { |x| x['hostname'] == node['hostname'] ? node : x } if !results.include?(node) && node.run_list.roles.include?(node['ceph']['admin']['role']) @@ -297,7 +335,6 @@ def ceph_chef_admin_nodes end def ceph_chef_mds_nodes - # results = search(:node, "role:#{node['ceph']['mds']['role']} AND chef_environment:#{node.chef_environment}") results = search(:node, "tags:#{node['ceph']['mds']['tag']}") results.map! { |x| x['hostname'] == node['hostname'] ? node : x } if !results.include?(node) && node.run_list.roles.include?(node['ceph']['mds']['role']) diff --git a/metadata.rb b/metadata.rb index 9c19228..5f8d893 100644 --- a/metadata.rb +++ b/metadata.rb @@ -1,10 +1,10 @@ name 'ceph-chef' maintainer 'Chris Jones' -maintainer_email 'cjones@cloudm2.com' +maintainer_email 'cjones303@bloomberg.net' license 'Apache v2.0' description 'Installs/Configures Ceph (Hammer and above)' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '0.9.1' +version '0.9.2' depends 'apache2', '>= 1.1.12' depends 'apt' @@ -14,3 +14,6 @@ supports 'ubuntu', '>= 14.04' supports 'redhat', '>= 7.1' supports 'centos', '>= 7.1' + +issues_url 'https://github.com/ceph/ceph-chef/issues' +source_url 'https://github.com/ceph/ceph-chef' diff --git a/recipes/conf.rb b/recipes/conf.rb index 5dfa47c..ef656a5 100644 --- a/recipes/conf.rb +++ b/recipes/conf.rb @@ -47,7 +47,8 @@ :fsid_secret => ceph_chef_fsid_secret, :mon_addresses => ceph_chef_mon_addresses, :is_rbd => node['ceph']['is_rbd'], - :is_rgw => ceph_chef_is_radosgw_node + :is_rgw => ceph_chef_is_radosgw_node, + :is_rest_api => ceph_is_rest_api_node } } mode '0644' diff --git a/recipes/default.rb b/recipes/default.rb index 4545938..28b8a4f 100644 --- a/recipes/default.rb +++ b/recipes/default.rb @@ -26,4 +26,9 @@ end # Can't put compile_time false because of templates -chef_gem 'netaddr' +# Since the cookbook will run where no net access exists then you should pre-install netaddr so check to see if it exists. +netadd = Mixlib::ShellOut.new('gem list | grep netaddr') +netadd.run_command +if !netadd.stdout + chef_gem 'netaddr' +end diff --git a/recipes/mon_start.rb b/recipes/mon_start.rb index 3c9e6f9..c114cf0 100644 --- a/recipes/mon_start.rb +++ b/recipes/mon_start.rb @@ -38,8 +38,12 @@ command 'systemctl start ceph.target' end else - execute 'raw mon start' do - command 'service ceph start mon' + # execute 'raw mon start' do + # command 'service ceph start mon' + # end + service 'ceph mon start' do + service_name 'ceph mon start' + action [:enable, :start] end end end diff --git a/recipes/mon_stop.rb b/recipes/mon_stop.rb index f3f3630..8e29057 100644 --- a/recipes/mon_stop.rb +++ b/recipes/mon_stop.rb @@ -32,7 +32,11 @@ action [:stop] end else - execute 'raw mon start' do - command 'service ceph stop mon' + # execute 'raw mon start' do + # command 'service ceph stop mon' + # end + service 'ceph mon stop' do + service_name 'ceph stop mon' + action :stop end end diff --git a/recipes/osd_start_all.rb b/recipes/osd_start_all.rb index b5e46a1..13e3e3e 100644 --- a/recipes/osd_start_all.rb +++ b/recipes/osd_start_all.rb @@ -32,7 +32,11 @@ supports :restart => true end else - execute 'raw osd start' do - command 'service ceph start osd' + # execute 'raw osd start' do + # command 'service ceph start osd' + # end + service 'ceph osd start' do + service_name 'ceph osd start' + action [:enable, :start] end end diff --git a/recipes/osd_stop_all.rb b/recipes/osd_stop_all.rb index 16945c9..5cb2994 100644 --- a/recipes/osd_stop_all.rb +++ b/recipes/osd_stop_all.rb @@ -31,7 +31,11 @@ action [:stop] end else - execute 'raw osd stop all' do - command 'service ceph stop osd' + # execute 'raw osd stop all' do + # command 'service ceph stop osd' + # end + service 'ceph stop osd' do + service_name 'ceph stop osd' + action :stop end end diff --git a/recipes/pools_create.rb b/recipes/pools_create.rb index b49f112..9c75aae 100644 --- a/recipes/pools_create.rb +++ b/recipes/pools_create.rb @@ -22,15 +22,16 @@ if node['ceph']['pools']['active'] node['ceph']['pools']['active'].each do |pool| # Create pool and set type (replicated or erasure - default is replicated) - node['ceph']['pools']["#{pool}"]['names'].each do |name| + # #{pool} + node['ceph']['pools'][pool]['names'].each do |name| pool_name = ".#{name}" ceph_chef_pool pool_name do action :create - pg_num node['ceph']['pools']["#{pool}"]['settings']['pg_num'] - pgp_num node['ceph']['pools']["#{pool}"]['settings']['pgp_num'] - type node['ceph']['pools']["#{pool}"]['settings']['type'] - options node['ceph']['pools']["#{pool}"]['settings']['options'] if node['ceph']['pools']["#{pool}"]['settings']['options'] + pg_num node['ceph']['pools'][pool]['settings']['pg_num'] + pgp_num node['ceph']['pools'][pool]['settings']['pgp_num'] + type node['ceph']['pools'][pool]['settings']['type'] + options node['ceph']['pools'][pool]['settings']['options'] if node['ceph']['pools'][pool]['settings']['options'] end # TODO: Need to add for calculated PGs options diff --git a/recipes/pools_delete.rb b/recipes/pools_delete.rb index 4504d26..cb9e266 100644 --- a/recipes/pools_delete.rb +++ b/recipes/pools_delete.rb @@ -24,7 +24,7 @@ if node['ceph']['pools']['active'] node['ceph']['pools']['active'].each do |pool| - node['ceph']['pools']["#{pool}"]['names'].each do |name| + node['ceph']['pools'][pool]['names'].each do |name| # pool_name = "#{node['ceph']['cluster']}.#{name}" pool_name = ".#{name}" diff --git a/recipes/pools_set.rb b/recipes/pools_set.rb index a942359..18def14 100644 --- a/recipes/pools_set.rb +++ b/recipes/pools_set.rb @@ -25,16 +25,16 @@ if node['ceph']['pools']['active'] node['ceph']['pools']['active'].each do |pool| # Create pool and set type (replicated or erasure - default is replicated) - node['ceph']['pools']["#{pool}"]['names'].each do |name| - cluster = ".#{node['ceph']['cluster']}" unless "#{node['ceph']['cluster']}".downcase == 'ceph' + node['ceph']['pools'][pool]['names'].each do |name| + cluster = ".#{node['ceph']['cluster']}" unless node['ceph']['cluster'].downcase == 'ceph' pool_name = "#{cluster}.#{name}" # TODO: Need to add for calculated PGs options # TODO: Need to add crush_rule_set # TODO: Add other options later for EC etc... - if node['ceph']['pools']["#{pool}"]['settings']['size'] - val = node['ceph']['pools']["#{pool}"]['settings']['size'] + if node['ceph']['pools'][pool]['settings']['size'] + val = node['ceph']['pools'][pool]['settings']['size'] else val = node['ceph']['osd']['size']['max'] end diff --git a/recipes/radosgw_users.rb b/recipes/radosgw_users.rb index c774432..0fbb9ec 100644 --- a/recipes/radosgw_users.rb +++ b/recipes/radosgw_users.rb @@ -19,9 +19,11 @@ # Create the admin user. These variables MUST exist for this to work. The default values can be found in # the radosgw.rb attributes file. They can also be overridden in multiple places. +# Admin user MUST have caps set properly. Without full rights, no admin functions can occur via the admin restful calls. ruby_block 'initialize-radosgw-admin-user' do block do rgw_admin = JSON.parse(%x[radosgw-admin user create --display-name="#{node['ceph']['radosgw']['user']['admin']['name']}" --uid="#{node['ceph']['radosgw']['user']['admin']['uid']}" --access_key="#{node['ceph']['radosgw']['user']['admin']['access_key']}" --secret="#{node['ceph']['radosgw']['user']['admin']['secret']}"]) + radosgw-admin caps add --uid="#{node['ceph']['radosgw']['user']['admin']['uid']}" --caps="users=*;buckets=*;metadata=*;usage=*;zone=*" end not_if "radosgw-admin user info --uid='#{node['ceph']['radosgw']['user']['admin']['uid']}'" end diff --git a/recipes/restapi.rb b/recipes/restapi.rb new file mode 100644 index 0000000..e831f3f --- /dev/null +++ b/recipes/restapi.rb @@ -0,0 +1,77 @@ +# +# Author: Chris Jones +# Cookbook: ceph +# +# Copyright 2015, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +node.default['ceph']['is_rest_api'] = true + +include_recipe 'ceph-chef' + +service_type = node['ceph']['mon']['init_style'] + +directory "/var/lib/ceph/restapi/#{node['ceph']['cluster']}-restapi" do + owner node['ceph']['owner'] + group node['ceph']['group'] + mode node['ceph']['mode'] + recursive true + action :create +end + +base_key = "/etc/ceph/#{node['ceph']['cluster']}.client.admin.keyring" + +# NOTE: If the restapi keyring exists and you are using the same key on for different nodes (load balancing) then +# this method will work well. Since the key is already part of the cluster the only thing needed is to copy it +# to the correct area (where ever the ceph.conf settings are pointing to on the given node). You can keep things +# simple by keeping the same ceph.conf the same (except for hostname info) for each restapi node. +execute 'write ceph-restapi-secret' do + command lazy { "ceph-authtool /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring --create-keyring --name=client.restapi --add-key='#{node['ceph']['restapi-secret']}'" } + only_if { ceph_chef_restapi_secret } + sensitive true if Chef::Resource::Execute.method_defined? :sensitive +end + +bash 'gen client-restapi-secret' do + code <<-EOH + ceph-authtool --create-keyring /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring + ceph-authtool /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring -n client.restapi --gen-key + ceph-authtool -n client.restapi --cap osd 'allow *' --cap mon '*' /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring + ceph -k #{base_key} auth add client.radosgw -i /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring + EOH + not_if { ceph_chef_restapi_secret } + notifies :create, 'ruby_block[save restapi_secret]', :immediately +end + +# This ruby_block saves the key if it is needed at any other point plus this and all node data is saved on the +# Chef Server for this given node +ruby_block 'save restapi_secret' do + block do + fetch = Mixlib::ShellOut.new("ceph-authtool /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring --print-key") + fetch.run_command + key = fetch.stdout + node.set['ceph']['restapi-secret'] = key.delete!("\n") + node.save + end + action :nothing +end + +# This is only here as part of completeness. +ruby_block 'restapi-finalize' do + block do + ['done', service_type].each do |ack| + ::File.open("/var/lib/ceph/restapi/#{node['ceph']['cluster']}-restapi/#{ack}", 'w').close + end + end +end diff --git a/templates/default/ceph.conf.erb b/templates/default/ceph.conf.erb index 8d4dde8..806b157 100644 --- a/templates/default/ceph.conf.erb +++ b/templates/default/ceph.conf.erb @@ -134,6 +134,19 @@ <% end -%> <% end -%> +<% if @is_rest_api -%> +[client.restapi.<%= node['hostname'] %>] + # public addr = <%= node['hostname'] %> + keyring = /etc/ceph/$cluster.client.restapi.keyring + restapi base url = <%= node['ceph']['restapi']['base_url'] %> + log file = /var/log/ceph/$cluster.client.restapi.<%= node['hostname'] -%>.log +<% if !node['ceph']['config']['restapi'].nil? -%> + <% node['ceph']['config']['restapi'].sort.each do |k, v| %> + <%= k %> = <%= v %> + <% end %> +<% end -%> +<% end -%> + <% node['ceph']['config-sections'].sort.each do |name, sect| %> [<%= name %>] <% sect.sort.each do |k, v| %> From ce86147f108fced555a638a22a3dc488390d9b52 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Sat, 9 Jan 2016 16:00:03 -0500 Subject: [PATCH 2/3] Added more options preparing for erasure coding --- README.md | 3 +-- attributes/radosgw.rb | 5 ++--- recipes/pools_set.rb | 22 ++++++++++++---------- recipes/restapi_start.rb | 25 +++++++++++++++++++++++++ templates/default/ceph.conf.erb | 4 ++-- 5 files changed, 42 insertions(+), 17 deletions(-) create mode 100644 recipes/restapi_start.rb diff --git a/README.md b/README.md index 745054a..cd1db71 100644 --- a/README.md +++ b/README.md @@ -146,8 +146,7 @@ Ceph Rados Gateway nodes should use the ceph-radosgw role * `node['ceph']['radosgw']['api_fqdn']` - what vhost to configure in the web server * `node['ceph']['radosgw']['admin_email']` - the admin email address to configure in the web server -* `node['ceph']['radosgw']['rgw_addr']` - the web server's bind address, such as *:80 -* `node['ceph']['radosgw']['rgw_port']` - if set, connects to the radosgw fastcgi over this port instead of a unix socket +* `node['ceph']['radosgw']['port']` - Port of the rgw. Defaults to 80 * `node['ceph']['radosgw']['webserver_companion']` - defaults to 'apache2', but it can be set to 'civetweb', or to false in order to leave it unconfigured * `node['ceph']['radosgw']['path']` - where to save the s3gw.fcgi file * `node['ceph']['config']['global']['rgw dns name']` - the main domain of the radosgw daemon, to calculate the bucket name from a subdomain diff --git a/attributes/radosgw.rb b/attributes/radosgw.rb index 197c1ca..a11fa15 100644 --- a/attributes/radosgw.rb +++ b/attributes/radosgw.rb @@ -35,8 +35,7 @@ default['ceph']['radosgw']['api_fqdn'] = 'localhost' default['ceph']['radosgw']['admin_email'] = 'admin@example.com' -default['ceph']['radosgw']['rgw_addr'] = '*:80' -default['ceph']['radosgw']['rgw_port'] = false +default['ceph']['radosgw']['port'] = 80 default['ceph']['radosgw']['webserver'] = 'civetweb' # init_style in each major section is allowed so that radosgw or osds or mons etc could be a different OS if required. @@ -56,7 +55,7 @@ default['ceph']['radosgw']['user']['test']['access_key'] = ceph_chef_secure_password_alphanum_upper(20) default['ceph']['radosgw']['user']['test']['secret'] = ceph_chef_secure_password(40) default['ceph']['radosgw']['user']['test']['max_buckets'] = 3 -default['ceph']['radosgw']['user']['test']['caps'] = 'usage=read; user=read; bucket=read;' +default['ceph']['radosgw']['user']['test']['caps'] = 'usage=read; user=read; bucket=*' default['ceph']['radosgw']['secret_file'] = '/etc/chef/secrets/ceph_chef_rgw' diff --git a/recipes/pools_set.rb b/recipes/pools_set.rb index 18def14..87e3d05 100644 --- a/recipes/pools_set.rb +++ b/recipes/pools_set.rb @@ -33,17 +33,19 @@ # TODO: Need to add crush_rule_set # TODO: Add other options later for EC etc... - if node['ceph']['pools'][pool]['settings']['size'] - val = node['ceph']['pools'][pool]['settings']['size'] - else - val = node['ceph']['osd']['size']['max'] - end + if node['ceph']['pools'][pool]['settings']['type'] == 'replicated' + if node['ceph']['pools'][pool]['settings']['size'] + val = node['ceph']['pools'][pool]['settings']['size'] + else + val = node['ceph']['osd']['size']['max'] + end - # Set... - ceph_chef_pool pool_name do - action :set - key 'size' - value val + # Set replicas... + ceph_chef_pool pool_name do + action :set + key 'size' + value val + end end end end diff --git a/recipes/restapi_start.rb b/recipes/restapi_start.rb new file mode 100644 index 0000000..bcbacb3 --- /dev/null +++ b/recipes/restapi_start.rb @@ -0,0 +1,25 @@ +# +# Author:: Chris Jones +# Cookbook Name:: ceph +# +# Copyright 2015, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +service_type = node['ceph']['mon']['init_style'] + +execute 'restapi-start' do + command 'nohup ceph-rest-api &' + not_if 'pgrep ceph-rest-api' +end diff --git a/templates/default/ceph.conf.erb b/templates/default/ceph.conf.erb index 806b157..371bf99 100644 --- a/templates/default/ceph.conf.erb +++ b/templates/default/ceph.conf.erb @@ -116,10 +116,10 @@ [client.radosgw.<%= node['hostname'] %>] host = <%= node['hostname'] %> keyring = /etc/ceph/$cluster.client.radosgw.keyring - <% if node['ceph']['radosgw']['rgw_port'] == false %> + <% if node['ceph']['radosgw']['port'].nil? %> rgw frontends = civetweb port=80 <% else -%> - rgw frontends = civetweb port=<%= node['ceph']['radosgw']['rgw_port'] %> + rgw frontends = civetweb port=<%= node['ceph']['radosgw']['port'] %> <% end -%> # rgw enable ops log = false # rgw enable usage log = false From 4ce7a9a77f0db7f005a796806f5a49def5e725ca Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Mon, 11 Jan 2016 23:10:00 -0500 Subject: [PATCH 3/3] Added restapi and base for index sharding --- metadata.rb | 2 +- recipes/admin_client.rb | 9 +++++++-- recipes/conf.rb | 2 +- recipes/default.rb | 12 +++++++----- recipes/mon_start.rb | 8 +++----- recipes/mon_stop.rb | 8 ++------ recipes/osd_start_all.rb | 5 +++-- recipes/osd_stop_all.rb | 8 ++------ recipes/radosgw.rb | 17 ++++++++++------- recipes/radosgw_users.rb | 5 +++-- recipes/restapi.rb | 15 +++++++-------- templates/default/ceph.conf.erb | 6 +++--- 12 files changed, 49 insertions(+), 48 deletions(-) diff --git a/metadata.rb b/metadata.rb index 5f8d893..daca2d4 100644 --- a/metadata.rb +++ b/metadata.rb @@ -4,7 +4,7 @@ license 'Apache v2.0' description 'Installs/Configures Ceph (Hammer and above)' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '0.9.2' +version '0.9.3' depends 'apache2', '>= 1.1.12' depends 'apt' diff --git a/recipes/admin_client.rb b/recipes/admin_client.rb index 80fc692..35349b1 100644 --- a/recipes/admin_client.rb +++ b/recipes/admin_client.rb @@ -17,17 +17,22 @@ # limitations under the License. # +keyring = "/etc/ceph/#{node['ceph']['cluster']}.client.admin.keyring" + # This will execute on other nodes besides the first mon node. execute 'format ceph-admin-secret as keyring' do - command lazy { "ceph-authtool --create-keyring /etc/ceph/#{node['ceph']['cluster']}.client.admin.keyring --name=client.admin --add-key='#{node['ceph']['admin-secret']}' --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *'" } + command lazy { "ceph-authtool --create-keyring #{keyring} --name=client.admin --add-key='#{node['ceph']['admin-secret']}' --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *'" } + creates keyring only_if { ceph_chef_admin_secret } sensitive true if Chef::Resource::Execute.method_defined? :sensitive end execute 'gen ceph-admin-secret' do - command lazy { "ceph-authtool --create-keyring /etc/ceph/#{node['ceph']['cluster']}.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *'" } + command lazy { "ceph-authtool --create-keyring #{keyring} --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *'" } + creates keyring not_if { ceph_chef_admin_secret } notifies :create, 'ruby_block[save ceph_chef_admin_secret]', :immediately + sensitive true if Chef::Resource::Execute.method_defined? :sensitive end ruby_block 'save ceph_chef_admin_secret' do diff --git a/recipes/conf.rb b/recipes/conf.rb index ef656a5..88907a0 100644 --- a/recipes/conf.rb +++ b/recipes/conf.rb @@ -48,7 +48,7 @@ :mon_addresses => ceph_chef_mon_addresses, :is_rbd => node['ceph']['is_rbd'], :is_rgw => ceph_chef_is_radosgw_node, - :is_rest_api => ceph_is_rest_api_node + :is_rest_api => ceph_chef_is_restapi_node } } mode '0644' diff --git a/recipes/default.rb b/recipes/default.rb index 28b8a4f..b8c3dc7 100644 --- a/recipes/default.rb +++ b/recipes/default.rb @@ -27,8 +27,10 @@ # Can't put compile_time false because of templates # Since the cookbook will run where no net access exists then you should pre-install netaddr so check to see if it exists. -netadd = Mixlib::ShellOut.new('gem list | grep netaddr') -netadd.run_command -if !netadd.stdout - chef_gem 'netaddr' -end +#netadd = Mixlib::ShellOut.new('gem list | grep netaddr') +#netadd.run_command +#if !netadd.stdout +# chef_gem 'netaddr' +#end + +chef_gem 'netaddr' diff --git a/recipes/mon_start.rb b/recipes/mon_start.rb index c114cf0..845ae9f 100644 --- a/recipes/mon_start.rb +++ b/recipes/mon_start.rb @@ -38,11 +38,9 @@ command 'systemctl start ceph.target' end else - # execute 'raw mon start' do - # command 'service ceph start mon' - # end - service 'ceph mon start' do - service_name 'ceph mon start' + service 'ceph_mon' do + service_name 'ceph' + supports :restart => true, :status => true action [:enable, :start] end end diff --git a/recipes/mon_stop.rb b/recipes/mon_stop.rb index 8e29057..f3f3630 100644 --- a/recipes/mon_stop.rb +++ b/recipes/mon_stop.rb @@ -32,11 +32,7 @@ action [:stop] end else - # execute 'raw mon start' do - # command 'service ceph stop mon' - # end - service 'ceph mon stop' do - service_name 'ceph stop mon' - action :stop + execute 'raw mon start' do + command 'service ceph stop mon' end end diff --git a/recipes/osd_start_all.rb b/recipes/osd_start_all.rb index 13e3e3e..4741101 100644 --- a/recipes/osd_start_all.rb +++ b/recipes/osd_start_all.rb @@ -35,8 +35,9 @@ # execute 'raw osd start' do # command 'service ceph start osd' # end - service 'ceph osd start' do - service_name 'ceph osd start' + service 'ceph_osd' do + service_name 'ceph' + supports :restart => true, :status => true action [:enable, :start] end end diff --git a/recipes/osd_stop_all.rb b/recipes/osd_stop_all.rb index 5cb2994..16945c9 100644 --- a/recipes/osd_stop_all.rb +++ b/recipes/osd_stop_all.rb @@ -31,11 +31,7 @@ action [:stop] end else - # execute 'raw osd stop all' do - # command 'service ceph stop osd' - # end - service 'ceph stop osd' do - service_name 'ceph stop osd' - action :stop + execute 'raw osd stop all' do + command 'service ceph stop osd' end end diff --git a/recipes/radosgw.rb b/recipes/radosgw.rb index 04f2da6..26ecd10 100644 --- a/recipes/radosgw.rb +++ b/recipes/radosgw.rb @@ -66,33 +66,36 @@ # NOTE: This base_key can also be the bootstrap-rgw key (ceph.keyring) if desired. Just change it here. base_key = "/etc/ceph/#{node['ceph']['cluster']}.client.admin.keyring" +keyring = "/etc/ceph/#{node['ceph']['cluster']}.client.radosgw.keyring" # NOTE: If the rgw keyring exists and you are using the same key on for different nodes (load balancing) then # this method will work well. Since the key is already part of the cluster the only thing needed is to copy it # to the correct area (where ever the ceph.conf settings are pointing to on the given node). You can keep things # simple by keeping the same ceph.conf the same (except for hostname info) for each rgw node. + execute 'write ceph-radosgw-secret' do - command lazy { "ceph-authtool /etc/ceph/#{node['ceph']['cluster']}.client.radosgw.keyring --create-keyring --name=client.radosgw.#{node['hostname']} --add-key='#{node['ceph']['radosgw-secret']}'" } + command lazy { "ceph-authtool #{keyring} --create-keyring --name=client.radosgw.#{node['hostname']} --add-key='#{node['ceph']['radosgw-secret']}'" } + creates keyring only_if { ceph_chef_radosgw_secret } sensitive true if Chef::Resource::Execute.method_defined? :sensitive end -bash 'gen client-radosgw-secret' do - code <<-EOH - ceph-authtool --create-keyring /etc/ceph/#{node['ceph']['cluster']}.client.radosgw.keyring - ceph-authtool /etc/ceph/#{node['ceph']['cluster']}.client.radosgw.keyring -n client.radosgw.#{node['hostname']} --gen-key - ceph-authtool -n client.radosgw.#{node['hostname']} --cap osd 'allow rwx' --cap mon 'allow rw' /etc/ceph/#{node['ceph']['cluster']}.client.radosgw.keyring +execute 'gen client-radosgw-secret' do + command <<-EOH + ceph-authtool --create-keyring #{keyring} -n client.radosgw.#{node['hostname']} --gen-key --cap osd 'allow rwx' --cap mon 'allow rw' ceph -k #{base_key} auth add client.radosgw.#{node['hostname']} -i /etc/ceph/#{node['ceph']['cluster']}.client.radosgw.keyring EOH + creates keyring not_if { ceph_chef_radosgw_secret } notifies :create, 'ruby_block[save radosgw_secret]', :immediately + sensitive true if Chef::Resource::Execute.method_defined? :sensitive end # This ruby_block saves the key if it is needed at any other point plus this and all node data is saved on the # Chef Server for this given node ruby_block 'save radosgw_secret' do block do - fetch = Mixlib::ShellOut.new("ceph-authtool /etc/ceph/#{node['ceph']['cluster']}.client.radosgw.keyring --print-key") + fetch = Mixlib::ShellOut.new("ceph-authtool #{keyring} --print-key") fetch.run_command key = fetch.stdout node.set['ceph']['radosgw-secret'] = key.delete!("\n") diff --git a/recipes/radosgw_users.rb b/recipes/radosgw_users.rb index 0fbb9ec..4147140 100644 --- a/recipes/radosgw_users.rb +++ b/recipes/radosgw_users.rb @@ -23,7 +23,7 @@ ruby_block 'initialize-radosgw-admin-user' do block do rgw_admin = JSON.parse(%x[radosgw-admin user create --display-name="#{node['ceph']['radosgw']['user']['admin']['name']}" --uid="#{node['ceph']['radosgw']['user']['admin']['uid']}" --access_key="#{node['ceph']['radosgw']['user']['admin']['access_key']}" --secret="#{node['ceph']['radosgw']['user']['admin']['secret']}"]) - radosgw-admin caps add --uid="#{node['ceph']['radosgw']['user']['admin']['uid']}" --caps="users=*;buckets=*;metadata=*;usage=*;zone=*" + rgw_admin_cap = JSON.parse(%x[radosgw-admin caps add --uid="#{node['ceph']['radosgw']['user']['admin']['uid']}" --caps="users=*;buckets=*;metadata=*;usage=*;zone=*"]) end not_if "radosgw-admin user info --uid='#{node['ceph']['radosgw']['user']['admin']['uid']}'" end @@ -32,7 +32,8 @@ if node['ceph']['radosgw']['user']['test']['uid'] ruby_block 'initialize-radosgw-test-user' do block do - rgw_admin = JSON.parse(%x[radosgw-admin user create --display-name="#{node['ceph']['radosgw']['user']['test']['name']}" --uid="#{node['ceph']['radosgw']['user']['test']['uid']}" --max-buckets=node['ceph']['radosgw']['user']['test']['max_buckets'] --access_key="#{node['ceph']['radosgw']['user']['test']['access_key']}" --secret="#{node['ceph']['radosgw']['user']['test']['secret']}" --caps="#{node['ceph']['radosgw']['user']['test']['caps']}"]) + rgw_tester = JSON.parse(%x[radosgw-admin user create --display-name="#{node['ceph']['radosgw']['user']['test']['name']}" --uid="#{node['ceph']['radosgw']['user']['test']['uid']}" --max-buckets=node['ceph']['radosgw']['user']['test']['max_buckets'] --access_key="#{node['ceph']['radosgw']['user']['test']['access_key']}" --secret="#{node['ceph']['radosgw']['user']['test']['secret']}" --caps="#{node['ceph']['radosgw']['user']['test']['caps']}"]) + rgw_tester_cap = JSON.parse(%x[radosgw-admin caps add --uid="#{node['ceph']['radosgw']['user']['test']['uid']}" --caps="#{node['ceph']['radosgw']['user']['test']['caps']}"]) end not_if "radosgw-admin user info --uid='#{node['ceph']['radosgw']['user']['test']['uid']}'" end diff --git a/recipes/restapi.rb b/recipes/restapi.rb index e831f3f..2b459fe 100644 --- a/recipes/restapi.rb +++ b/recipes/restapi.rb @@ -32,26 +32,25 @@ end base_key = "/etc/ceph/#{node['ceph']['cluster']}.client.admin.keyring" +keyring = "/etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring" # NOTE: If the restapi keyring exists and you are using the same key on for different nodes (load balancing) then # this method will work well. Since the key is already part of the cluster the only thing needed is to copy it # to the correct area (where ever the ceph.conf settings are pointing to on the given node). You can keep things # simple by keeping the same ceph.conf the same (except for hostname info) for each restapi node. execute 'write ceph-restapi-secret' do - command lazy { "ceph-authtool /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring --create-keyring --name=client.restapi --add-key='#{node['ceph']['restapi-secret']}'" } + command lazy { "ceph-authtool #{keyring} --create-keyring --name=client.restapi --add-key='#{node['ceph']['restapi-secret']}'" } only_if { ceph_chef_restapi_secret } sensitive true if Chef::Resource::Execute.method_defined? :sensitive end -bash 'gen client-restapi-secret' do - code <<-EOH - ceph-authtool --create-keyring /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring - ceph-authtool /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring -n client.restapi --gen-key - ceph-authtool -n client.restapi --cap osd 'allow *' --cap mon '*' /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring - ceph -k #{base_key} auth add client.radosgw -i /etc/ceph/#{node['ceph']['cluster']}.client.restapi.keyring - EOH +# command lazy { "ceph-authtool --create-keyring #{keyring} -n client.restapi.#{node['hostname']} --gen-key --cap osd 'allow *' --cap mon 'allow *'" } +execute 'gen client-restapi-secret' do + command lazy { "ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o #{keyring}" } + creates keyring not_if { ceph_chef_restapi_secret } notifies :create, 'ruby_block[save restapi_secret]', :immediately + sensitive true if Chef::Resource::Execute.method_defined? :sensitive end # This ruby_block saves the key if it is needed at any other point plus this and all node data is saved on the diff --git a/templates/default/ceph.conf.erb b/templates/default/ceph.conf.erb index 371bf99..2b46a54 100644 --- a/templates/default/ceph.conf.erb +++ b/templates/default/ceph.conf.erb @@ -135,11 +135,11 @@ <% end -%> <% if @is_rest_api -%> -[client.restapi.<%= node['hostname'] %>] - # public addr = <%= node['hostname'] %> +[client.restapi] + public addr = <%= node['ipaddress'] %>:<%= node['ceph']['restapi']['port'] %> keyring = /etc/ceph/$cluster.client.restapi.keyring restapi base url = <%= node['ceph']['restapi']['base_url'] %> - log file = /var/log/ceph/$cluster.client.restapi.<%= node['hostname'] -%>.log + log file = /var/log/ceph/$cluster.client.restapi.log <% if !node['ceph']['config']['restapi'].nil? -%> <% node['ceph']['config']['restapi'].sort.each do |k, v| %> <%= k %> = <%= v %>