Allow 4.1 to install ceph and riak properly

Berksfile

@@ -1,7 +1,8 @@
 source "http://api.berkshelf.com"
 
 metadata
-cookbook "ceph-deploy", github: "shaon/ceph-deploy"
+cookbook "haproxy", github: "shaon/haproxy-cookbook"
+cookbook "ceph-cluster", github: "shaon/ceph-cluster"
 cookbook "riakcs-cluster", github: "shaon/riakcs-cluster"
 cookbook "riak-cs-create-admin-user", github: "hectcastro/chef-riak-cs-create-admin-user"
 cookbook "riak-cs-ssl", github: "hectcastro/chef-riak-cs-ssl"

attributes/default.rb

@@ -35,6 +35,7 @@
 default['eucalyptus']['admin-ssh-pub-key'] = ""
 default["eucalyptus"]["home-directory"] = "/"
 default["eucalyptus"]["set-bind-addr"] = false
+# default["eucalyptus"]["bind-interface"] = 'eth0'
 default["eucalyptus"]["log-level"] = "INFO"
 default["eucalyptus"]["user"] = "eucalyptus"
 default["eucalyptus"]["cloud-opts"] = ""

libraries/bind-addr.rb

@@ -0,0 +1,34 @@
+#
+# Cookbook Name:: eucalyptus
+# Library:: bind-addr
+#
+#Copyright [2014] [Eucalyptus Systems]
+##
+##Licensed under the Apache License, Version 2.0 (the "License");
+##you may not use this file except in compliance with the License.
+##You may obtain a copy of the License at
+##
+##    http://www.apache.org/licenses/LICENSE-2.0
+##
+##    Unless required by applicable law or agreed to in writing, software
+##    distributed under the License is distributed on an "AS IS" BASIS,
+##    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+##    See the License for the specific language governing permissions and
+##    limitations under the License.
+##
+#
+
+module Eucalyptus
+  module BindAddr
+    def self.get_bind_interface_ip(node)
+      bind_interface = node["eucalyptus"]["bind-interface"]
+      raise "Setting the bind interface was requested but not bind-interface parameter was set" if bind_interface.nil?
+      if node["network"]["interfaces"].has_key? bind_interface
+        bind_addr = node[:network][:interfaces][bind_interface][:addresses].find {|addr, addr_info| addr_info[:family] == "inet"}.first
+        bind_addr
+      else
+        raise "Unable to find requested bind interface #{bind_interface} on #{node["ipaddress"]}"
+      end
+    end
+  end
+end

libraries/ceph.rb

@@ -17,59 +17,108 @@ def self.is_ceph?(node)
     end
 
     def self.make_ceph_config(node)
-      clusters = node['eucalyptus']['topology']['clusters']
+      if node['ceph'] != nil
+        mon_bootstrap = node['ceph']['topology']['mon_bootstrap']['ipaddr']
+        environment = node.chef_environment
 
-      clusters.each do |name, info|
-        ceph_cluster_user = node['eucalyptus']['topology']['clusters'][name]['ceph_cluster']['ceph_user']
+        mon_hostnames = []
+        mon_ipaddrs = []
+        mons = node['ceph']['topology']['mons']
+        if mons != nil
+          mons.each do |value|
+            mon_hostnames << value['hostname']
+            mon_ipaddrs << "#{value['ipaddr']}:6789"
+          end
+        end
+        mon_hostnames << node['ceph']['topology']['mon_bootstrap']['hostname']
+        mon_ipaddrs << "#{node['ceph']['topology']['mon_bootstrap']['ipaddr']}:6789"
 
-        keyring_content = "[client." + ceph_cluster_user + "]\n\t"
 
-        ceph_cluster_keyring = node['eucalyptus']['topology']['clusters'][name]['ceph_cluster']['keyring']
-        ceph_cluster_keyring.each do |keyring_key, keyring_value|
-          keyring_content = "#{keyring_content}" + "#{keyring_key} = #{keyring_value}\n"
-        end
+        file_name = "/etc/ceph/ceph.conf"
+        Chef::Log.info "Getting all attributes from #{mon_bootstrap}"
+        bootstrap_node = Chef::Search::Query.new.search(:node, "addresses:#{mon_bootstrap}").first.first
+        config_data = bootstrap_node.attributes['ceph']['config']['conf_data']
+
+        mons_config_content = "[mon]\n"
+        mons_config_content = "#{mons_config_content}" + "mon host = " + mon_hostnames.join(",") + "\n"
+        mons_config_content = "#{mons_config_content}" + "mon addr = " + mon_ipaddrs.join(",") + "\n"
+        config_data = Base64.decode64(config_data) + "#{mons_config_content}"
 
-        file_name = "/etc/ceph/ceph.client." + ceph_cluster_user + ".keyring"
-        Chef::Log.info "Writing keyring file: #{file_name}"
         File.open(file_name, 'w') do |file|
-          file.puts keyring_content
+          file.puts config_data
         end
         FileUtils.chmod 0744, file_name
 
-        config_content = "[global]\n"
-        ceph_cluster_global = node['eucalyptus']['topology']['clusters'][name]['ceph_cluster']['global']
-        ceph_cluster_global.each do |ceph_key, ceph_value|
-          Chef::Log.info "shaon - cluster_detail: name: #{ceph_key} and info: #{ceph_value}"
-          config_content = "#{config_content}" + "#{ceph_key} = #{ceph_value}\n"
+        file_name = "/etc/ceph/ceph.client.admin.keyring"
+        keyring_data = bootstrap_node.attributes['ceph']['config']['keyring_data']
+        File.open(file_name, 'w') do |file|
+          file.puts Base64.decode64(keyring_data)
         end
+      else
+        clusters = node['eucalyptus']['topology']['clusters']
+        clusters.each do |name, info|
+          ceph_cluster_user = node['eucalyptus']['topology']['clusters'][name]['ceph_cluster']['ceph_user']
 
-        config_content = "#{config_content}" + "[mon]\n"
-        ceph_cluster_mon = node['eucalyptus']['topology']['clusters'][name]['ceph_cluster']['mon']
-        ceph_cluster_mon.each do |ceph_key, ceph_value|
-          Chef::Log.info "shaon - cluster_detail: name: #{ceph_key} and info: #{ceph_value}"
-          config_content = "#{config_content}" + "#{ceph_key} = #{ceph_value}\n"
-        end
+          keyring_content = "[client." + ceph_cluster_user + "]\n\t"
 
-        file_name = "/etc/ceph/ceph.conf"
-        Chef::Log.info "Writing config file: #{file_name}"
-        File.open(file_name, 'w') do |file|
-          file.puts config_content
+          ceph_cluster_keyring = node['eucalyptus']['topology']['clusters'][name]['ceph_cluster']['keyring']
+          ceph_cluster_keyring.each do |keyring_key, keyring_value|
+            keyring_content = "#{keyring_content}" + "#{keyring_key} = #{keyring_value}\n"
+          end
+
+          file_name = "/etc/ceph/ceph.client." + ceph_cluster_user + ".keyring"
+          Chef::Log.info "Writing keyring file: #{file_name}"
+          File.open(file_name, 'w') do |file|
+            file.puts keyring_content
+          end
+          FileUtils.chmod 0744, file_name
+
+          config_content = "[global]\n"
+          ceph_cluster_global = node['eucalyptus']['topology']['clusters'][name]['ceph_cluster']['global']
+          ceph_cluster_global.each do |ceph_key, ceph_value|
+            Chef::Log.info "shaon - cluster_detail: name: #{ceph_key} and info: #{ceph_value}"
+            config_content = "#{config_content}" + "#{ceph_key} = #{ceph_value}\n"
+          end
+
+          config_content = "#{config_content}" + "[mon]\n"
+          ceph_cluster_mon = node['eucalyptus']['topology']['clusters'][name]['ceph_cluster']['mon']
+          ceph_cluster_mon.each do |ceph_key, ceph_value|
+            Chef::Log.info "shaon - cluster_detail: name: #{ceph_key} and info: #{ceph_value}"
+            config_content = "#{config_content}" + "#{ceph_key} = #{ceph_value}\n"
+          end
+
+          file_name = "/etc/ceph/ceph.conf"
+          Chef::Log.info "Writing config file: #{file_name}"
+          File.open(file_name, 'w') do |file|
+            file.puts config_content
+          end
         end
       end
     end
 
     def self.set_ceph_credentials(node)
       self.make_ceph_config(node)
-      cluster_name = Eucalyptus::KeySync.get_local_cluster_name(node)
-      ceph_cluster_user = node['eucalyptus']['topology']['clusters'][cluster_name]['ceph_cluster']['ceph_user']
-      file_name = "ceph.client." + ceph_cluster_user + ".keyring"
-      if ::File.exists?('/etc/eucalyptus/eucalyptus.conf')
-        Chef::Log.info "Writing to /etc/eucalyptus/eucalyptus.conf file"
-        file = Chef::Util::FileEdit.new("/etc/eucalyptus/eucalyptus.conf")
-        file.insert_line_if_no_match("/CEPH_USER_NAME=\"#{ceph_cluster_user}\"/", "CEPH_USER_NAME=\"#{ceph_cluster_user}\"")
-        file.insert_line_if_no_match("/CEPH_KEYRING_PATH=\"/etc/ceph/#{file_name}\"/", "CEPH_KEYRING_PATH=\"/etc/ceph/#{file_name}\"")
-        file.insert_line_if_no_match("/CEPH_CONFIG_PATH=\"/etc/ceph/ceph.conf\"/", "CEPH_CONFIG_PATH=\"/etc/ceph/ceph.conf\"")
-        file.write_file
+      if node['ceph'] != nil
+        if ::File.exists?('/etc/eucalyptus/eucalyptus.conf')
+          Chef::Log.info "Writing new ceph creds to /etc/eucalyptus/eucalyptus.conf file"
+          file = Chef::Util::FileEdit.new("/etc/eucalyptus/eucalyptus.conf")
+          file.insert_line_if_no_match("/CEPH_USER_NAME=\"admin\"/", "CEPH_USER_NAME=\"admin\"")
+          file.insert_line_if_no_match("/CEPH_KEYRING_PATH=\"/etc/ceph/ceph.client.admin.keyring\"/", "CEPH_KEYRING_PATH=\"/etc/ceph/ceph.client.admin.keyring\"")
+          file.insert_line_if_no_match("/CEPH_CONFIG_PATH=\"/etc/ceph/ceph.conf\"/", "CEPH_CONFIG_PATH=\"/etc/ceph/ceph.conf\"")
+          file.write_file
+        end
+      else
+        cluster_name = Eucalyptus::KeySync.get_local_cluster_name(node)
+        ceph_cluster_user = node['eucalyptus']['topology']['clusters'][cluster_name]['ceph_cluster']['ceph_user']
+        file_name = "ceph.client." + ceph_cluster_user + ".keyring"
+        if ::File.exists?('/etc/eucalyptus/eucalyptus.conf')
+          Chef::Log.info "Writing to /etc/eucalyptus/eucalyptus.conf file"
+          file = Chef::Util::FileEdit.new("/etc/eucalyptus/eucalyptus.conf")
+          file.insert_line_if_no_match("/CEPH_USER_NAME=\"#{ceph_cluster_user}\"/", "CEPH_USER_NAME=\"#{ceph_cluster_user}\"")
+          file.insert_line_if_no_match("/CEPH_KEYRING_PATH=\"/etc/ceph/#{file_name}\"/", "CEPH_KEYRING_PATH=\"/etc/ceph/#{file_name}\"")
+          file.insert_line_if_no_match("/CEPH_CONFIG_PATH=\"/etc/ceph/ceph.conf\"/", "CEPH_CONFIG_PATH=\"/etc/ceph/ceph.conf\"")
+          file.write_file
+        end
       end
     end
 

libraries/create_riakcs_user.rb

@@ -34,6 +34,28 @@ def self.create_riakcs_user(name, email, fqdn, port)
       raise e
     end
 
+    def self.download_riak_credentials(node)
+      head = nil
+      head_ip = node['riakcs_cluster']['topology']['head']['ipaddr']
+      environment = node.chef_environment
+      Chef::Log.info "Getting all attributes from head"
+      head = Chef::Search::Query.new.search(:node, "addresses:#{head_ip}").first.first
+      cert = head.attributes['riak_cs']['credentials']
+      node.set['riak_cs']['credentials'] = cert
+      node.save
+
+      file_name = "/root/creds.txt"
+      File.open(file_name, 'w') do |file|
+        file.puts Base64.decode64(node['riak_cs']['credentials'])
+      end
+      FileUtils.chmod 0644, file_name
+      hash = Hash[File.read('/root/creds.txt').split("\n").map{|i|i.split(':')}]
+      admin_key = hash['RIAKCS_ACCESS_KEY_ID']
+      admin_secret = hash['RIAKCS_SECRET_ACCESS_KEY']
+
+      [ admin_key, admin_secret ]
+    end
+
   end
 end
 

recipes/cloud-service.rb

@@ -48,13 +48,12 @@
 end
 
 if node["eucalyptus"]["set-bind-addr"] and not node["eucalyptus"]["cloud-opts"].include?("bind-addr")
-  bind_addr = node["ipaddress"]
-  node["network"]["interfaces"].each do |if_name, if_info|
-    if_info["addresses"].each do |addr, addr_info|
-      if node["eucalyptus"]["topology"]["user-facing"].include?(addr)
-        bind_addr = addr
-      end
-    end
+  if node["eucalyptus"]["bind-interface"]
+    # Auto detect IP from interface name
+    bind_addr = Eucalyptus::BindAddr.get_bind_interface_ip(node)
+  else
+    # Use default gw interface IP
+    bind_addr = node["ipaddress"]
   end
   node.override['eucalyptus']['cloud-opts'] = node['eucalyptus']['cloud-opts'] + " --bind-addr=" + bind_addr
 end

recipes/configure.rb

@@ -22,7 +22,44 @@
 modify_property = "#{command_prefix}/usr/sbin/euca-modify-property"
 describe_services = "#{command_prefix}/usr/sbin/euca-describe-services"
 describe_property = "#{command_prefix}/usr/sbin/euca-describe-properties"
-if node['eucalyptus']['topology']['riakcs']
+
+
+if node['riak_cs']
+  admin_key, admin_secret = RiakCSHelper::CreateUser.download_riak_credentials(node)
+  Chef::Log.info "Existing RiakCS admin_key: #{admin_key}"
+  Chef::Log.info "Existing RiakCS admin_secret: #{admin_secret}"
+  execute "Set-objectstorage.providerclient-to-riakcs" do
+    command "#{modify_property} -p objectstorage.providerclient=riakcs"
+    retries 15
+    retry_delay 20
+  end
+
+  if node['riakcs_cluster']['topology']['load_balancer']
+    if node['haproxy']['incoming_port']
+      s3_endpoint = "#{node['riakcs_cluster']['topology']['load_balancer']}:#{node['haproxy']['incoming_port']}"
+    else
+      s3_endpoint = "#{node['riakcs_cluster']['topology']['load_balancer']}:80"
+    end
+  else
+    s3_endpoint = "#{node['riakcs_cluster']['topology']['head']['ipaddr']}:8080"
+  end
+
+  execute "Set S3 endpoint" do
+    command "#{modify_property} -p objectstorage.s3provider.s3endpoint=#{s3_endpoint}"
+    retries 15
+    retry_delay 20
+  end
+  execute "Set providerclient access-key" do
+    command "#{modify_property} -p objectstorage.s3provider.s3accesskey=#{admin_key}"
+    retries 5
+    retry_delay 20
+  end
+  execute "Set providerclient secret-key" do
+    command "#{modify_property} -p objectstorage.s3provider.s3secretkey=#{admin_secret}"
+    retries 5
+    retry_delay 20
+  end
+elsif node['eucalyptus']['topology']['riakcs']
   execute "Set OSG providerclient to riakcs" do
     command "#{modify_property} -p objectstorage.providerclient=riakcs"
     retries 15
@@ -180,6 +217,8 @@
   end
   execute "source #{node['eucalyptus']['admin-cred-dir']}/eucarc && export EUCALYPTUS=#{node["eucalyptus"]["home-directory"]} && #{disable_proxy} euca-install-imaging-worker --install-default" do
     only_if "#{describe_property} services.imaging.worker.image | grep 'NULL'"
+    retries 5
+    retry_delay 20
   end
   execute "source #{node['eucalyptus']['admin-cred-dir']}/eucarc && export EUCALYPTUS=#{node["eucalyptus"]["home-directory"]} && #{disable_proxy} euca-install-load-balancer --install-default" do
     only_if "#{describe_property} services.loadbalancing.worker.image | grep 'NULL'"

recipes/nuke.rb

@@ -26,10 +26,6 @@
   service "eucanetd" do
     action [ :stop ]
   end
-
-  execute "eucanetd -F || true" do
-    only_if "which eucanetd"
-  end
 end
 
 service "eucalyptus-cc" do

recipes/storage-controller.rb

@@ -22,7 +22,14 @@
 
 ### Set bind-addr if necessary
 if node["eucalyptus"]["set-bind-addr"] and not node["eucalyptus"]["cloud-opts"].include?("bind-addr")
-  node.override['eucalyptus']['cloud-opts'] = node['eucalyptus']['cloud-opts'] + " --bind-addr=" + node["eucalyptus"]["topology"]['clusters'][Eucalyptus::KeySync.get_local_cluster_name(node)]["sc-1"]
+  if node["eucalyptus"]["bind-interface"]
+    # Auto detect IP from interface name
+    bind_addr = Eucalyptus::BindAddr.get_bind_interface_ip(node)
+  else
+    # Use default gw interface IP
+    bind_addr = node["ipaddress"]
+  end
+  node.override['eucalyptus']['cloud-opts'] = node['eucalyptus']['cloud-opts'] + " --bind-addr=" + bind_addr
 end
 
 if node["eucalyptus"]["install-type"] == "packages"

recipes/walrus.rb

@@ -42,7 +42,14 @@
 end
 
 if node["eucalyptus"]["set-bind-addr"] and not node["eucalyptus"]["cloud-opts"].include?("bind-addr")
-  node.override['eucalyptus']['cloud-opts'] = node['eucalyptus']['cloud-opts'] + " --bind-addr=" + node["eucalyptus"]["topology"]["walrus"]
+  if node["eucalyptus"]["bind-interface"]
+    # Auto detect IP from interface name
+    bind_addr = Eucalyptus::BindAddr.get_bind_interface_ip(node)
+  else
+    # Use default gw interface IP
+    bind_addr = node["ipaddress"]
+  end
+  node.override['eucalyptus']['cloud-opts'] = node['eucalyptus']['cloud-opts'] + " --bind-addr=" + bind_addr
 end
 
 template "eucalyptus.conf" do