Prevent transient-storage based clusters from prompting for cluster type upon cluster reboot

Author: afgane

cm/app.py

@@ -1,6 +1,7 @@
 import config
 import logging
 import logging.config
+import os
 import sys
 from cm.clouds.cloud_config import CloudConfig
 from cm.framework import messages
@@ -40,6 +41,10 @@ def __init__(self, **kwargs):
         print "Python version: ", sys.version_info[:2]
         self.PERSISTENT_DATA_VERSION = 3  # Current expected and generated PD version
         self.DEPLOYMENT_VERSION = 2
+        # Instance persistent data file. This file gets created for
+        # test/transient cluster types and stores the cluster config. In case
+        # of a reboot, read the file to automatically recreate the services.
+        self.INSTANCE_PD_FILE = '/mnt/persistent_data-current.yaml'
         cc = CloudConfig(app=self)
         # Get the type of cloud currently running on
         self.cloud_type = cc.get_cloud_type()
@@ -96,21 +101,30 @@ def __init__(self, **kwargs):
         # This enables cluster configuration to be recovered on cluster re-
         # instantiation
         self.manager = None
+        pd = None
         if self.use_object_store and 'bucket_cluster' in self.config:
-            log.debug("Getting pd.yaml")
+            log.debug("Looking for existing cluster persistent data (PD).")
             validate = True if self.cloud_type == 'ec2' else False
             if not self.TESTFLAG and misc.get_file_from_bucket(
                     self.cloud_interface.get_s3_connection(),
                     self.config['bucket_cluster'],
                     'persistent_data.yaml', 'pd.yaml',
                     validate=validate):
-                pd = misc.load_yaml_file('pd.yaml')
-                self.config.user_data = misc.merge_yaml_objects(self.config.user_data, pd)
-                self.config.user_data = misc.normalize_user_data(self, self.config.user_data)
-            else:
-                log.debug("Setting deployment_version to {0}".format(self.DEPLOYMENT_VERSION))
-                # This is a new cluster so default to the current version
-                self.config.user_data['deployment_version'] = self.DEPLOYMENT_VERSION
+                        log.debug("Loading bucket PD file pd.yaml")
+                        pd = misc.load_yaml_file('pd.yaml')
+        # Have not found the file in the cluster bucket, look on the instance
+        if not pd:
+            if os.path.exists(self.INSTANCE_PD_FILE):
+                log.debug("Loading instance PD file {0}".format(self.INSTANCE_PD_FILE))
+                pd = misc.load_yaml_file(self.INSTANCE_PD_FILE)
+        if pd:
+            self.config.user_data = misc.merge_yaml_objects(self.config.user_data, pd)
+            self.config.user_data = misc.normalize_user_data(self, self.config.user_data)
+        else:
+            log.debug("No PD to go by. Setting deployment_version to {0}."
+                      .format(self.DEPLOYMENT_VERSION))
+            # This is a new cluster so default to the current deployment version
+            self.config.user_data['deployment_version'] = self.DEPLOYMENT_VERSION
 
     def startup(self):
         if 'role' in self.config:

cm/master.py

@@ -409,6 +409,7 @@ def start(self):
         if not self.initial_cluster_type:  # this can get set by _handle_old_cluster_conf_format
             self.initial_cluster_type = self.app.config.get('cluster_type', None)
             self.userdata_cluster_type = self.app.config.get('initial_cluster_type', None)
+            self.cluster_storage_type = self.app.config.get('cluster_storage_type', None)
             if self.initial_cluster_type is not None:
                 cc_detail = "Configuring a previously existing cluster of type {0}"\
                     .format(self.initial_cluster_type)
@@ -984,6 +985,7 @@ def shutdown(self, sd_apps=True, sd_filesystems=True, sd_instances=True,
         # will persist so no point in poluting the list of buckets)
         if delete_cluster or (self.cluster_storage_type == 'transient' and not rebooting):
             self.delete_cluster()
+            misc.remove(self.app.INSTANCE_PD_FILE)
         self.cluster_status = cluster_status.TERMINATED
         log.info("Cluster %s shut down at %s (uptime: %s). If not done automatically, "
                  "manually terminate the master instance (and any remaining instances "
@@ -2357,6 +2359,7 @@ def create_cluster_config_file(self, file_name='persistent_data-current.yaml', a
             cc['filesystems'] = fss
             cc['services'] = svcs
             cc['cluster_type'] = self.app.manager.initial_cluster_type
+            cc['cluster_storage_type'] = self.app.manager.cluster_storage_type
             cc['cluster_name'] = self.app.config['cluster_name']
             cc['placement'] = self.app.cloud_interface.get_zone()
             cc['machine_image_id'] = self.app.cloud_interface.get_ami()
@@ -2382,10 +2385,15 @@ def store_cluster_config(self):
         In addition, store the local Galaxy configuration files to the cluster's
         bucket (do so only if they are not already there).
         """
+        # Create a cluster configuration file
+        cc_file_name = self.create_cluster_config_file()
         if self.app.manager.initial_cluster_type == 'Test' or \
            self.app.manager.cluster_storage_type == 'transient':
+            # Place the cluster configuration file to a locaiton that lives
+            # across cluster reboots
+            misc.move(cc_file_name, self.app.INSTANCE_PD_FILE)
             log.debug("This is a transient cluster; we do not create a cluster "
-                      "bucket or store cluster configuration for this type.")
+                      "bucket to store cluster configuration for this type.")
             return
         log.debug("Storing cluster configuration to cluster's bucket")
         s3_conn = self.app.cloud_interface.get_s3_connection()
@@ -2397,7 +2405,6 @@ def store_cluster_config(self):
             misc.create_bucket(s3_conn, self.app.config['bucket_cluster'])
         # Save/update the current Galaxy cluster configuration to cluster's
         # bucket
-        cc_file_name = self.create_cluster_config_file()
         misc.save_file_to_bucket(s3_conn, self.app.config['bucket_cluster'],
                                  'persistent_data.yaml', cc_file_name)
         log.debug("Saving current instance boot script (%s) to cluster bucket "

cm/util/misc.py

@@ -491,7 +491,6 @@ def adjust_bucket_acl(s3_conn, bucket_name, users_whose_grant_to_remove):
             # through the list of grants for bucket's users and the list of users
             # whose grant to remove and create a list of bucket grants to keep
             for g in bucket.get_acl().acl.grants:
-                # log.debug("Grant -> permission: %s, user name: %s, grant type: %s" % (g.permission, g.display_name, g.type))
                 # Public (i.e., group) permissions are kept under 'type' field
                 # so check that first
                 if g.type == 'Group' and 'Group' in users_whose_grant_to_remove:
@@ -567,7 +566,8 @@ def file_in_bucket_older_than_local(s3_conn, bucket_name, remote_filename, local
                 local_filename, remote_filename, e))
             return True
     else:
-        log.debug("Checking age of file in bucket (%s) against local file (%s) but file in bucket is None; updating file in bucket."
+        log.debug("Checking age of file in bucket (%s) against local file (%s) "
+                  "but file in bucket is None; updating file in bucket."
                   % (remote_filename, local_filename))
         return True
 
@@ -668,6 +668,7 @@ def delete_file_from_bucket(conn, bucket_name, remote_filename):
                 remote_filename, bucket_name, e))
     return False
 
+
 def update_file_in_bucket(conn, bucket_name, local_filepath):
     """
     Updates file in bucket from its local counterpart.
@@ -689,9 +690,7 @@ def update_file_in_bucket(conn, bucket_name, local_filepath):
                       "cluster bucket '%s' as '%s'" %
                       (local_filepath, bucket_name,
                        filename))
-            save_file_to_bucket(conn,
-                                     bucket_name,
-                                     filename, local_filepath)
+            save_file_to_bucket(conn, bucket_name, filename, local_filepath)
         else:
             log.debug("No instance post start script (%s)" % local_filepath)
     else:
@@ -981,6 +980,34 @@ def make_dir(path, owner=None):
         log.debug("Directory '%s' exists." % path)
 
 
+def move(source, destination):
+    """
+    Move the ``source`` file to ``destination``.
+
+    A convenience wrapper for python's ``shutil.move`` method that simply
+    wrapps the call in a try/catch block.
+    """
+    try:
+        log.debug('Moving file {0} to {1}'.format(source, destination))
+        shutil.move(source, destination)
+    except IOError, ioe:
+        log.error("IOError moving {0} to {1}: {2}".format(source, destination, ioe))
+
+
+def remove(path):
+    """
+    Remove (delete) the file ``path``.
+
+    A convenience wrapper for python's ``os.remove`` method that simply
+    wrapps the call in a try/catch block.
+    """
+    try:
+        log.debug('Removing file {0}'.format(path))
+        os.remove(path)
+    except IOError, ioe:
+        log.error("IOError removing {0}: {2}".format(path, ioe))
+
+
 def add_to_etc_hosts(ip_address, hosts=[]):
     """
     Add a line with the list of ``hosts`` for the given ``ip_address`` to