Cy 2734 cleanup authurl (#112)

* handle this error:

kubernetes-client/python#388

* getting started

need to figure out what is the right way to handle the delete when there is ambiguety in the resources

* finished adding the search for resource ID stuff

* works for single resources

* works - will fix the tests later

* fix testss

* trying that

* trying to make this work in python3

* final changes

Author: EarthmanT

.circleci/config.yml

@@ -19,20 +19,23 @@ executors:
     docker:
       - image: circleci/python:3.6
 
-  centos7:
-    docker:
-      - image: amd64/centos:centos7.3.1611
+  wagon_generator:
+    machine:
+      image: ubuntu-1604:201903-01
 
   cloudify-machine:
-
     machine:
       image: ubuntu-1604:201903-01
 
 
 commands:
   run_unittest:
     steps:
-      - checkout
+      - run:
+          name: "Pull Submodules"
+          command: |
+            git submodule init
+            git submodule update --remote --recursive
       - run:
           name: install tox
           command: pip install --user tox
@@ -71,38 +74,59 @@ commands:
               exit 1
             fi
 
-  generate_wagon:
+  generate_py27_wagon:
     steps:
       - run:
-          name: Install dependencies
-          command: yum -y install python-devel gcc openssl git libxslt-devel libxml2-devel openldap-devel libffi-devel openssl-devel libvirt-devel
-      - run:
-          name: Download pip
-          command: curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
-      - run:
-          name: Install pip
-          command: python get-pip.py
+          name: Create Workspace Build directory.
+          command: mkdir -p workspace/build
       - run:
-          name: Upgrade pip
-          command: pip install --upgrade pip==9.0.1
+          name: Build py27 Wagon
+          command: |
+              git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
+              docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7
+              docker run -v ~/project/:/packaging cloudify-centos-7-wagon-builder
       - run:
-          name: Install virtualenv
-          command: pip install virtualenv==15.1.0
+          name: copy wagon to workspace
+          command: cp *.wgn workspace/build/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - build/*
+
+  generate_py3_wagon:
+    steps:
       - run:
-          name: Init virtualenv
-          command: virtualenv env
+          name: Create Workspace Build directory.
+          command: mkdir -p workspace/build
       - run:
-          name: Install wagon
-          command: pip install wagon==0.3.2
+          name: Build py3 Wagon
+          command: |
+              git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
+              docker build -t cloudify-centos-7-py3-wagon-builder cloudify-wagon-build-containers/centos_7_py3
+              docker run -v ~/project/:/packaging cloudify-centos-7-py3-wagon-builder
       - run:
-          name: many_linux
-          command: echo "manylinux1_compatible = False" > "env/bin/_manylinux.py"
+          name: copy wagon to workspace
+          command: cp *.wgn workspace/build/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - build/*
+
+
+  generate_rhel_py27_wagon:
+    steps:
       - run:
-          name: make workspace
+          name: Create Workspace Build directory.
           command: mkdir -p workspace/build
       - run:
-          name: Create wagon
-          command: source env/bin/activate && wagon create -s . -v -o workspace/build -f -a '--no-cache-dir -c constraints.txt'
+          name: Build RHEL 27 Wagon
+          command: |
+              git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
+              docker build -t cloudify-redhat-7-wagon-builder cloudify-wagon-build-containers/redhat_7 --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD"
+              docker run -v ~/project/:/packaging cloudify-redhat-7-wagon-builder
+      - run:
+          name: copy wagon to workspace
+          command: cp *.wgn workspace/build/
       - persist_to_workspace:
           root: workspace
           paths:
@@ -113,7 +137,11 @@ commands:
       - run:
           name: "Pull Submodules"
           command: |
-            git submodule update --init --recursive
+            git submodule init
+            git submodule update --remote --recursive
+            cd examples/service
+            git submodule init
+            git submodule update --remote --recursive
       - attach_workspace:
           at: workspace
       - run:
@@ -124,13 +152,13 @@ commands:
           command: pip install -U cloudify==5.0.0 pytest==4.6.3 pyyaml==3.10
       - run:
           name: download manager docker image
-          command: wget http://repository.cloudifysource.org/cloudify/5.0.0/ga-release/cloudify-docker-manager-5.0.0.tar
+          command: wget http://repository.cloudifysource.org/cloudify/5.0.5/ga-release/cloudify-docker-manager-5.0.5.tar
       - run:
           name: load docker image
-          command: docker load -i cloudify-docker-manager-5.0.0.tar
+          command: docker load -i cloudify-docker-manager-5.0.5.tar
       - run:
           name: retain space by dumping the tar
-          command: rm cloudify-docker-manager-5.0.0.tar
+          command: rm cloudify-docker-manager-5.0.5.tar
       - run:
           name: show images
           command: docker images
@@ -179,11 +207,13 @@ jobs:
   unittests_py27:
     executor: py27
     steps:
+      - checkout
       - run_unittest
 
   unittests_py36:
     executor: py36
     steps:
+      - checkout
       - run_unittest
 
   py3_compat:
@@ -193,10 +223,18 @@ jobs:
       - check_py3_compat
 
   wagon:
-    executor: centos7
+    executor: wagon_generator
+    steps:
+      - checkout
+      - generate_py27_wagon
+      # - generate_py3_wagon  # TODO: When we have Python3 Manager Integration Test.
+
+  rhel_wagon:
+    executor: wagon_generator
     steps:
       - checkout
-      - generate_wagon
+      - generate_rhel_py27_wagon
+
 
   integration_tests:
     executor: cloudify-machine
@@ -216,28 +254,32 @@ jobs:
 workflows:
   version: 2
   tests:
-    jobs: &all_jobs
+    jobs:
       - py3_compat
       - unittests_py27
       - unittests_py36
       - wagon:
           filters:
             branches:
-              only: /([0-9\.]*\-build|master|dev)/
+              only: /([0-9\.]*\-build|master)/
+      - rhel_wagon:
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master)/
       - integration_tests:
           requires:
-            - py3_compat
-            - unittests_py27
-            - unittests_py36
             - wagon
+            - rhel_wagon
           filters:
             branches:
-              only: /([0-9\.]*\-build|master|dev)/
+              only: /([0-9\.]*\-build|master)/
       - release:
           filters:
             branches:
               only: /master/
           requires:
+            - wagon
+            - rhel_wagon
             - integration_tests
   nightly:
     triggers:
@@ -247,4 +289,22 @@ workflows:
           branches:
             only:
               - master
-    jobs: *all_jobs
+    jobs:
+      - py3_compat
+      - unittests_py27
+      - unittests_py36
+      - wagon:
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master)/
+      - rhel_wagon:
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master)/
+      - integration_tests:
+          requires:
+            - wagon
+            - rhel_wagon
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master)/

CHANGELOG.txt

@@ -96,4 +96,8 @@
   didnt derived from ResourceBase type.
 
 2.7.2:
-  - Support Python 3
+  - Support Python 3
+
+2.8.0:
+  - Remove trailing slash in URL.
+  - Support modifying a resource type and number during update workflow.

cloudify_kubernetes/decorators.py

@@ -21,12 +21,14 @@
 )
 from cloudify.decorators import operation
 
+from ._compat import text_type
 from .utils import (generate_traceback_exception,
                     retrieve_path,
                     get_node,
                     get_instance,
                     NODE_PROPERTY_FILE,
-                    NODE_PROPERTY_FILE_RESOURCE_PATH)
+                    NODE_PROPERTY_FILE_RESOURCE_PATH,
+                    INSTANCE_RUNTIME_PROPERTY_KUBERNETES)
 from .k8s import (CloudifyKubernetesClient,
                   KubernetesApiAuthenticationVariants,
                   KubernetesApiConfigurationVariants,
@@ -41,7 +43,6 @@
 RELATIONSHIP_TYPE_MANAGED_BY_MASTER = (
     'cloudify.kubernetes.relationships.managed_by_master'
 )
-INSTANCE_RUNTIME_PROPERTY_KUBERNETES = 'kubernetes'
 
 
 def _retrieve_master(resource_instance):
@@ -74,8 +75,10 @@ def _retrieve_property(_ctx, property_name):
 
 
 def _multidefinition_resource_task(task, definitions, kwargs,
-                                   retrieve_mapping, use_existing=False,
-                                   cleanup_runtime_properties=False):
+                                   retrieve_mapping,
+                                   use_existing=False,
+                                   cleanup_runtime_properties=False,
+                                   resource_state_function=None):
     curr_num = 0
     # we have several definitions (not one!)
     multicalls = len(definitions) > 1
@@ -85,7 +88,7 @@ def _multidefinition_resource_task(task, definitions, kwargs,
         # save original path only in case multicalls
         origin_path = kwargs[
             NODE_PROPERTY_FILE].get(NODE_PROPERTY_FILE_RESOURCE_PATH)
-    elif NODE_PROPERTY_FILE in ctx.node.properties and multicalls:
+    elif NODE_PROPERTY_FILE in ctx.node.properties:
         # copy origin file name to kwargs
         kwargs[NODE_PROPERTY_FILE] = ctx.node.properties[NODE_PROPERTY_FILE]
         # save origin path
@@ -101,30 +104,46 @@ def _multidefinition_resource_task(task, definitions, kwargs,
             kwargs[NODE_PROPERTY_FILE][NODE_PROPERTY_FILE_RESOURCE_PATH] = (
                 "{name}#{curr_num}".format(
                     name=origin_path,
-                    curr_num=str(curr_num)
+                    curr_num=text_type(curr_num)
                 ))
             curr_num += 1
+
         # check current state
         path = retrieve_path(kwargs)
-        if path:
+        resource_id = definition.metadata.get('name')
+        if path and resource_state_function and resource_id:
+            current_state = resource_state_function(
+                resource_id=resource_id, **kwargs)
+        elif resource_state_function and resource_id:
+            current_state = resource_state_function(
+                resource_id=resource_id, **kwargs)
+        elif path:
             current_state = ctx.instance.runtime_properties.get(
                 INSTANCE_RUNTIME_PROPERTY_KUBERNETES, {}).get(path)
         else:
             current_state = ctx.instance.runtime_properties.get(
                 INSTANCE_RUNTIME_PROPERTY_KUBERNETES)
-        # ignore prexisted state
+
+        # ignore pre-existing state
         if not use_existing and current_state:
-            ctx.logger.info("Ignore existing object state")
-            continue
+            ctx.logger.info(
+                "The resource {0} unexpectedly exists. "
+                "Not executing operation.".format(definition.to_dict()))
+            ctx.instance.runtime_properties['__perform_task'] = False
         # ignore if we dont have any object yet
-        if use_existing and not current_state:
-            ctx.logger.info("Ignore unexisted object state")
-            continue
-        # finally run
+        elif use_existing and not current_state:
+            ctx.logger.info(
+                "Expected resource {0} to exist, but it does not. "
+                "Not executing operation.".format(definition.to_dict()))
+            ctx.instance.runtime_properties['__perform_task'] = False
+        else:
+            ctx.instance.runtime_properties['__perform_task'] = True
         task(**kwargs)
+        del ctx.instance.runtime_properties['__perform_task']
         # cleanup after successful run
         if current_state and cleanup_runtime_properties:
-            if path:
+            if path and path in ctx.instance.runtime_properties[
+                    INSTANCE_RUNTIME_PROPERTY_KUBERNETES]:
                 del ctx.instance.runtime_properties[
                     INSTANCE_RUNTIME_PROPERTY_KUBERNETES][path]
             else:
@@ -143,9 +162,11 @@ def _multidefinition_resource_task(task, definitions, kwargs,
 
 def resource_task(retrieve_resource_definition=None,
                   retrieve_resources_definitions=None,
-                  retrieve_mapping=None, use_existing=False,
-                  cleanup_runtime_properties=False):
-    def decorator(task, **kwargs):
+                  retrieve_mapping=None,
+                  use_existing=False,
+                  cleanup_runtime_properties=False,
+                  resource_state_function=None):
+    def decorator(task, **_):
         def wrapper(**kwargs):
             try:
                 definitions = []
@@ -159,7 +180,9 @@ def wrapper(**kwargs):
                 _multidefinition_resource_task(
                     task, definitions, kwargs, retrieve_mapping,
                     use_existing=use_existing,
-                    cleanup_runtime_properties=cleanup_runtime_properties)
+                    cleanup_runtime_properties=cleanup_runtime_properties,
+                    resource_state_function=resource_state_function
+                )
             except (KuberentesMappingNotFoundError,
                     KuberentesInvalidPayloadClassError,
                     KuberentesInvalidApiClassError,
@@ -202,7 +225,7 @@ def wrapper(**kwargs):
     return decorator
 
 
-def with_kubernetes_client(function):
+def with_kubernetes_client(fn):
     def wrapper(**kwargs):
         configuration_property = _retrieve_property(
             ctx,
@@ -228,7 +251,7 @@ def wrapper(**kwargs):
                 )
             )
 
-            function(**kwargs)
+            fn(**kwargs)
         except KuberentesApiInitializationFailedError as e:
             error_traceback = generate_traceback_exception()
             ctx.logger.error(