diff options
| author | Kenny Woodson <kwoodson@redhat.com> | 2016-03-31 16:29:20 -0400 | 
|---|---|---|
| committer | Kenny Woodson <kwoodson@redhat.com> | 2016-04-04 12:27:37 -0400 | 
| commit | fcf8e6f1af68797e4a54efb22a47095fc4e3bedf (patch) | |
| tree | 4339ee6a75490cf5cba20a278545d9932a2b0bab | |
| parent | 9451e288a77ed09cad19ef4fe27479f5b808277f (diff) | |
| download | openshift-fcf8e6f1af68797e4a54efb22a47095fc4e3bedf.tar.gz openshift-fcf8e6f1af68797e4a54efb22a47095fc4e3bedf.tar.bz2 openshift-fcf8e6f1af68797e4a54efb22a47095fc4e3bedf.tar.xz openshift-fcf8e6f1af68797e4a54efb22a47095fc4e3bedf.zip | |
Yedit enhancements
| -rw-r--r-- | roles/lib_openshift_api/build/ansible/edit.py | 77 | ||||
| -rwxr-xr-x | roles/lib_openshift_api/build/generate.py | 12 | ||||
| -rw-r--r-- | roles/lib_openshift_api/build/src/base.py | 55 | ||||
| -rw-r--r-- | roles/lib_openshift_api/build/src/edit.py | 49 | ||||
| -rwxr-xr-x | roles/lib_openshift_api/build/test/edit.yml | 53 | ||||
| -rw-r--r-- | roles/lib_openshift_api/build/test/files/dc.yml | 9 | ||||
| -rw-r--r-- | roles/lib_openshift_api/library/oc_edit.py | 604 | ||||
| -rw-r--r-- | roles/lib_openshift_api/library/oc_obj.py | 194 | ||||
| -rw-r--r-- | roles/lib_openshift_api/library/oc_secret.py | 194 | ||||
| -rw-r--r-- | roles/lib_yaml_editor/build/ansible/yedit.py | 4 | ||||
| -rwxr-xr-x | roles/lib_yaml_editor/build/generate.py | 9 | ||||
| -rw-r--r-- | roles/lib_yaml_editor/build/src/base.py | 8 | ||||
| -rw-r--r-- | roles/lib_yaml_editor/build/src/yedit.py | 139 | ||||
| -rw-r--r-- | roles/lib_yaml_editor/build/test/foo.yml | 2 | ||||
| -rw-r--r-- | roles/lib_yaml_editor/library/yedit.py | 151 | ||||
| -rw-r--r-- | test/env-setup | 2 | ||||
| -rwxr-xr-x | test/units/yedit_test.py | 68 | 
17 files changed, 1371 insertions, 259 deletions
| diff --git a/roles/lib_openshift_api/build/ansible/edit.py b/roles/lib_openshift_api/build/ansible/edit.py new file mode 100644 index 000000000..d48bc7a01 --- /dev/null +++ b/roles/lib_openshift_api/build/ansible/edit.py @@ -0,0 +1,77 @@ +# pylint: skip-file + +def main(): +    ''' +    ansible oc module for services +    ''' + +    module = AnsibleModule( +        argument_spec=dict( +            kubeconfig=dict(default='/etc/origin/master/admin.kubeconfig', type='str'), +            state=dict(default='present', type='str', +                       choices=['present']), +            debug=dict(default=False, type='bool'), +            namespace=dict(default='default', type='str'), +            name=dict(default=None, type='str'), +            kind=dict(required=True, +                      type='str', +                      choices=['dc', 'deploymentconfig', +                               'svc', 'service', +                               'secret', +                              ]), +            file_name=dict(default=None, type='str'), +            file_format=dict(default='yaml', type='str'), +            content=dict(default=None, type='dict'), +            force=dict(default=False, type='bool'), +        ), +        supports_check_mode=True, +    ) +    ocedit = Edit(module.params['kind'], +                  module.params['namespace'], +                  module.params['name'], +                  kubeconfig=module.params['kubeconfig'], +                  verbose=module.params['debug']) + +    state = module.params['state'] + +    api_rval = ocedit.get() + +    ######## +    # Create +    ######## +    if not Utils.exists(api_rval['results'], module.params['name']): +        module.fail_json(msg=api_rval) + +    ######## +    # Update +    ######## +    api_rval = ocedit.update(module.params['file_name'], +                             module.params['content'], +                             module.params['force'], +                             module.params['file_format']) + + +    if api_rval['returncode'] != 0: +        module.fail_json(msg=api_rval) + +    if api_rval.has_key('updated') and not api_rval['updated']: +        module.exit_json(changed=False, results=api_rval, state="present") + +    # return the created object +    api_rval = ocedit.get() + +    if api_rval['returncode'] != 0: +        module.fail_json(msg=api_rval) + +    module.exit_json(changed=True, results=api_rval, state="present") + +    module.exit_json(failed=True, +                     changed=False, +                     results='Unknown state passed. %s' % state, +                     state="unknown") + +# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import, locally-disabled +# import module snippets.  This are required +from ansible.module_utils.basic import * + +main() diff --git a/roles/lib_openshift_api/build/generate.py b/roles/lib_openshift_api/build/generate.py index 877ca1766..cf3f61d2c 100755 --- a/roles/lib_openshift_api/build/generate.py +++ b/roles/lib_openshift_api/build/generate.py @@ -15,6 +15,7 @@ GEN_STR = "#!/usr/bin/env python\n"                                   + \            "#   | |) | (_) | | .` | (_) || |   | _|| |) | |  | |\n"   + \            "#   |___/ \___/  |_|\_|\___/ |_|   |___|___/___| |_|\n" +OPENSHIFT_ANSIBLE_PATH = os.path.dirname(os.path.realpath(__file__))  FILES = {'oc_obj.py': ['src/base.py', @@ -27,18 +28,23 @@ FILES = {'oc_obj.py': ['src/base.py',                            'src/secret.py',                            'ansible/secret.py',                           ], +         'oc_edit.py': ['src/base.py', +                        '../../lib_yaml_editor/build/src/yedit.py', +                        'src/edit.py', +                        'ansible/edit.py', +                       ],          }  def main():      ''' combine the necessary files to create the ansible module ''' -    openshift_ansible = ('../library/') +    library = os.path.join(OPENSHIFT_ANSIBLE_PATH, '..', 'library/')      for fname, parts in FILES.items(): -        with open(os.path.join(openshift_ansible, fname), 'w') as afd: +        with open(os.path.join(library, fname), 'w') as afd:              afd.seek(0)              afd.write(GEN_STR)              for fpart in parts: -                with open(fpart) as pfd: +                with open(os.path.join(OPENSHIFT_ANSIBLE_PATH, fpart)) as pfd:                      # first line is pylint disable so skip it                      for idx, line in enumerate(pfd):                          if idx == 0 and 'skip-file' in line: diff --git a/roles/lib_openshift_api/build/src/base.py b/roles/lib_openshift_api/build/src/base.py index 31c102e5d..66831c4e2 100644 --- a/roles/lib_openshift_api/build/src/base.py +++ b/roles/lib_openshift_api/build/src/base.py @@ -8,7 +8,15 @@ import json  import os  import shutil  import subprocess +import re +  import yaml +# This is here because of a bug that causes yaml +# to incorrectly handle timezone info on timestamps +def timestamp_constructor(_, node): +    '''return timestamps as strings''' +    return str(node.value) +yaml.add_constructor(u'tag:yaml.org,2002:timestamp', timestamp_constructor)  # pylint: disable=too-few-public-methods  class OpenShiftCLI(object): @@ -32,8 +40,14 @@ class OpenShiftCLI(object):          fname = '/tmp/%s' % rname          yed = Yedit(fname, res['results'][0]) +        changes = []          for key, value in content.items(): -            yed.put(key, value) +            changes.append(yed.put(key, value)) + +        if any([not change[0] for change in changes]): +            return {'returncode': 0, 'updated': False} + +        yed.write()          atexit.register(Utils.cleanup, [fname]) @@ -76,7 +90,9 @@ class OpenShiftCLI(object):          cmds = ['/usr/bin/oc']          cmds.extend(cmd) +        rval = {}          results = '' +        err = None          if self.verbose:              print ' '.join(cmds) @@ -85,27 +101,42 @@ class OpenShiftCLI(object):                                  stdout=subprocess.PIPE,                                  stderr=subprocess.PIPE,                                  env={'KUBECONFIG': self.kubeconfig}) +          proc.wait() +        stdout = proc.stdout.read() +        stderr = proc.stderr.read() + +        rval = {"returncode": proc.returncode, +                "results": results, +               } +          if proc.returncode == 0:              if output:                  try: -                    results = json.loads(proc.stdout.read()) +                    rval['results'] = json.loads(stdout)                  except ValueError as err:                      if "No JSON object could be decoded" in err.message: -                        results = err.message +                        err = err.message              if self.verbose: -                print proc.stderr.read() -                print results +                print stdout +                print stderr                  print -            return {"returncode": proc.returncode, "results": results} +            if err: +                rval.update({"err": err, +                             "stderr": stderr, +                             "stdout": stdout, +                             "cmd": cmds +                            }) -        return {"returncode": proc.returncode, -                "stderr": proc.stderr.read(), -                "stdout": proc.stdout.read(), -                "results": {} -               } +        else: +            rval.update({"stderr": stderr, +                         "stdout": stdout, +                         "results": {}, +                        }) + +        return rval  class Utils(object):      ''' utilities for openshiftcli modules ''' @@ -179,7 +210,7 @@ class Utils(object):              contents = sfd.read()          if sfile_type == 'yaml': -            contents = yaml.load(contents) +            contents = yaml.safe_load(contents)          elif sfile_type == 'json':              contents = json.loads(contents) diff --git a/roles/lib_openshift_api/build/src/edit.py b/roles/lib_openshift_api/build/src/edit.py new file mode 100644 index 000000000..7020ace47 --- /dev/null +++ b/roles/lib_openshift_api/build/src/edit.py @@ -0,0 +1,49 @@ +# pylint: skip-file + +class Edit(OpenShiftCLI): +    ''' Class to wrap the oc command line tools +    ''' +    # pylint: disable=too-many-arguments +    def __init__(self, +                 kind, +                 namespace, +                 resource_name=None, +                 kubeconfig='/etc/origin/master/admin.kubeconfig', +                 verbose=False): +        ''' Constructor for OpenshiftOC ''' +        super(Edit, self).__init__(namespace, kubeconfig) +        self.namespace = namespace +        self.kind = kind +        self.name = resource_name +        self.kubeconfig = kubeconfig +        self.verbose = verbose + +    def get(self): +        '''return a secret by name ''' +        return self._get(self.kind, self.name) + +    def update(self, file_name, content, force=False, content_type='yaml'): +        '''run update ''' +        if file_name: +            if content_type == 'yaml': +                data = yaml.load(open(file_name)) +            elif content_type == 'json': +                data = json.loads(open(file_name).read()) + +            changes = [] +            yed = Yedit(file_name, data) +            for key, value in content.items(): +                changes.append(yed.put(key, value)) + +            if any([not change[0] for change in changes]): +                return {'returncode': 0, 'updated': False} + +            yed.write() + +            atexit.register(Utils.cleanup, [file_name]) + +            return self._replace(file_name, force=force) + +        return self._replace_content(self.kind, self.name, content, force=force) + + diff --git a/roles/lib_openshift_api/build/test/edit.yml b/roles/lib_openshift_api/build/test/edit.yml new file mode 100755 index 000000000..9aa01303a --- /dev/null +++ b/roles/lib_openshift_api/build/test/edit.yml @@ -0,0 +1,53 @@ +#!/usr/bin/ansible-playbook +--- +- hosts: "oo_clusterid_mwoodson:&oo_version_3:&oo_master_primary" +  gather_facts: no +  user: root + +  post_tasks: +  - copy: +      dest: "/tmp/{{ item }}" +      src: "files/{{ item }}" +    with_items: +    - dc.yml + +  - name: present dc +    oc_edit: +      kind: dc +      namespace: default +      name: router +      content: +        spec.template.spec.containers[0].ports[0].containerPort: 80 +        spec.template.spec.containers[0].ports[0].hostPort: 80 +    register: dcout + +  - debug: +      var: dcout + +  - name: present dc +    oc_edit: +      kind: dc +      namespace: default +      name: router +      content: +        spec.template.spec.containers[0].ports[0].containerPort: 81 +        spec.template.spec.containers[0].ports[0].hostPort: 81 +      file_format: yaml +    register: dcout + +  - debug: +      var: dcout + +  - name: present dc +    oc_edit: +      kind: dc +      namespace: default +      name: router +      content: +        spec.template.spec.containers[0].ports[0].containerPort: 80 +        spec.template.spec.containers[0].ports[0].hostPort: 80 +      file_format: yaml +    register: dcout + +  - debug: +      var: dcout diff --git a/roles/lib_openshift_api/build/test/files/dc.yml b/roles/lib_openshift_api/build/test/files/dc.yml index 7992c90dd..24f690ef4 100644 --- a/roles/lib_openshift_api/build/test/files/dc.yml +++ b/roles/lib_openshift_api/build/test/files/dc.yml @@ -1,14 +1,14 @@  apiVersion: v1  kind: DeploymentConfig  metadata: -  creationTimestamp: 2016-03-18T19:47:45Z +  creationTimestamp: 2016-04-01T15:23:29Z    labels:      router: router    name: router    namespace: default -  resourceVersion: "84016" +  resourceVersion: "1338477"    selfLink: /oapi/v1/namespaces/default/deploymentconfigs/router -  uid: 48f8b9d9-ed42-11e5-9903-0a9a9d4e7f2b +  uid: b00c7eba-f81d-11e5-809b-0a581f893e3f  spec:    replicas: 2    selector: @@ -117,5 +117,4 @@ status:    details:      causes:      - type: ConfigChange -  latestVersion: 1 - +  latestVersion: 12 diff --git a/roles/lib_openshift_api/library/oc_edit.py b/roles/lib_openshift_api/library/oc_edit.py new file mode 100644 index 000000000..0135dba63 --- /dev/null +++ b/roles/lib_openshift_api/library/oc_edit.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python +#     ___ ___ _  _ ___ ___    _ _____ ___ ___ +#    / __| __| \| | __| _ \  /_\_   _| __|   \ +#   | (_ | _|| .` | _||   / / _ \| | | _|| |) | +#    \___|___|_|\_|___|_|_\/_/_\_\_|_|___|___/_ _____ +#   |   \ / _ \  | \| |/ _ \_   _| | __|   \_ _|_   _| +#   | |) | (_) | | .` | (_) || |   | _|| |) | |  | | +#   |___/ \___/  |_|\_|\___/ |_|   |___|___/___| |_| +''' +   OpenShiftCLI class that wraps the oc commands in a subprocess +''' + +import atexit +import json +import os +import shutil +import subprocess +import re + +import yaml +# This is here because of a bug that causes yaml +# to incorrectly handle timezone info on timestamps +def timestamp_constructor(_, node): +    '''return timestamps as strings''' +    return str(node.value) +yaml.add_constructor(u'tag:yaml.org,2002:timestamp', timestamp_constructor) + +# pylint: disable=too-few-public-methods +class OpenShiftCLI(object): +    ''' Class to wrap the oc command line tools ''' +    def __init__(self, +                 namespace, +                 kubeconfig='/etc/origin/master/admin.kubeconfig', +                 verbose=False): +        ''' Constructor for OpenshiftOC ''' +        self.namespace = namespace +        self.verbose = verbose +        self.kubeconfig = kubeconfig + +    # Pylint allows only 5 arguments to be passed. +    # pylint: disable=too-many-arguments +    def _replace_content(self, resource, rname, content, force=False): +        ''' replace the current object with the content ''' +        res = self._get(resource, rname) +        if not res['results']: +            return res + +        fname = '/tmp/%s' % rname +        yed = Yedit(fname, res['results'][0]) +        changes = [] +        for key, value in content.items(): +            changes.append(yed.put(key, value)) + +        if any([not change[0] for change in changes]): +            return {'returncode': 0, 'updated': False} + +        yed.write() + +        atexit.register(Utils.cleanup, [fname]) + +        return self._replace(fname, force) + +    def _replace(self, fname, force=False): +        '''return all pods ''' +        cmd = ['-n', self.namespace, 'replace', '-f', fname] +        if force: +            cmd.append('--force') +        return self.oc_cmd(cmd) + +    def _create(self, fname): +        '''return all pods ''' +        return self.oc_cmd(['create', '-f', fname, '-n', self.namespace]) + +    def _delete(self, resource, rname): +        '''return all pods ''' +        return self.oc_cmd(['delete', resource, rname, '-n', self.namespace]) + +    def _get(self, resource, rname=None): +        '''return a secret by name ''' +        cmd = ['get', resource, '-o', 'json', '-n', self.namespace] +        if rname: +            cmd.append(rname) + +        rval = self.oc_cmd(cmd, output=True) + +        # Ensure results are retuned in an array +        if rval.has_key('items'): +            rval['results'] = rval['items'] +        elif not isinstance(rval['results'], list): +            rval['results'] = [rval['results']] + +        return rval + +    def oc_cmd(self, cmd, output=False): +        '''Base command for oc ''' +        #cmds = ['/usr/bin/oc', '--config', self.kubeconfig] +        cmds = ['/usr/bin/oc'] +        cmds.extend(cmd) + +        rval = {} +        results = '' +        err = None + +        if self.verbose: +            print ' '.join(cmds) + +        proc = subprocess.Popen(cmds, +                                stdout=subprocess.PIPE, +                                stderr=subprocess.PIPE, +                                env={'KUBECONFIG': self.kubeconfig}) + +        proc.wait() +        stdout = proc.stdout.read() +        stderr = proc.stderr.read() + +        rval = {"returncode": proc.returncode, +                "results": results, +               } + +        if proc.returncode == 0: +            if output: +                try: +                    rval['results'] = json.loads(stdout) +                except ValueError as err: +                    if "No JSON object could be decoded" in err.message: +                        err = err.message + +            if self.verbose: +                print stdout +                print stderr +                print + +            if err: +                rval.update({"err": err, +                             "stderr": stderr, +                             "stdout": stdout, +                             "cmd": cmds +                            }) + +        else: +            rval.update({"stderr": stderr, +                         "stdout": stdout, +                         "results": {}, +                        }) + +        return rval + +class Utils(object): +    ''' utilities for openshiftcli modules ''' +    @staticmethod +    def create_file(rname, data, ftype=None): +        ''' create a file in tmp with name and contents''' +        path = os.path.join('/tmp', rname) +        with open(path, 'w') as fds: +            if ftype == 'yaml': +                fds.write(yaml.safe_dump(data, default_flow_style=False)) + +            elif ftype == 'json': +                fds.write(json.dumps(data)) +            else: +                fds.write(data) + +        # Register cleanup when module is done +        atexit.register(Utils.cleanup, [path]) +        return path + +    @staticmethod +    def create_files_from_contents(data): +        '''Turn an array of dict: filename, content into a files array''' +        files = [] + +        for sfile in data: +            path = Utils.create_file(sfile['path'], sfile['content']) +            files.append(path) + +        return files + +    @staticmethod +    def cleanup(files): +        '''Clean up on exit ''' +        for sfile in files: +            if os.path.exists(sfile): +                if os.path.isdir(sfile): +                    shutil.rmtree(sfile) +                elif os.path.isfile(sfile): +                    os.remove(sfile) + + +    @staticmethod +    def exists(results, _name): +        ''' Check to see if the results include the name ''' +        if not results: +            return False + + +        if Utils.find_result(results, _name): +            return True + +        return False + +    @staticmethod +    def find_result(results, _name): +        ''' Find the specified result by name''' +        rval = None +        for result in results: +            if result.has_key('metadata') and result['metadata']['name'] == _name: +                rval = result +                break + +        return rval + +    @staticmethod +    def get_resource_file(sfile, sfile_type='yaml'): +        ''' return the service file  ''' +        contents = None +        with open(sfile) as sfd: +            contents = sfd.read() + +        if sfile_type == 'yaml': +            contents = yaml.safe_load(contents) +        elif sfile_type == 'json': +            contents = json.loads(contents) + +        return contents + +    # Disabling too-many-branches.  This is a yaml dictionary comparison function +    # pylint: disable=too-many-branches,too-many-return-statements +    @staticmethod +    def check_def_equal(user_def, result_def, debug=False): +        ''' Given a user defined definition, compare it with the results given back by our query.  ''' + +        # Currently these values are autogenerated and we do not need to check them +        skip = ['metadata', 'status'] + +        for key, value in result_def.items(): +            if key in skip: +                continue + +            # Both are lists +            if isinstance(value, list): +                if not isinstance(user_def[key], list): +                    return False + +                # lists should be identical +                if value != user_def[key]: +                    return False + +            # recurse on a dictionary +            elif isinstance(value, dict): +                if not isinstance(user_def[key], dict): +                    if debug: +                        print "dict returned false not instance of dict" +                    return False + +                # before passing ensure keys match +                api_values = set(value.keys()) - set(skip) +                user_values = set(user_def[key].keys()) - set(skip) +                if api_values != user_values: +                    if debug: +                        print api_values +                        print user_values +                        print "keys are not equal in dict" +                    return False + +                result = Utils.check_def_equal(user_def[key], value, debug=debug) +                if not result: +                    if debug: +                        print "dict returned false" +                    return False + +            # Verify each key, value pair is the same +            else: +                if not user_def.has_key(key) or value != user_def[key]: +                    if debug: +                        print "value not equal; user_def does not have key" +                        print value +                        print user_def[key] +                    return False + +        return True + +class YeditException(Exception): +    ''' Exception class for Yedit ''' +    pass + +class Yedit(object): +    ''' Class to modify yaml files ''' +    re_valid_key = r"(((\[-?\d+\])|(\w+)).?)+$" +    re_key = r"(?:\[(-?\d+)\])|(\w+)" + +    def __init__(self, filename=None, content=None, content_type='yaml'): +        self.content = content +        self.filename = filename +        self.__yaml_dict = content +        self.content_type = content_type +        if self.filename and not self.content: +            self.load(content_type=self.content_type) + +    @property +    def yaml_dict(self): +        ''' getter method for yaml_dict ''' +        return self.__yaml_dict + +    @yaml_dict.setter +    def yaml_dict(self, value): +        ''' setter method for yaml_dict ''' +        self.__yaml_dict = value + +    @staticmethod +    def remove_entry(data, key): +        ''' remove data at location key ''' +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)] +            else: +                return None + +        # process last index for remove +        # expected list entry +        if key_indexes[-1][0]: +            if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +                del data[int(key_indexes[-1][0])] + +        # expected dict entry +        elif key_indexes[-1][1]: +            if isinstance(data, dict): +                del data[key_indexes[-1][1]] + +    @staticmethod +    def add_entry(data, key, item=None): +        ''' Get an item from a dictionary with key notation a.b.c +            d = {'a': {'b': 'c'}}} +            key = a.b +            return c +        ''' +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        curr_data = data + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key: +                if isinstance(data, dict) and data.has_key(dict_key): +                    data = data[dict_key] +                    continue + +                data[dict_key] = {} +                data = data[dict_key] + +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)] +            else: +                return None + +        # process last index for add +        # expected list entry +        if key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +            data[int(key_indexes[-1][0])] = item + +        # expected dict entry +        elif key_indexes[-1][1] and isinstance(data, dict): +            data[key_indexes[-1][1]] = item + +        return curr_data + +    @staticmethod +    def get_entry(data, key): +        ''' Get an item from a dictionary with key notation a.b.c +            d = {'a': {'b': 'c'}}} +            key = a.b +            return c +        ''' +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)] +            else: +                return None + +        return data + +    def write(self): +        ''' write to file ''' +        if not self.filename: +            raise YeditException('Please specify a filename.') + +        with open(self.filename, 'w') as yfd: +            yfd.write(yaml.safe_dump(self.yaml_dict, default_flow_style=False)) + +    def read(self): +        ''' write to file ''' +        # check if it exists +        if not self.exists(): +            return None + +        contents = None +        with open(self.filename) as yfd: +            contents = yfd.read() + +        return contents + +    def exists(self): +        ''' return whether file exists ''' +        if os.path.exists(self.filename): +            return True + +        return False + +    def load(self, content_type='yaml'): +        ''' return yaml file ''' +        contents = self.read() + +        if not contents: +            return None + +        # check if it is yaml +        try: +            if content_type == 'yaml': +                self.yaml_dict = yaml.load(contents) +            elif content_type == 'json': +                self.yaml_dict = json.loads(contents) +        except yaml.YAMLError as _: +            # Error loading yaml or json +            return None + +        return self.yaml_dict + +    def get(self, key): +        ''' get a specified key''' +        try: +            entry = Yedit.get_entry(self.yaml_dict, key) +        except KeyError as _: +            entry = None + +        return entry + +    def delete(self, key): +        ''' put key, value into a yaml file ''' +        try: +            entry = Yedit.get_entry(self.yaml_dict, key) +        except KeyError as _: +            entry = None +        if not entry: +            return  (False, self.yaml_dict) + +        Yedit.remove_entry(self.yaml_dict, key) +        return (True, self.yaml_dict) + +    def put(self, key, value): +        ''' put key, value into a yaml file ''' +        try: +            entry = Yedit.get_entry(self.yaml_dict, key) +        except KeyError as _: +            entry = None + +        if entry == value: +            return (False, self.yaml_dict) + +        Yedit.add_entry(self.yaml_dict, key, value) +        return (True, self.yaml_dict) + +    def create(self, key, value): +        ''' create the file ''' +        if not self.exists(): +            self.yaml_dict = {key: value} +            return (True, self.yaml_dict) + +        return (False, self.yaml_dict) + +class Edit(OpenShiftCLI): +    ''' Class to wrap the oc command line tools +    ''' +    # pylint: disable=too-many-arguments +    def __init__(self, +                 kind, +                 namespace, +                 resource_name=None, +                 kubeconfig='/etc/origin/master/admin.kubeconfig', +                 verbose=False): +        ''' Constructor for OpenshiftOC ''' +        super(Edit, self).__init__(namespace, kubeconfig) +        self.namespace = namespace +        self.kind = kind +        self.name = resource_name +        self.kubeconfig = kubeconfig +        self.verbose = verbose + +    def get(self): +        '''return a secret by name ''' +        return self._get(self.kind, self.name) + +    def update(self, file_name, content, force=False, content_type='yaml'): +        '''run update ''' +        if file_name: +            if content_type == 'yaml': +                data = yaml.load(open(file_name)) +            elif content_type == 'json': +                data = json.loads(open(file_name).read()) + +            changes = [] +            yed = Yedit(file_name, data) +            for key, value in content.items(): +                changes.append(yed.put(key, value)) + +            if any([not change[0] for change in changes]): +                return {'returncode': 0, 'updated': False} + +            yed.write() + +            atexit.register(Utils.cleanup, [file_name]) + +            return self._replace(file_name, force=force) + +        return self._replace_content(self.kind, self.name, content, force=force) + + + +def main(): +    ''' +    ansible oc module for services +    ''' + +    module = AnsibleModule( +        argument_spec=dict( +            kubeconfig=dict(default='/etc/origin/master/admin.kubeconfig', type='str'), +            state=dict(default='present', type='str', +                       choices=['present']), +            debug=dict(default=False, type='bool'), +            namespace=dict(default='default', type='str'), +            name=dict(default=None, type='str'), +            kind=dict(required=True, +                      type='str', +                      choices=['dc', 'deploymentconfig', +                               'svc', 'service', +                               'secret', +                              ]), +            file_name=dict(default=None, type='str'), +            file_format=dict(default='yaml', type='str'), +            content=dict(default=None, type='dict'), +            force=dict(default=False, type='bool'), +        ), +        supports_check_mode=True, +    ) +    ocedit = Edit(module.params['kind'], +                  module.params['namespace'], +                  module.params['name'], +                  kubeconfig=module.params['kubeconfig'], +                  verbose=module.params['debug']) + +    state = module.params['state'] + +    api_rval = ocedit.get() + +    ######## +    # Create +    ######## +    if not Utils.exists(api_rval['results'], module.params['name']): +        module.fail_json(msg=api_rval) + +    ######## +    # Update +    ######## +    api_rval = ocedit.update(module.params['file_name'], +                             module.params['content'], +                             module.params['force'], +                             module.params['file_format']) + + +    if api_rval['returncode'] != 0: +        module.fail_json(msg=api_rval) + +    if api_rval.has_key('updated') and not api_rval['updated']: +        module.exit_json(changed=False, results=api_rval, state="present") + +    # return the created object +    api_rval = ocedit.get() + +    if api_rval['returncode'] != 0: +        module.fail_json(msg=api_rval) + +    module.exit_json(changed=True, results=api_rval, state="present") + +    module.exit_json(failed=True, +                     changed=False, +                     results='Unknown state passed. %s' % state, +                     state="unknown") + +# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import, locally-disabled +# import module snippets.  This are required +from ansible.module_utils.basic import * + +main() diff --git a/roles/lib_openshift_api/library/oc_obj.py b/roles/lib_openshift_api/library/oc_obj.py index fa31416c0..27135e02e 100644 --- a/roles/lib_openshift_api/library/oc_obj.py +++ b/roles/lib_openshift_api/library/oc_obj.py @@ -15,7 +15,15 @@ import json  import os  import shutil  import subprocess +import re +  import yaml +# This is here because of a bug that causes yaml +# to incorrectly handle timezone info on timestamps +def timestamp_constructor(_, node): +    '''return timestamps as strings''' +    return str(node.value) +yaml.add_constructor(u'tag:yaml.org,2002:timestamp', timestamp_constructor)  # pylint: disable=too-few-public-methods  class OpenShiftCLI(object): @@ -39,8 +47,14 @@ class OpenShiftCLI(object):          fname = '/tmp/%s' % rname          yed = Yedit(fname, res['results'][0]) +        changes = []          for key, value in content.items(): -            yed.put(key, value) +            changes.append(yed.put(key, value)) + +        if any([not change[0] for change in changes]): +            return {'returncode': 0, 'updated': False} + +        yed.write()          atexit.register(Utils.cleanup, [fname]) @@ -83,7 +97,9 @@ class OpenShiftCLI(object):          cmds = ['/usr/bin/oc']          cmds.extend(cmd) +        rval = {}          results = '' +        err = None          if self.verbose:              print ' '.join(cmds) @@ -92,27 +108,42 @@ class OpenShiftCLI(object):                                  stdout=subprocess.PIPE,                                  stderr=subprocess.PIPE,                                  env={'KUBECONFIG': self.kubeconfig}) +          proc.wait() +        stdout = proc.stdout.read() +        stderr = proc.stderr.read() + +        rval = {"returncode": proc.returncode, +                "results": results, +               } +          if proc.returncode == 0:              if output:                  try: -                    results = json.loads(proc.stdout.read()) +                    rval['results'] = json.loads(stdout)                  except ValueError as err:                      if "No JSON object could be decoded" in err.message: -                        results = err.message +                        err = err.message              if self.verbose: -                print proc.stderr.read() -                print results +                print stdout +                print stderr                  print -            return {"returncode": proc.returncode, "results": results} +            if err: +                rval.update({"err": err, +                             "stderr": stderr, +                             "stdout": stdout, +                             "cmd": cmds +                            }) -        return {"returncode": proc.returncode, -                "stderr": proc.stderr.read(), -                "stdout": proc.stdout.read(), -                "results": {} -               } +        else: +            rval.update({"stderr": stderr, +                         "stdout": stdout, +                         "results": {}, +                        }) + +        return rval  class Utils(object):      ''' utilities for openshiftcli modules ''' @@ -186,7 +217,7 @@ class Utils(object):              contents = sfd.read()          if sfile_type == 'yaml': -            contents = yaml.load(contents) +            contents = yaml.safe_load(contents)          elif sfile_type == 'json':              contents = json.loads(contents) @@ -254,15 +285,16 @@ class YeditException(Exception):  class Yedit(object):      ''' Class to modify yaml files ''' +    re_valid_key = r"(((\[-?\d+\])|(\w+)).?)+$" +    re_key = r"(?:\[(-?\d+)\])|(\w+)" -    def __init__(self, filename=None, content=None): +    def __init__(self, filename=None, content=None, content_type='yaml'):          self.content = content          self.filename = filename          self.__yaml_dict = content +        self.content_type = content_type          if self.filename and not self.content: -            self.get() -        elif self.filename and self.content: -            self.write() +            self.load(content_type=self.content_type)      @property      def yaml_dict(self): @@ -275,58 +307,89 @@ class Yedit(object):          self.__yaml_dict = value      @staticmethod -    def remove_entry(data, keys): -        ''' remove an item from a dictionary with key notation a.b.c -            d = {'a': {'b': 'c'}}} -            keys = a.b -            item = c -        ''' -        if "." in keys: -            key, rest = keys.split(".", 1) -            if key in data.keys(): -                Yedit.remove_entry(data[key], rest) -        else: -            del data[keys] +    def remove_entry(data, key): +        ''' remove data at location key ''' +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)] +            else: +                return None + +        # process last index for remove +        # expected list entry +        if key_indexes[-1][0]: +            if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +                del data[int(key_indexes[-1][0])] + +        # expected dict entry +        elif key_indexes[-1][1]: +            if isinstance(data, dict): +                del data[key_indexes[-1][1]]      @staticmethod -    def add_entry(data, keys, item): -        ''' Add an item to a dictionary with key notation a.b.c +    def add_entry(data, key, item=None): +        ''' Get an item from a dictionary with key notation a.b.c              d = {'a': {'b': 'c'}}} -            keys = a.b -            item = c +            key = a.b +            return c          ''' -        if "." in keys: -            key, rest = keys.split(".", 1) -            if key not in data: -                data[key] = {} +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        curr_data = data + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key: +                if isinstance(data, dict) and data.has_key(dict_key): +                    data = data[dict_key] +                    continue + +                data[dict_key] = {} +                data = data[dict_key] -            if not isinstance(data, dict): -                raise YeditException('Invalid add_entry called on a [%s] of type [%s].' % (data, type(data))) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)]              else: -                Yedit.add_entry(data[key], rest, item) +                return None -        else: -            data[keys] = item +        # process last index for add +        # expected list entry +        if key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +            data[int(key_indexes[-1][0])] = item +        # expected dict entry +        elif key_indexes[-1][1] and isinstance(data, dict): +            data[key_indexes[-1][1]] = item + +        return curr_data      @staticmethod -    def get_entry(data, keys): +    def get_entry(data, key):          ''' Get an item from a dictionary with key notation a.b.c              d = {'a': {'b': 'c'}}} -            keys = a.b +            key = a.b              return c          ''' -        if keys and "." in keys: -            key, rest = keys.split(".", 1) -            if not isinstance(data[key], dict): -                raise YeditException('Invalid get_entry called on a [%s] of type [%s].' % (data, type(data))) +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)]              else: -                return Yedit.get_entry(data[key], rest) - -        else: -            return data.get(keys, None) +                return None +        return data      def write(self):          ''' write to file ''' @@ -355,7 +418,7 @@ class Yedit(object):          return False -    def get(self): +    def load(self, content_type='yaml'):          ''' return yaml file '''          contents = self.read() @@ -364,13 +427,25 @@ class Yedit(object):          # check if it is yaml          try: -            self.yaml_dict = yaml.load(contents) +            if content_type == 'yaml': +                self.yaml_dict = yaml.load(contents) +            elif content_type == 'json': +                self.yaml_dict = json.loads(contents)          except yaml.YAMLError as _: -            # Error loading yaml +            # Error loading yaml or json              return None          return self.yaml_dict +    def get(self, key): +        ''' get a specified key''' +        try: +            entry = Yedit.get_entry(self.yaml_dict, key) +        except KeyError as _: +            entry = None + +        return entry +      def delete(self, key):          ''' put key, value into a yaml file '''          try: @@ -381,8 +456,7 @@ class Yedit(object):              return  (False, self.yaml_dict)          Yedit.remove_entry(self.yaml_dict, key) -        self.write() -        return (True, self.get()) +        return (True, self.yaml_dict)      def put(self, key, value):          ''' put key, value into a yaml file ''' @@ -395,17 +469,15 @@ class Yedit(object):              return (False, self.yaml_dict)          Yedit.add_entry(self.yaml_dict, key, value) -        self.write() -        return (True, self.get()) +        return (True, self.yaml_dict)      def create(self, key, value):          ''' create the file '''          if not self.exists():              self.yaml_dict = {key: value} -            self.write() -            return (True, self.get()) +            return (True, self.yaml_dict) -        return (False, self.get()) +        return (False, self.yaml_dict)  class OCObject(OpenShiftCLI):      ''' Class to wrap the oc command line tools ''' diff --git a/roles/lib_openshift_api/library/oc_secret.py b/roles/lib_openshift_api/library/oc_secret.py index 8253fd4ad..8e5800e52 100644 --- a/roles/lib_openshift_api/library/oc_secret.py +++ b/roles/lib_openshift_api/library/oc_secret.py @@ -15,7 +15,15 @@ import json  import os  import shutil  import subprocess +import re +  import yaml +# This is here because of a bug that causes yaml +# to incorrectly handle timezone info on timestamps +def timestamp_constructor(_, node): +    '''return timestamps as strings''' +    return str(node.value) +yaml.add_constructor(u'tag:yaml.org,2002:timestamp', timestamp_constructor)  # pylint: disable=too-few-public-methods  class OpenShiftCLI(object): @@ -39,8 +47,14 @@ class OpenShiftCLI(object):          fname = '/tmp/%s' % rname          yed = Yedit(fname, res['results'][0]) +        changes = []          for key, value in content.items(): -            yed.put(key, value) +            changes.append(yed.put(key, value)) + +        if any([not change[0] for change in changes]): +            return {'returncode': 0, 'updated': False} + +        yed.write()          atexit.register(Utils.cleanup, [fname]) @@ -83,7 +97,9 @@ class OpenShiftCLI(object):          cmds = ['/usr/bin/oc']          cmds.extend(cmd) +        rval = {}          results = '' +        err = None          if self.verbose:              print ' '.join(cmds) @@ -92,27 +108,42 @@ class OpenShiftCLI(object):                                  stdout=subprocess.PIPE,                                  stderr=subprocess.PIPE,                                  env={'KUBECONFIG': self.kubeconfig}) +          proc.wait() +        stdout = proc.stdout.read() +        stderr = proc.stderr.read() + +        rval = {"returncode": proc.returncode, +                "results": results, +               } +          if proc.returncode == 0:              if output:                  try: -                    results = json.loads(proc.stdout.read()) +                    rval['results'] = json.loads(stdout)                  except ValueError as err:                      if "No JSON object could be decoded" in err.message: -                        results = err.message +                        err = err.message              if self.verbose: -                print proc.stderr.read() -                print results +                print stdout +                print stderr                  print -            return {"returncode": proc.returncode, "results": results} +            if err: +                rval.update({"err": err, +                             "stderr": stderr, +                             "stdout": stdout, +                             "cmd": cmds +                            }) -        return {"returncode": proc.returncode, -                "stderr": proc.stderr.read(), -                "stdout": proc.stdout.read(), -                "results": {} -               } +        else: +            rval.update({"stderr": stderr, +                         "stdout": stdout, +                         "results": {}, +                        }) + +        return rval  class Utils(object):      ''' utilities for openshiftcli modules ''' @@ -186,7 +217,7 @@ class Utils(object):              contents = sfd.read()          if sfile_type == 'yaml': -            contents = yaml.load(contents) +            contents = yaml.safe_load(contents)          elif sfile_type == 'json':              contents = json.loads(contents) @@ -254,15 +285,16 @@ class YeditException(Exception):  class Yedit(object):      ''' Class to modify yaml files ''' +    re_valid_key = r"(((\[-?\d+\])|(\w+)).?)+$" +    re_key = r"(?:\[(-?\d+)\])|(\w+)" -    def __init__(self, filename=None, content=None): +    def __init__(self, filename=None, content=None, content_type='yaml'):          self.content = content          self.filename = filename          self.__yaml_dict = content +        self.content_type = content_type          if self.filename and not self.content: -            self.get() -        elif self.filename and self.content: -            self.write() +            self.load(content_type=self.content_type)      @property      def yaml_dict(self): @@ -275,58 +307,89 @@ class Yedit(object):          self.__yaml_dict = value      @staticmethod -    def remove_entry(data, keys): -        ''' remove an item from a dictionary with key notation a.b.c -            d = {'a': {'b': 'c'}}} -            keys = a.b -            item = c -        ''' -        if "." in keys: -            key, rest = keys.split(".", 1) -            if key in data.keys(): -                Yedit.remove_entry(data[key], rest) -        else: -            del data[keys] +    def remove_entry(data, key): +        ''' remove data at location key ''' +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)] +            else: +                return None + +        # process last index for remove +        # expected list entry +        if key_indexes[-1][0]: +            if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +                del data[int(key_indexes[-1][0])] + +        # expected dict entry +        elif key_indexes[-1][1]: +            if isinstance(data, dict): +                del data[key_indexes[-1][1]]      @staticmethod -    def add_entry(data, keys, item): -        ''' Add an item to a dictionary with key notation a.b.c +    def add_entry(data, key, item=None): +        ''' Get an item from a dictionary with key notation a.b.c              d = {'a': {'b': 'c'}}} -            keys = a.b -            item = c +            key = a.b +            return c          ''' -        if "." in keys: -            key, rest = keys.split(".", 1) -            if key not in data: -                data[key] = {} +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        curr_data = data + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key: +                if isinstance(data, dict) and data.has_key(dict_key): +                    data = data[dict_key] +                    continue -            if not isinstance(data, dict): -                raise YeditException('Invalid add_entry called on a [%s] of type [%s].' % (data, type(data))) +                data[dict_key] = {} +                data = data[dict_key] + +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)]              else: -                Yedit.add_entry(data[key], rest, item) +                return None -        else: -            data[keys] = item +        # process last index for add +        # expected list entry +        if key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +            data[int(key_indexes[-1][0])] = item +        # expected dict entry +        elif key_indexes[-1][1] and isinstance(data, dict): +            data[key_indexes[-1][1]] = item + +        return curr_data      @staticmethod -    def get_entry(data, keys): +    def get_entry(data, key):          ''' Get an item from a dictionary with key notation a.b.c              d = {'a': {'b': 'c'}}} -            keys = a.b +            key = a.b              return c          ''' -        if keys and "." in keys: -            key, rest = keys.split(".", 1) -            if not isinstance(data[key], dict): -                raise YeditException('Invalid get_entry called on a [%s] of type [%s].' % (data, type(data))) +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)]              else: -                return Yedit.get_entry(data[key], rest) - -        else: -            return data.get(keys, None) +                return None +        return data      def write(self):          ''' write to file ''' @@ -355,7 +418,7 @@ class Yedit(object):          return False -    def get(self): +    def load(self, content_type='yaml'):          ''' return yaml file '''          contents = self.read() @@ -364,13 +427,25 @@ class Yedit(object):          # check if it is yaml          try: -            self.yaml_dict = yaml.load(contents) +            if content_type == 'yaml': +                self.yaml_dict = yaml.load(contents) +            elif content_type == 'json': +                self.yaml_dict = json.loads(contents)          except yaml.YAMLError as _: -            # Error loading yaml +            # Error loading yaml or json              return None          return self.yaml_dict +    def get(self, key): +        ''' get a specified key''' +        try: +            entry = Yedit.get_entry(self.yaml_dict, key) +        except KeyError as _: +            entry = None + +        return entry +      def delete(self, key):          ''' put key, value into a yaml file '''          try: @@ -381,8 +456,7 @@ class Yedit(object):              return  (False, self.yaml_dict)          Yedit.remove_entry(self.yaml_dict, key) -        self.write() -        return (True, self.get()) +        return (True, self.yaml_dict)      def put(self, key, value):          ''' put key, value into a yaml file ''' @@ -395,17 +469,15 @@ class Yedit(object):              return (False, self.yaml_dict)          Yedit.add_entry(self.yaml_dict, key, value) -        self.write() -        return (True, self.get()) +        return (True, self.yaml_dict)      def create(self, key, value):          ''' create the file '''          if not self.exists():              self.yaml_dict = {key: value} -            self.write() -            return (True, self.get()) +            return (True, self.yaml_dict) -        return (False, self.get()) +        return (False, self.yaml_dict)  class Secret(OpenShiftCLI):      ''' Class to wrap the oc command line tools diff --git a/roles/lib_yaml_editor/build/ansible/yedit.py b/roles/lib_yaml_editor/build/ansible/yedit.py index bf868fb71..a4c0d40b3 100644 --- a/roles/lib_yaml_editor/build/ansible/yedit.py +++ b/roles/lib_yaml_editor/build/ansible/yedit.py @@ -24,7 +24,7 @@ def main():      yamlfile = Yedit(module.params['src'], module.params['content']) -    rval = yamlfile.get() +    rval = yamlfile.load()      if not rval and state != 'present':          module.fail_json(msg='Error opening file [%s].  Verify that the' + \                               ' file exists, that it is has correct permissions, and is valid yaml.') @@ -51,7 +51,7 @@ def main():              rval = yamlfile.create(module.params['key'], value)          else:              yamlfile.write() -            rval = yamlfile.get() +            rval = yamlfile.load()          module.exit_json(changed=rval[0], results=rval[1], state="present")      module.exit_json(failed=True, diff --git a/roles/lib_yaml_editor/build/generate.py b/roles/lib_yaml_editor/build/generate.py index 0df4efb92..312e4d0ee 100755 --- a/roles/lib_yaml_editor/build/generate.py +++ b/roles/lib_yaml_editor/build/generate.py @@ -15,19 +15,20 @@ GEN_STR = "#!/usr/bin/env python\n"                                  + \            "#   | |) | (_) | | .` | (_) || |   | _|| |) | |  | |\n"   + \            "#   |___/ \___/  |_|\_|\___/ |_|   |___|___/___| |_|\n" +OPENSHIFT_ANSIBLE_PATH = os.path.dirname(os.path.realpath(__file__)) +  FILES = {'yedit.py':  ['src/base.py', 'src/yedit.py', 'ansible/yedit.py'],          } -  def main():      ''' combine the necessary files to create the ansible module ''' -    openshift_ansible = ('../library/') +    library = os.path.join(OPENSHIFT_ANSIBLE_PATH, '..', 'library/')      for fname, parts in FILES.items(): -        with open(os.path.join(openshift_ansible, fname), 'w') as afd: +        with open(os.path.join(library, fname), 'w') as afd:              afd.seek(0)              afd.write(GEN_STR)              for fpart in parts: -                with open(fpart) as pfd: +                with open(os.path.join(OPENSHIFT_ANSIBLE_PATH, fpart)) as pfd:                      # first line is pylint disable so skip it                      for idx, line in enumerate(pfd):                          if idx == 0 and 'skip-file' in line: diff --git a/roles/lib_yaml_editor/build/src/base.py b/roles/lib_yaml_editor/build/src/base.py index ad8b041cf..9e43d45dc 100644 --- a/roles/lib_yaml_editor/build/src/base.py +++ b/roles/lib_yaml_editor/build/src/base.py @@ -5,5 +5,13 @@ module for managing yaml files  '''  import os +import re +  import yaml +# This is here because of a bug that causes yaml +# to incorrectly handle timezone info on timestamps +def timestamp_constructor(_, node): +    ''' return timestamps as strings''' +    return str(node.value) +yaml.add_constructor(u'tag:yaml.org,2002:timestamp', timestamp_constructor) diff --git a/roles/lib_yaml_editor/build/src/yedit.py b/roles/lib_yaml_editor/build/src/yedit.py index 4f6a91d8b..faef577ae 100644 --- a/roles/lib_yaml_editor/build/src/yedit.py +++ b/roles/lib_yaml_editor/build/src/yedit.py @@ -6,15 +6,16 @@ class YeditException(Exception):  class Yedit(object):      ''' Class to modify yaml files ''' +    re_valid_key = r"(((\[-?\d+\])|(\w+)).?)+$" +    re_key = r"(?:\[(-?\d+)\])|(\w+)" -    def __init__(self, filename=None, content=None): +    def __init__(self, filename=None, content=None, content_type='yaml'):          self.content = content          self.filename = filename          self.__yaml_dict = content +        self.content_type = content_type          if self.filename and not self.content: -            self.get() -        elif self.filename and self.content: -            self.write() +            self.load(content_type=self.content_type)      @property      def yaml_dict(self): @@ -27,58 +28,89 @@ class Yedit(object):          self.__yaml_dict = value      @staticmethod -    def remove_entry(data, keys): -        ''' remove an item from a dictionary with key notation a.b.c -            d = {'a': {'b': 'c'}}} -            keys = a.b -            item = c -        ''' -        if "." in keys: -            key, rest = keys.split(".", 1) -            if key in data.keys(): -                Yedit.remove_entry(data[key], rest) -        else: -            del data[keys] +    def remove_entry(data, key): +        ''' remove data at location key ''' +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)] +            else: +                return None + +        # process last index for remove +        # expected list entry +        if key_indexes[-1][0]: +            if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +                del data[int(key_indexes[-1][0])] + +        # expected dict entry +        elif key_indexes[-1][1]: +            if isinstance(data, dict): +                del data[key_indexes[-1][1]]      @staticmethod -    def add_entry(data, keys, item): -        ''' Add an item to a dictionary with key notation a.b.c +    def add_entry(data, key, item=None): +        ''' Get an item from a dictionary with key notation a.b.c              d = {'a': {'b': 'c'}}} -            keys = a.b -            item = c +            key = a.b +            return c          ''' -        if "." in keys: -            key, rest = keys.split(".", 1) -            if key not in data: -                data[key] = {} +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        curr_data = data -            if not isinstance(data, dict): -                raise YeditException('Invalid add_entry called on a [%s] of type [%s].' % (data, type(data))) +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key: +                if isinstance(data, dict) and data.has_key(dict_key): +                    data = data[dict_key] +                    continue + +                data[dict_key] = {} +                data = data[dict_key] + +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)]              else: -                Yedit.add_entry(data[key], rest, item) +                return None + +        # process last index for add +        # expected list entry +        if key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +            data[int(key_indexes[-1][0])] = item -        else: -            data[keys] = item +        # expected dict entry +        elif key_indexes[-1][1] and isinstance(data, dict): +            data[key_indexes[-1][1]] = item +        return curr_data      @staticmethod -    def get_entry(data, keys): +    def get_entry(data, key):          ''' Get an item from a dictionary with key notation a.b.c              d = {'a': {'b': 'c'}}} -            keys = a.b +            key = a.b              return c          ''' -        if keys and "." in keys: -            key, rest = keys.split(".", 1) -            if not isinstance(data[key], dict): -                raise YeditException('Invalid get_entry called on a [%s] of type [%s].' % (data, type(data))) +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)]              else: -                return Yedit.get_entry(data[key], rest) - -        else: -            return data.get(keys, None) +                return None +        return data      def write(self):          ''' write to file ''' @@ -107,7 +139,7 @@ class Yedit(object):          return False -    def get(self): +    def load(self, content_type='yaml'):          ''' return yaml file '''          contents = self.read() @@ -116,13 +148,25 @@ class Yedit(object):          # check if it is yaml          try: -            self.yaml_dict = yaml.load(contents) +            if content_type == 'yaml': +                self.yaml_dict = yaml.load(contents) +            elif content_type == 'json': +                self.yaml_dict = json.loads(contents)          except yaml.YAMLError as _: -            # Error loading yaml +            # Error loading yaml or json              return None          return self.yaml_dict +    def get(self, key): +        ''' get a specified key''' +        try: +            entry = Yedit.get_entry(self.yaml_dict, key) +        except KeyError as _: +            entry = None + +        return entry +      def delete(self, key):          ''' put key, value into a yaml file '''          try: @@ -133,8 +177,7 @@ class Yedit(object):              return  (False, self.yaml_dict)          Yedit.remove_entry(self.yaml_dict, key) -        self.write() -        return (True, self.get()) +        return (True, self.yaml_dict)      def put(self, key, value):          ''' put key, value into a yaml file ''' @@ -147,14 +190,12 @@ class Yedit(object):              return (False, self.yaml_dict)          Yedit.add_entry(self.yaml_dict, key, value) -        self.write() -        return (True, self.get()) +        return (True, self.yaml_dict)      def create(self, key, value):          ''' create the file '''          if not self.exists():              self.yaml_dict = {key: value} -            self.write() -            return (True, self.get()) +            return (True, self.yaml_dict) -        return (False, self.get()) +        return (False, self.yaml_dict) diff --git a/roles/lib_yaml_editor/build/test/foo.yml b/roles/lib_yaml_editor/build/test/foo.yml index 2a7a89ce2..20e9ff3fe 100644 --- a/roles/lib_yaml_editor/build/test/foo.yml +++ b/roles/lib_yaml_editor/build/test/foo.yml @@ -1 +1 @@ -foo: barplus +foo: bar diff --git a/roles/lib_yaml_editor/library/yedit.py b/roles/lib_yaml_editor/library/yedit.py index f375fd8e2..696ece63b 100644 --- a/roles/lib_yaml_editor/library/yedit.py +++ b/roles/lib_yaml_editor/library/yedit.py @@ -12,7 +12,15 @@ module for managing yaml files  '''  import os +import re +  import yaml +# This is here because of a bug that causes yaml +# to incorrectly handle timezone info on timestamps +def timestamp_constructor(_, node): +    ''' return timestamps as strings''' +    return str(node.value) +yaml.add_constructor(u'tag:yaml.org,2002:timestamp', timestamp_constructor)  class YeditException(Exception): @@ -21,15 +29,16 @@ class YeditException(Exception):  class Yedit(object):      ''' Class to modify yaml files ''' +    re_valid_key = r"(((\[-?\d+\])|(\w+)).?)+$" +    re_key = r"(?:\[(-?\d+)\])|(\w+)" -    def __init__(self, filename=None, content=None): +    def __init__(self, filename=None, content=None, content_type='yaml'):          self.content = content          self.filename = filename          self.__yaml_dict = content +        self.content_type = content_type          if self.filename and not self.content: -            self.get() -        elif self.filename and self.content: -            self.write() +            self.load(content_type=self.content_type)      @property      def yaml_dict(self): @@ -42,58 +51,89 @@ class Yedit(object):          self.__yaml_dict = value      @staticmethod -    def remove_entry(data, keys): -        ''' remove an item from a dictionary with key notation a.b.c -            d = {'a': {'b': 'c'}}} -            keys = a.b -            item = c -        ''' -        if "." in keys: -            key, rest = keys.split(".", 1) -            if key in data.keys(): -                Yedit.remove_entry(data[key], rest) -        else: -            del data[keys] +    def remove_entry(data, key): +        ''' remove data at location key ''' +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)] +            else: +                return None + +        # process last index for remove +        # expected list entry +        if key_indexes[-1][0]: +            if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +                del data[int(key_indexes[-1][0])] + +        # expected dict entry +        elif key_indexes[-1][1]: +            if isinstance(data, dict): +                del data[key_indexes[-1][1]]      @staticmethod -    def add_entry(data, keys, item): -        ''' Add an item to a dictionary with key notation a.b.c +    def add_entry(data, key, item=None): +        ''' Get an item from a dictionary with key notation a.b.c              d = {'a': {'b': 'c'}}} -            keys = a.b -            item = c +            key = a.b +            return c          ''' -        if "." in keys: -            key, rest = keys.split(".", 1) -            if key not in data: -                data[key] = {} +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None + +        curr_data = data -            if not isinstance(data, dict): -                raise YeditException('Invalid add_entry called on a [%s] of type [%s].' % (data, type(data))) +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes[:-1]: +            if dict_key: +                if isinstance(data, dict) and data.has_key(dict_key): +                    data = data[dict_key] +                    continue + +                data[dict_key] = {} +                data = data[dict_key] + +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)]              else: -                Yedit.add_entry(data[key], rest, item) +                return None -        else: -            data[keys] = item +        # process last index for add +        # expected list entry +        if key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: +            data[int(key_indexes[-1][0])] = item + +        # expected dict entry +        elif key_indexes[-1][1] and isinstance(data, dict): +            data[key_indexes[-1][1]] = item +        return curr_data      @staticmethod -    def get_entry(data, keys): +    def get_entry(data, key):          ''' Get an item from a dictionary with key notation a.b.c              d = {'a': {'b': 'c'}}} -            keys = a.b +            key = a.b              return c          ''' -        if keys and "." in keys: -            key, rest = keys.split(".", 1) -            if not isinstance(data[key], dict): -                raise YeditException('Invalid get_entry called on a [%s] of type [%s].' % (data, type(data))) +        if not (key and re.match(Yedit.re_valid_key, key) and isinstance(data, (list, dict))): +            return None +        key_indexes = re.findall(Yedit.re_key, key) +        for arr_ind, dict_key in key_indexes: +            if dict_key and isinstance(data, dict): +                data = data.get(dict_key, None) +            elif arr_ind and isinstance(data, list) and int(arr_ind) <= len(data) - 1: +                data = data[int(arr_ind)]              else: -                return Yedit.get_entry(data[key], rest) - -        else: -            return data.get(keys, None) +                return None +        return data      def write(self):          ''' write to file ''' @@ -122,7 +162,7 @@ class Yedit(object):          return False -    def get(self): +    def load(self, content_type='yaml'):          ''' return yaml file '''          contents = self.read() @@ -131,13 +171,25 @@ class Yedit(object):          # check if it is yaml          try: -            self.yaml_dict = yaml.load(contents) +            if content_type == 'yaml': +                self.yaml_dict = yaml.load(contents) +            elif content_type == 'json': +                self.yaml_dict = json.loads(contents)          except yaml.YAMLError as _: -            # Error loading yaml +            # Error loading yaml or json              return None          return self.yaml_dict +    def get(self, key): +        ''' get a specified key''' +        try: +            entry = Yedit.get_entry(self.yaml_dict, key) +        except KeyError as _: +            entry = None + +        return entry +      def delete(self, key):          ''' put key, value into a yaml file '''          try: @@ -148,8 +200,7 @@ class Yedit(object):              return  (False, self.yaml_dict)          Yedit.remove_entry(self.yaml_dict, key) -        self.write() -        return (True, self.get()) +        return (True, self.yaml_dict)      def put(self, key, value):          ''' put key, value into a yaml file ''' @@ -162,17 +213,15 @@ class Yedit(object):              return (False, self.yaml_dict)          Yedit.add_entry(self.yaml_dict, key, value) -        self.write() -        return (True, self.get()) +        return (True, self.yaml_dict)      def create(self, key, value):          ''' create the file '''          if not self.exists():              self.yaml_dict = {key: value} -            self.write() -            return (True, self.get()) +            return (True, self.yaml_dict) -        return (False, self.get()) +        return (False, self.yaml_dict)  def main():      ''' @@ -198,7 +247,7 @@ def main():      yamlfile = Yedit(module.params['src'], module.params['content']) -    rval = yamlfile.get() +    rval = yamlfile.load()      if not rval and state != 'present':          module.fail_json(msg='Error opening file [%s].  Verify that the' + \                               ' file exists, that it is has correct permissions, and is valid yaml.') @@ -225,7 +274,7 @@ def main():              rval = yamlfile.create(module.params['key'], value)          else:              yamlfile.write() -            rval = yamlfile.get() +            rval = yamlfile.load()          module.exit_json(changed=rval[0], results=rval[1], state="present")      module.exit_json(failed=True, diff --git a/test/env-setup b/test/env-setup index b05df0f9e..7456a641b 100644 --- a/test/env-setup +++ b/test/env-setup @@ -2,7 +2,7 @@  CUR_PATH=$(pwd) -PREFIX_PYTHONPATH=$CUR_PATH/inventory/:$CUR_PATH/roles/lib_yaml_editor/build/src +PREFIX_PYTHONPATH=$CUR_PATH/inventory/:$CUR_PATH/roles/lib_yaml_editor/library  export PYTHONPATH=$PREFIX_PYTHONPATH:$PYTHONPATH diff --git a/test/units/yedit_test.py b/test/units/yedit_test.py index e701cfa7c..09a65e888 100755 --- a/test/units/yedit_test.py +++ b/test/units/yedit_test.py @@ -16,7 +16,7 @@ class YeditTest(unittest.TestCase):       Test class for yedit      '''      data = {'a': 'a', -            'b': {'c': {'d': ['e', 'f', 'g']}}, +            'b': {'c': {'d': [{'e': 'x'}, 'f', 'g']}},             }      filename = 'yedit_test.yml' @@ -27,10 +27,9 @@ class YeditTest(unittest.TestCase):          yed.yaml_dict = YeditTest.data          yed.write() -    def test_get(self): +    def test_load(self):          ''' Testing a get '''          yed = Yedit('yedit_test.yml') -          self.assertEqual(yed.yaml_dict, self.data)      def test_write(self): @@ -38,7 +37,6 @@ class YeditTest(unittest.TestCase):          yed = Yedit('yedit_test.yml')          yed.put('key1', 1)          yed.write() -        yed.get()          self.assertTrue(yed.yaml_dict.has_key('key1'))          self.assertEqual(yed.yaml_dict['key1'], 1) @@ -47,14 +45,15 @@ class YeditTest(unittest.TestCase):          yed = Yedit('yedit_test.yml')          yed.put('x.y.z', 'modified')          yed.write() -        self.assertEqual(Yedit.get_entry(yed.get(), 'x.y.z'), 'modified') +        yed.load() +        self.assertEqual(yed.get('x.y.z'), 'modified')      def test_delete_a(self):          '''Testing a simple delete '''          yed = Yedit('yedit_test.yml')          yed.delete('a')          yed.write() -        yed.get() +        yed.load()          self.assertTrue(not yed.yaml_dict.has_key('a'))      def test_delete_b_c(self): @@ -62,7 +61,7 @@ class YeditTest(unittest.TestCase):          yed = Yedit('yedit_test.yml')          yed.delete('b.c')          yed.write() -        yed.get() +        yed.load()          self.assertTrue(yed.yaml_dict.has_key('b'))          self.assertFalse(yed.yaml_dict['b'].has_key('c')) @@ -72,7 +71,7 @@ class YeditTest(unittest.TestCase):          yed = Yedit('yedit_test.yml')          yed.create('foo', 'bar')          yed.write() -        yed.get() +        yed.load()          self.assertTrue(yed.yaml_dict.has_key('foo'))          self.assertTrue(yed.yaml_dict['foo'] == 'bar') @@ -81,10 +80,61 @@ class YeditTest(unittest.TestCase):          content = {"foo": "bar"}          yed = Yedit("yedit_test.yml", content)          yed.write() -        yed.get() +        yed.load()          self.assertTrue(yed.yaml_dict.has_key('foo'))          self.assertTrue(yed.yaml_dict['foo'], 'bar') +    def test_array_insert(self): +        '''Testing a create with content ''' +        yed = Yedit("yedit_test.yml") +        yed.put('b.c.d[0]', 'inject') +        self.assertTrue(yed.get('b.c.d[0]') == 'inject') + +    def test_array_insert_first_index(self): +        '''Testing a create with content ''' +        yed = Yedit("yedit_test.yml") +        yed.put('b.c.d[0]', 'inject') +        self.assertTrue(yed.get('b.c.d[1]') == 'f') + +    def test_array_insert_second_index(self): +        '''Testing a create with content ''' +        yed = Yedit("yedit_test.yml") +        yed.put('b.c.d[0]', 'inject') +        self.assertTrue(yed.get('b.c.d[2]') == 'g') + +    def test_dict_array_dict_access(self): +        '''Testing a create with content''' +        yed = Yedit("yedit_test.yml") +        yed.put('b.c.d[0]', [{'x': {'y': 'inject'}}]) +        self.assertTrue(yed.get('b.c.d[0].[0].x.y') == 'inject') + +    def test_dict_array_dict_replace(self): +        '''Testing multilevel delete''' +        yed = Yedit("yedit_test.yml") +        yed.put('b.c.d[0]', [{'x': {'y': 'inject'}}]) +        yed.put('b.c.d[0].[0].x.y', 'testing') +        self.assertTrue(yed.yaml_dict.has_key('b')) +        self.assertTrue(yed.yaml_dict['b'].has_key('c')) +        self.assertTrue(yed.yaml_dict['b']['c'].has_key('d')) +        self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'], list)) +        self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0], list)) +        self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0][0], dict)) +        self.assertTrue(yed.yaml_dict['b']['c']['d'][0][0]['x'].has_key('y')) +        self.assertTrue(yed.yaml_dict['b']['c']['d'][0][0]['x']['y'], 'testing') + +    def test_dict_array_dict_remove(self): +        '''Testing multilevel delete''' +        yed = Yedit("yedit_test.yml") +        yed.put('b.c.d[0]', [{'x': {'y': 'inject'}}]) +        yed.delete('b.c.d[0].[0].x.y') +        self.assertTrue(yed.yaml_dict.has_key('b')) +        self.assertTrue(yed.yaml_dict['b'].has_key('c')) +        self.assertTrue(yed.yaml_dict['b']['c'].has_key('d')) +        self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'], list)) +        self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0], list)) +        self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0][0], dict)) +        self.assertFalse(yed.yaml_dict['b']['c']['d'][0][0]['x'].has_key('y')) +      def tearDown(self):          '''TearDown method'''          os.unlink(YeditTest.filename) | 
