# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2013 TrilioData, Inc.
# All Rights Reserved.

"""
Handles all requests relating to compute + nova.
"""

import time
import glob
import itertools
import inspect
import pkgutil
import os
import imp
import pkg_resources
import six
from six.moves.urllib import parse
from threading import Lock
from collections import namedtuple
from functools import wraps

from oslo_config import cfg
from oslo_serialization import base64

from novaclient import exceptions as nova_exception
from novaclient import extension as nova_extension
from novaclient import client as nova_client

from neutronclient.common import exceptions as nc_exc

from keystoneclient import exceptions as keystone_exceptions
from keystoneauth1.identity import v3
from keystoneauth1 import session

from workloadmgr.db import base
from workloadmgr.db.workloadmgrdb import WorkloadMgrDB
from workloadmgr.common import context as wlm_context
from workloadmgr import exception
from workloadmgr.openstack.common.gettextutils import _
from workloadmgr.openstack.common import excutils
from workloadmgr.openstack.common import log as logging
from workloadmgr.common import clients

from workloadmgr import autolog
from workloadmgr.decorators import retry

LOG = logging.getLogger(__name__)
Logger = autolog.Logger(LOG)

nova_opts = [
    cfg.StrOpt('nova_admin_auth_url',
               default='http://localhost:5000/v2.0',
               help='auth url for connecting to nova in admin context'),
    cfg.StrOpt('nova_admin_username',
               default='admin',
               help='tenant name for connecting to nova in admin context'),
    cfg.StrOpt('nova_admin_password',
               default='password',
               help='password for connecting to nova in admin context',
               secret=True),
    cfg.StrOpt('nova_admin_tenant_name',
               default='admin',
               help='tenant name for connecting to nova in admin context'),
    cfg.StrOpt('nova_production_endpoint_template',
               default='http://localhost:8774/v2/%(project_id)s',
               help='nova production endpoint e.g. http://localhost:8774/v2/%(project_id)s'),
    cfg.StrOpt('nova_tvault_endpoint_template',
               default='http://localhost:8774/v2/%(project_id)s',
               help='nova tvault endpoint e.g. http://localhost:8774/v2/%(project_id)s'),
    cfg.StrOpt('nova_production_region_name',
               default=None,
               help='region name for connecting to nova in admin context'),
    cfg.StrOpt('nova_tvault_region_name',
               default=None,
               help='region name for connecting to nova in admin context'),
    cfg.BoolOpt('nova_api_insecure',
                default=True,
                help='if set, ignore any SSL validation issues'),
    cfg.StrOpt('nova_auth_system',
               default='keystone',
               help='auth system for connecting to '
                    'nova in admin context'),
    cfg.IntOpt('nova_url_timeout',
               default=600,
               help='timeout value for connecting to nova in seconds'),
    cfg.StrOpt('cloud_admin_user_id',
               default='admin',
               help='cloud admin user id for cloud admin context.'),
    cfg.StrOpt('cloud_admin_project_id',
               default='admin',
               help='project id for connecting to cloud admin context',
               secret=True),
    cfg.StrOpt('cloud_admin_domain',
               default='default',
               help='domain name for connecting to cloud admin context'),
    cfg.FloatOpt('nova_api_version',
               default=2,
               help='Nova API version used to initialize the nova client.'),
]

CONF = cfg.CONF
CONF.register_opts(nova_opts)

LOG = logging.getLogger(__name__)

novalock = Lock()


class ObjectDummy(object):
    pass


def synchronized(lock):
    """ Synchronization decorator. """
    def wrap(f):
        def new_function(*args, **kw):
            lock.acquire()
            try:
                return f(*args, **kw)
            finally:
                lock.release()
        return new_function
    return wrap


def _discover_extensions(version):
    extensions = []
    for name, module in itertools.chain(
            _discover_via_python_path(),
            _discover_via_contrib_path(version),
            _discover_via_entry_points()):
        extension = nova_extension.Extension(name, module)
        extensions.append(extension)
    return extensions


def _discover_via_python_path():
    for (module_loader, name, _ispkg) in pkgutil.iter_modules():
        if name.endswith('_python_novaclient_ext'):
            if not hasattr(module_loader, 'load_module'):
                # Python 2.6 compat: actually get an ImpImporter obj
                module_loader = module_loader.find_module(name)

            module = module_loader.load_module(name)
            if hasattr(module, 'extension_name'):
                name = module.extension_name

            yield name, module


def _discover_via_contrib_path(version):
    module_path = os.path.dirname(os.path.abspath(__file__))
    version_str = "v%s" % version.replace('.', '_')
    ext_path = os.path.join(module_path, version_str, 'contrib')
    ext_glob = os.path.join(ext_path, "*.py")

    for ext_path in glob.iglob(ext_glob):
        name = os.path.basename(ext_path)[:-3]

        if name == "__init__":
            continue

        module = imp.load_source(name, ext_path)
        yield name, module


def _discover_via_entry_points():
    for ep in pkg_resources.iter_entry_points('novaclient.extension'):
        name = ep.name
        module = ep.load()
        yield name, module


try:
    # load keystone_authtoken by importing keystonemiddleware
    # if it is already loaded, just ignore the exception
    cfg.CONF.import_group('keystone_authtoken',
                          'keystonemiddleware.auth_token')
except BaseException:
    pass


def _get_trusts(user_id, tenant_id):
    db = WorkloadMgrDB().db
    context = wlm_context.RequestContext(
        user_id=user_id,
        project_id=tenant_id)

    settings = db.setting_get_all_by_project(
        context, context.project_id)

    trust = [t for t in settings if t.type == "trust_id" and
             t.project_id == context.project_id and
             t.user_id == context.user_id]
    return trust


def _get_tenant_context(context, cloud_admin=False):
    def _get_context(trust, tenant_id, trustor_user_id, user_domain_id, roles=[]):
        try:
            trust_id = trust[0].value
            req_context = wlm_context.RequestContext(
                username=CONF.keystone_authtoken.admin_user,
                password=CONF.keystone_authtoken.admin_password,
                trust_id=trust_id,
                tenant_id=tenant_id,
                trustor_user_id=trustor_user_id,
                user_domain_id=user_domain_id,
                roles=roles,
                is_admin=False)
            return req_context
        except Exception as ex:
            LOG.exception(ex)
            raise ex

    try:
        from workloadmgr import workloads as workloadAPI

        # Fetch cloud admin trust
        cloud_trust = _get_trusts('cloud_admin', CONF.cloud_admin_project_id)

        if len(cloud_trust) < 1:
            msg = _("No cloud admin trust found. Please recreate using CLI")
            workloadAPI.api.AUDITLOG.log(context, msg, None)
            LOG.exception(msg)
            raise Exception(msg)

        cloud_admin_context = _get_context(cloud_trust, CONF.cloud_admin_project_id,
                                           CONF.cloud_admin_user_id, CONF.cloud_admin_domain)

        try:
            clients.initialise()
            client_plugin = clients.Clients(cloud_admin_context)
            admin_kclient = client_plugin.client("keystone")
        except exception.AuthorizationFailure:
            msg = _("Failed to authenticate using saved cloud admin trust.\
                      Please recreate cloud admin trust using CLI.")
            workloadAPI.api.AUDITLOG.log(context, msg, None)
            LOG.exception(msg)
            raise Exception(msg)
        except Exception as ex:
            LOG.exception(ex)
            raise ex

        if cloud_admin is True:
            cloud_admin_context.auth_token = admin_kclient.auth_token
            cloud_admin_context.user_id = getattr(context, 'user_id', 'NA')
            cloud_admin_context.user = getattr(context, 'user', 'NA')
            cloud_admin_context.tenant = getattr(context, 'tenant', 'NA')
            return cloud_admin_context
        else:
            trust = _get_trusts(context.user_id, context.project_id)
            if len(trust) < 1:
                msg = _("Assign valid trustee role to tenant %s") % getattr(context, 'project_id', 'NA')
                workloadAPI.api.AUDITLOG.log(context, msg, None)
                LOG.info(msg)
                raise Exception(_("token cannot be created using saved "
                                  "trust id for user %s, tenant %s") %
                                (context.user_id, context.project_id))

            roles = getattr(context, 'roles', [])

            if not roles:
                roles = [CONF.trustee_role]

            tenant_context = _get_context(trust, context.project_id, context.user_id, \
                                          CONF.triliovault_user_domain_id, roles=roles)

            clients.initialise()
            client_plugin = clients.Clients(tenant_context)

            try:
                kclient = client_plugin.client("keystone")
            except keystone_exceptions.Forbidden:
                msg = _("Assign valid trustee role to tenant %s") % context.project_id
                raise Exception(msg)

            tenant_context.auth_token = kclient.auth_token
            tenant_context.user_id = context.user_id

            tenant_context.user = getattr(context, 'user', None)
            tenant_context.project_id = getattr(context, 'project_id', None)

            return tenant_context
    except Exception as ex:
        LOG.exception(ex)
        raise ex


def novaclient(context, production=True, refresh_token=False, extensions=None, **kwargs):
    trust = _get_trusts(context.user_id, context.project_id)
    if hasattr(context, 'user_domain_id'):
        if context.user_domain_id is None:
            user_domain_id = 'default'
        else:
            user_domain_id = context.user_domain_id
    elif hasattr(context, 'user_domain'):
        if context.user_domain is None:
            user_domain_id = 'default'
        else:
            user_domain_id = context.user_domain
    else:
        user_domain_id = 'default'
    # pick the first trust. Usually it should not be more than one trust
    if len(trust):
        trust_id = trust[0].value

        if refresh_token:
            context = wlm_context.RequestContext(
                username=CONF.keystone_authtoken.admin_user,
                password=CONF.keystone_authtoken.admin_password,
                trust_id=trust_id,
                tenant_id=context.project_id,
                trustor_user_id=context.user_id,
                user_domain_id=CONF.triliovault_user_domain_id,
                is_admin=False)
        else:
            context = wlm_context.RequestContext(
                trustor_user_id=context.user_id,
                project_id=context.project_id,
                auth_token=context.auth_token,
                trust_id=trust_id,
                user_domain_id=user_domain_id,
                is_admin=False)

        clients.initialise()
        nova_plugin = clients.Clients(context)
        novaclient = nova_plugin.client("nova", **kwargs)
        novaclient.client_plugin = novaclient
    else:
        # trusts are not enabled
        if refresh_token:
            if production:
                url = CONF.nova_production_endpoint_template.replace(
                    '%(project_id)s', context.project_id)
                url = url.replace("/v2.1/", "/v2/")
                novaclient = nova_client.Client(
                    CONF.nova_api_version,
                    CONF.nova_admin_username,
                    CONF.nova_admin_password,
                    project_id=context.project_id,
                    auth_url=CONF.keystone_authtoken.www_authenticate_uri,
                    user_domain_id=user_domain_id,
                    insecure=CONF.nova_api_insecure,
                    extensions=extensions,
                    timeout=CONF.nova_url_timeout,
                    endpoint_type=CONF.clients.endpoint_type)
            else:
                url = CONF.nova_tvault_endpoint_template.replace(
                    '%(project_id)s', context.project_id)
                novaclient = nova_client.Client(
                    CONF.nova_api_version,
                    CONF.nova_admin_username,
                    CONF.nova_admin_password,
                    project_id=context.project_id,
                    auth_url=CONF.keystone_authtoken.www_authenticate_uri,
                    user_domain_id=user_domain_id,
                    insecure=CONF.nova_api_insecure,
                    extensions=extensions,
                    timeout=CONF.nova_url_timeout,
                    endpoint_type=CONF.clients.endpoint_type)
            LOG.debug(_('Novaclient connection created using URL: %s') % url)
        else:
            if production:
                url = CONF.nova_production_endpoint_template % context.to_dict()
                url = url.replace("/v2.1/", "/v2/")
            else:
                url = CONF.nova_tvault_endpoint_template % context.to_dict()
            LOG.debug(_('Novaclient connection created using URL: %s') % url)
            novaclient = nova_client.Client(CONF.nova_api_version,
                                            context.user_id,
                                            context.auth_token,
                                            project_id=context.project_id,
                                            auth_url=CONF.keystone_authtoken.www_authenticate_uri,
                                            user_domain_id=user_domain_id,
                                            insecure=CONF.nova_api_insecure,
                                            extensions=extensions,
                                            timeout=CONF.nova_url_timeout,
                                            endpoint_type=CONF.clients.endpoint_type)

            # noauth extracts user_id:tenant_id from auth_token
            novaclient.client.auth_token = context.auth_token or '%s:%s' % (
                context.user_id, context.project_id)
            novaclient.client.management_url = url

    return novaclient


def novaclient2(auth_url, username, password,
                tenant_name, nova_endpoint_template):
    httpclient = client.HTTPClient(
        user=username,
        password=password,
        projectid=tenant_name,
        service_type='compute',
        endpoint_type=CONF.clients.endpoint_type,
        region_name=CONF.nova_production_region_name,
        auth_url=auth_url,
        timeout=CONF.nova_url_timeout,
        auth_system=CONF.nova_auth_system,
        insecure=CONF.nova_api_insecure)
    httpclient.authenticate()
    url = nova_endpoint_template.replace(
        '%(project_id)s', httpclient.tenant_id)
    c = nova_client.Client(CONF.nova_api_version,
                           username,
                           password,
                           project_id=httpclient.tenant_id,
                           auth_url=url,
                           insecure=CONF.nova_api_insecure,
                           extensions=None,
                           timeout=CONF.nova_url_timeout,
                           endpoint_type=CONF.clients.endpoint_type)
    LOG.debug(_('Novaclient connection created using URL: %s') % url)
    c.client.auth_token = httpclient.auth_token
    c.client.management_url = url
    return c


def nova_micro_client(context, version):
    try:
        user_domain_id = 'default'
        if hasattr(context, 'user_domain_id') and context.user_domain_id:
            user_domain_id = context.user_domain_id
        elif hasattr(context, 'user_domain') and context.user_domain:
            user_domain_id = context.user_domain

        project_domain_id = 'default'
        if hasattr(context, 'project_domain_id') and context.project_domain_id:
            project_domain_id = context.project_domain_id
        elif hasattr(context, 'project_domain') and context.project_domain:
            project_domain_id = context.project_domain

        params = {
             'auth_url': CONF.keystone_authtoken.www_authenticate_uri,
             'region_name': CONF.region_name_for_services,
             'endpoint_type': CONF.clients.endpoint_type,
             'insecure': CONF.clients.insecure,
             'cacert': CONF.clients.cafile,
             'project_domain_id': project_domain_id,
             'user_domain_id': user_domain_id
             }
        if hasattr(context, "password") and context.password:
            params['username'] = context.username
            params['password'] = context.password
        else:
            auth = v3.Token(auth_url=CONF.keystone_authtoken.www_authenticate_uri,
                            project_id=context.project_id,
                            token=context.auth_token)
            sess = session.Session(auth=auth, verify=CONF.clients.cafile)
            params['session'] = sess
        return nova_client.Client(version, **params)
    except nova_exception.Unauthorized as ex:
        LOG.exception(ex)
        raise ex
    except Exception as ex:
        LOG.exception(ex)
        raise ex


def exception_handler(ignore_exception=False,
                      refresh_token=True, contego=False, **kwargs):
    def exception_handler_decorator(func):
        @wraps(func)
        def func_wrapper(*args, **argv):
            try:
                try:
                    extensions = None
                    if contego is True:
                        extensions = _discover_extensions('1.1')
                    client = novaclient(args[1], args[0]._production,
                                        refresh_token=False,
                                        extensions=extensions, **kwargs)
                    argv.update({'client': client})
                    return func(*args, **argv)
                except (nc_exc.NeutronClientException, nova_exception.Unauthorized) as unauth_ex:
                    if refresh_token is True:
                        argv.pop('client')
                        client = novaclient(args[1], args[0]._production,
                                            refresh_token=True,
                                            extensions=extensions)
                        argv.update({'client': client})
                        return func(*args, **argv)
            except Exception as ex:
                if ignore_exception is True:
                    LOG.exception(ex)
                    if nova_exception.BadRequest in \
                            inspect.getmro(ex.__class__) or \
                            nova_exception.NotFound in \
                            inspect.getmro(ex.__class__):
                        return
                    return

                if contego is True:
                    msg = "Unable to call %s; Please check contego " \
                          "logs for more details" % func.__name__
                    if hasattr(ex, 'code') and ex.code == 413:
                        msg = str(ex)
                    raise exception.ErrorOccurred(reason=msg)
                else:
                    raise

        return func_wrapper

    return exception_handler_decorator


class API(base.Base):
    """API for interacting with the volume manager."""

    def __init__(self, production=True):
        self._production = production

    @synchronized(novalock)
    @exception_handler(ignore_exception=True)
    def get_hypervisors(self, context, **kwargs):
        client = kwargs['client']
        hypervisors = novaclient(
            context,
            self._production,
            True).hypervisors.list()
        return hypervisors

    @synchronized(novalock)
    @exception_handler(ignore_exception=False, version='2.60')
    def create_server(
            self,
            context,
            name,
            image,
            flavor,
            meta=None,
            files=None,
            reservation_id=None,
            min_count=None,
            max_count=None,
            security_groups=None,
            userdata=None,
            key_name=None,
            availability_zone=None,
            block_device_mapping=None,
            nics=None,
            scheduler_hints=None,
            **kwargs):
        """
        Create (boot) a new server.
        :param name: Something to name the server.
        :param image: The :class:`Image` to boot with.
        :param flavor: The :class:`Flavor` to boot onto.
        :param production: If True, production Nova will be used.
        :param meta: A dict of arbitrary key/value metadata to store for this
                     server. A maximum of five entries is allowed, and both
                     keys and values must be 255 characters or less.
        :param files: A dict of files to overrwrite on the server upon boot.
                      Keys are file names (i.e. ``/etc/passwd``) and values
                      are the file contents (either as a string or as a
                      file-like object). A maximum of five entries is allowed,
                      and each file must be 10k or less.
        :param userdata: user data to pass to be exposed by the metadata
                      server this can be a file type object as well or a
                      string.
        :param reservation_id: a UUID for the set of servers being requested.
        :param key_name: (optional extension) name of previously created
                      keypair to inject into the instance.
        :param availability_zone: Name of the availability zone for instance
                                  placement.
        :param block_device_mapping: (optional extension) A dict of block
                      device mappings for this server.
        :param nics:  (optional extension) an ordered list of nics to be
                      added to this server, with information about
                      connected networks, fixed ips, port etc.
        :param scheduler_hints: (optional extension) arbitrary key-value pairs
                            specified by the client to help boot an instance
        :param config_drive: (optional extension) value for config drive
                            either boolean, or volume-id
        """

        client = kwargs['client']
        if userdata:
            try:
                userdata = base64.decode_as_text(userdata)
            except Exception as ex:
                LOG.debug("Failed to decode user_data")
                LOG.debug(ex)

        item = client.servers.create(
            name,
            image,
            flavor,
            meta=meta,
            files=files,
            reservation_id=reservation_id,
            min_count=min_count,
            max_count=max_count,
            security_groups=security_groups,
            userdata=userdata,
            key_name=key_name,
            availability_zone=availability_zone,
            block_device_mapping=block_device_mapping,
            nics=nics,
            scheduler_hints=scheduler_hints,
            **kwargs)
        time.sleep(15)
        return item

    def _get_servers(self, context, search_opts=None, admin=False, **kwargs):
        """
        Get all the servers for a particular tenant or all tenants
        :rtype: :class:`Server`
        """
        if search_opts is None:
            search_opts = {}
        if admin:
            search_opts['all_tenants'] = True
        else:
            search_opts['project_id'] = context.project_id

        servers = None
        client = kwargs['client']
        servers = client.servers.list(True, search_opts)

        return servers

    @exception_handler(ignore_exception=False)
    def get_servers(self, context, search_opts=None, admin=False, **kwargs):
        return self._get_servers(context, search_opts, admin, **kwargs)

    @exception_handler(ignore_exception=False)
    def get_server(self, context, name, admin=False, **kwargs):
        """
        Get the server given the name
        :rtype: :class:`Server`
        """
        server = None
        client = kwargs['client']
        client = novaclient(context, self._production, admin)
        return client.servers.find(name=name)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_security_group_by_id(self, context, secid, admin=False, **kwargs):
        """
        Get the security group given the name
        :rtype: :int:`secuirty id`
        """
        client = kwargs['client']
        return client.security_groups.get(secid)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_security_groups(self, context, admin=False, **kwargs):
        """
        Get the security group given the name
        :rtype: :int:`secuirty id`
        """
        client = kwargs['client']
        return client.security_groups.list()

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_server_raw_payload(self, context, id, **kwargs):
        try:
            client = kwargs['client']
            resp, body = client.servers.api.client.get("/servers/%s" % id)
            return resp.text
        except Exception as ex:
            LOG.exception(ex)
            return None

    @exception_handler(ignore_exception=False)
    def get_server_by_id(self, context, id, admin=False,
                         search_opts=None, **kwargs):
        """
        :param id to query.
        :rtype: :class:`Server`
        """
        if search_opts is None:
            servers = self._get_servers(
                context, search_opts, admin=admin, **kwargs)
            for server in servers:
                if server.id == id:
                    return server
            return None
        else:
            qparams = {}

            client = kwargs['client']
            for opt, val in six.iteritems(search_opts):
                if val:
                    qparams[opt] = val
                if qparams:
                    new_qparams = sorted(list(qparams.items()), key=lambda x: x[0])
                    query_string = "?%s" % parse.urlencode(new_qparams)
                else:
                    query_string = ""
                server = client.servers._get(
                    "/servers/%s%s" %
                    (id, query_string), "server")
                return server

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def stop(self, context, server, **kwargs):
        """
        Stop the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        return client.servers.stop(server=server)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def start(self, context, server, **kwargs):
        """
        Start the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        return client.servers.start(server=server)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def suspend(self, context, server, **kwargs):
        """
        Suspend the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        client = novaclient(context, self._production)
        return client.servers.suspend(server=server)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def reboot(self, context, server, reboot_type='SOFT', **kwargs):
        """
        Suspend the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        return client.servers.reboot(server=server, reboot_type=reboot_type)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def resume(self, context, server, **kwargs):
        """
        Resume the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        return client.servers.resume(server=server)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def pause(self, context, server, **kwargs):
        """
        Pause the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        return client.servers.pause(server=server)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def unpause(self, context, server, **kwargs):
        """
        UnPause the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        return client.servers.unpause(server=server)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def delete(self, context, server, **kwargs):
        """
        Delete the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        return client.servers.delete(server=server)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def force_delete(self, context, server, **kwargs):
        """
        Force Delete the server given the id
        :param server: The :class:`Server` (or its ID) to query.
        """

        client = kwargs['client']
        return client.servers.force_delete(server=server)

    @synchronized(novalock)
    @exception_handler(ignore_exception=True)
    def set_meta_item(self, context, server_id, key, value, **kwargs):
        """
        Adds a metadata item to the server given key value
        :param server: The :class:`Server` (or its ID) to query.
        """

        server = namedtuple('server', 'id')
        s = server(id=server_id)
        client = kwargs['client']

        return client.servers.set_meta_item(server=s, key=key, value=value)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def delete_meta(self, context, server_id, keys, **kwargs):
        """
        Delete metadata of the server given the server id and keys
        :param server: The :class:`Server` (or its ID) to query.
        :param keys: meta data keys
        """

        server = namedtuple('server', 'id')
        s = server(id=server_id)
        client = kwargs['client']

        return client.servers.delete_meta(server=s, keys=keys)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def list_security_group(self, context, server_id, **kwargs):
        """
        List security groups on the server
        :param server: The :class:`Server` (or its ID) to query.
        """

        server = namedtuple('server', 'id')
        s = server(id=server_id)
        client = kwargs['client']

        return client.servers.list_security_group(server=s)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def add_security_group(self, context, server_id,
                           security_group_id, **kwargs):
        """
        Add security group identified by security group id
        :param server: The :class:`Server` (or its ID) to query.
        :param security_group_id: Security group id
        """

        server = namedtuple('server', 'id')
        s = server(id=server_id)
        client = kwargs['client']

        return client.servers.add_security_group(
            server=s, security_group=security_group_id)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def remove_security_group(self, context, server_id,
                              security_group_id, **kwargs):
        """
        Removes a security group identified by the security group_id
        :param server: The :class:`Server` (or its ID) to query.
        :param security_group_id: Security group id
        """

        server = namedtuple('server', 'id')
        s = server(id=server_id)
        client = kwargs['client']

        return client.servers.remove_security_group(
            server=s, security_group=security_group_id)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def add_floating_ip(self, context, server_id,
                        floating_ip, fixed_ip, **kwargs):
        """
        Add floating ip to the server
        :param server: The :class:`Server` (or its ID) to query.
        :param floating_ip: Floating IP
        """

        server = namedtuple('server', 'id')
        s = server(id=server_id)
        client = kwargs['client']

        return client.servers.add_floating_ip(server=s, address=floating_ip,
                                              fixed_address=fixed_ip)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def floating_ip_list(self, context, **kwargs):
        """
        Add floating ip to the server
        :param server: The :class:`Server` (or its ID) to query.
        :param floating_ip: Floating IP
        """

        client = kwargs['client']

        return client.floating_ips.list()

    @synchronized(novalock)
    # Support Cinder multi-attach
    @exception_handler(ignore_exception=False, version='2.60') 
    def attach_volume(self, context, server_id, volume_id, device, **kwargs):
        """
        Attach a volume identified by the volume ID to the given server ID
        :param server_id: The ID of the server
        :param volume_id: The ID of the volume to attach.
        :param device: The device name
        :rtype: :class:`Volume`
        """

        client = kwargs['client']
        return client.volumes.create_server_volume(
            server_id, volume_id, device)

    # Support Cinder multi-attach
    @exception_handler(ignore_exception=False, version='2.60')
    def detach_volume(self, context, server_id, volume_id, attachment_id=None, **kwargs):
        """
        Detach a volume identified by the volume ID to the given server ID
        :param server_id: The ID of the server
        :param volume_id: The ID of the volume to attach.
        :rtype: :class:`Volume`
        """

        client = kwargs['client']
        return client.volumes.delete_server_volume(server_id, volume_id, attachment_id)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_image(self, context, id, **kwargs):
        """
        Get the image given the name
        :param name: name of the image
        :rtype: :class:`Image`
        """

        client = kwargs['client']
        return client.glance.find_image(id)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_flavors(self, context, is_public=True, **kwargs):
        """
        Get the list of flavors
        :param is_public: public flavors
        :rtype: :class:`Flavor`
        """
        client = kwargs['client']
        return client.flavors.list(is_public=is_public)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_flavor_by_name(self, context, name, **kwargs):
        """
        Get the flavors given the name
        :param name: name of the flavors
        :rtype: :class:`Flavor`
        """

        client = kwargs['client']
        return client.flavors.find(name=name)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_flavor_by_id(self, context, id, **kwargs):
        """
        Get the flavor given the id
        :param name: id of the flavors
        :rtype: :class:`Flavor`
        """
        client = kwargs['client']
        return client.flavors.get(id)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def create_flavor(self, context, name, memory, vcpus,
                      root_gb, ephemeral_gb, **kwargs):
        """
        Create a new flavor
        :rtype: :class:`Flavor`
        """

        client = kwargs['client']
        return client.flavors.create(name, memory, vcpus, root_gb,
                                     flavorid="auto", ephemeral=ephemeral_gb)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def delete_flavor(self, context, id, **kwargs):
        """
        Delete the falvor given the flavor name
        """

        client = kwargs['client']
        return client.flavors.delete(id)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_keypairs(self, context, **kwargs):
        """
        Get the list of keypairs
        :rtype: :class:`keypair`
        """
        client = kwargs['client']
        return client.keypairs.list()

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def delete_keypair(self, context, name, **kwargs):
        """
        Delete the keypair by name
        """
        client = kwargs['client']
        return client.keypairs.delete(name)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def create_keypair(self, context, name, public_key, key_type, **kwargs):
        """
        Create new keypairs
        """
        user_id = context.user_id if hasattr(context, 'user_id') else None
        # novaclient 2.10 supports to create keypair by providing it's type and user_id
        client = nova_micro_client(context, version='2.10')
        return client.keypairs.create(name, public_key=public_key, key_type=key_type, user_id=user_id)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_keypair_by_name(self, context, name, **kwargs):
        """
        Get the keypair given the name
        :param name: name of the keypair
        :rtype: :class:`keypair`
        """
        user_id = context.user_id if hasattr(context, 'user_id') else None
        # novaclient 2.10 supports to fetch keypair details of specified user_id
        client = nova_micro_client(context, version='2.10')
        return client.keypairs.get(name, user_id=user_id)

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_interfaces(self, context, server, **kwargs):
        """
        List attached network interfaces
        :param server: The :class:`Server` (or its ID) to query.
        """
        client = kwargs['client']
        try:
            return client.servers.interface_list(server=server)
        except nova_exception.HTTPNotImplemented:
            # This is configured to use nova network
            server = client.servers.get(server)
            return server._info['addresses']

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_networks(self, context, **kwargs):
        """
        Get the list of nova networks
        :param is_public: public networks
        :rtype: :class:`Network`
        """
        client = kwargs['client']
        return client.networks.list()

    @synchronized(novalock)
    @exception_handler(ignore_exception=False)
    def get_fixed_ip(self, context, ip, **kwargs):
        """
        Get the IP address information
        :param IP4 address:
        """
        client = kwargs['client']
        return client.fixed_ips.get(ip)

    def contego_service_status(self, context, host=None, ip=None):
        """
        Get contego service status running on a compute node
        """
        contego_service_info = {}
        all_services = []
        try:
            try:
                client = novaclient(context, self._production)

                if host == 'all' and ip == 'all':
                    all_services = client.services.list()
                elif host != 'all':
                    all_services = client.services.list(host=host)
                elif ip != 'all':
                    for hypervisor in client.hypervisors.list():
                        if hypervisor.host_ip == ip:
                            all_services = client.services.list(
                                host=hypervisor.hypervisor_hostname)

                for service in all_services:
                    if service.binary in 'contego':
                        contego_service_info[service.host] = ({"id": service.id,
                                                               "name": service.binary,
                                                               "status": service.status,
                                                               "running_state": service.state
                                                               })
                return contego_service_info
            except nova_exception.Unauthorized as unauth_ex:
                client = novaclient(context, self._production, admin=True)
                all_services = client.services.list()

                for service in all_services:
                    if service.binary in 'contego':
                        contego_service_info[service.host] = ({"id": service.id,
                                                               "name": service.binary,
                                                               "status": service.status,
                                                               "running_state": service.state
                                                               })
                return contego_service_info
            except Exception as ex:
                LOG.exception(ex)
        except Exception as ex:
            LOG.exception(ex)
            msg = 'Unable to get the status of contego service'
            raise exception.ErrorOccurred(msg)

    @synchronized(novalock)
    def get_server_user_data(self, context, server_id, version='2.3'):
        """
        Fetch user_data of the given server id
        :param context: context
        :param server_id: An ID to query.
        :param version: micro version for nova client
        """
        try:
            custom_nova_client = nova_micro_client(context, version)
            server = namedtuple('server', 'id')
            server_obj = server(id=server_id)
            server_obj = custom_nova_client.servers.get(server_obj)
            if server_obj:
                userdata_value = [value for key, value in server_obj.to_dict().items() if key.endswith("user_data")]
                if userdata_value:
                    return userdata_value[0] or None
            return None
        except Exception as ex:
            LOG.exception(ex)
            raise ex

    def get_server(self, context, server_id, version='2.1'):
        """
        Fetch host of the given server id
        :param context: context
        :param server_id: An ID to query.
        :param version: micro version for nova client
        """
        try:
            custom_nova_client = nova_micro_client(context, version)
            server = namedtuple('server', 'id')
            server_obj = server(id=server_id)
            nova_server_obj = None
            try:
                nova_server_obj = custom_nova_client.servers.get(server_obj)
            except nova_exception.NotFound as ex:
                # Case where VM is deleted
                search_opts={'uuid': server_id, 'deleted': True}
                nova_server_obj = custom_nova_client.servers.list(search_opts=search_opts)
                if nova_server_obj:
                    nova_server_obj = nova_server_obj[0]
            if nova_server_obj:
                return nova_server_obj
            else:
                raise Exception("Unable to fetch VM details for VM ID: %s.", server_id)

        except Exception as ex:
            LOG.exception(ex)
            raise ex