# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2013 TrilioData, Inc.
# All Rights Reserved.

"""Implementation of SQLAlchemy backend."""

from datetime import datetime, timedelta
import os
import uuid
import json
import warnings
import threading

from oslo_config import cfg

import sqlalchemy
import sqlalchemy.orm as sa_orm
import sqlalchemy.sql as sa_sql

from sqlalchemy import or_
from sqlalchemy.orm import joinedload
from sqlalchemy.sql.expression import literal_column, cast
from sqlalchemy.sql import func
from sqlalchemy import and_
from sqlalchemy import Integer
from sqlalchemy import asc, desc

from workloadmgr.common import sqlalchemyutils
from workloadmgr import db
from workloadmgr.db.sqlalchemy import models
from workloadmgr.db.sqlalchemy.session import get_session
from workloadmgr import exception
from workloadmgr import flags
from workloadmgr.openstack.common.gettextutils import _
from workloadmgr.openstack.common import log as logging
from workloadmgr.openstack.common import timeutils
from workloadmgr.openstack.common import uuidutils
from workloadmgr.apscheduler import job
from workloadmgr.vault import vault
from workloadmgr.virt import qemuimages
from workloadmgr.openstack.common.gettextutils import _

FLAGS = flags.FLAGS

LOG = logging.getLogger(__name__)

lock = threading.Lock()

CONF = cfg.CONF


def is_admin_context(context):
    """Indicates if the request context is an administrator."""
    if not context:
        warnings.warn(_('Use of empty request context is deprecated'),
                      DeprecationWarning)
        raise Exception('die')
    return context.is_admin


def is_user_context(context):
    """Indicates if the request context is a normal user."""
    if not context:
        return False
    if context.is_admin:
        return False
    if not context.user_id or not context.project_id:
        return False
    return True


def authorize_project_context(context, project_id):
    """Ensures a request has permission to access the given project."""
    if is_user_context(context):
        if not context.project_id:
            raise exception.NotAuthorized()
        elif context.project_id != project_id:
            raise exception.NotAuthorized()


def authorize_user_context(context, user_id):
    """Ensures a request has permission to access the given user."""
    if is_user_context(context):
        if not context.user_id:
            raise exception.NotAuthorized()
        elif context.user_id != user_id:
            raise exception.NotAuthorized()


def authorize_quota_class_context(context, class_name):
    """Ensures a request has permission to access the given quota class."""
    if is_user_context(context):
        if not context.quota_class:
            raise exception.NotAuthorized()
        elif context.quota_class != class_name:
            raise exception.NotAuthorized()


def require_admin_context(f):
    """Decorator to require admin request context.
    The first argument to the wrapped function must be the context.
    """

    def wrapper(*args, **kwargs):
        if not is_admin_context(args[0]):
            raise exception.AdminRequired()
        return f(*args, **kwargs)

    return wrapper


def require_context(f):
    """Decorator to require *any* user or admin context.
    This does no authorization for user or project access matching, see
    :py:func:`authorize_project_context` and
    :py:func:`authorize_user_context`.
    The first argument to the wrapped function must be the context.
    """

    def wrapper(*args, **kwargs):
        if not is_admin_context(args[0]) and not is_user_context(args[0]):
            raise exception.NotAuthorized()
        return f(*args, **kwargs)

    return wrapper


def model_query(context, *args, **kwargs):
    """Query helper that accounts for context's `read_deleted` field.
    :param context: context to query under
    :param session: if present, the session to use
    :param read_deleted: if present, overrides context's read_deleted field.
    :param project_only: if present and context is user-type, then restrict
            query to match the context's project_id.
    """
    session = kwargs.get('session') or get_session()
    read_deleted = kwargs.get('read_deleted')
    if read_deleted is None and context is not None:
        read_deleted = context.read_deleted
    project_only = kwargs.get('project_only')

    query = session.query(*args)
    if read_deleted == 'no':
        query = query.filter_by(deleted=False)
    elif read_deleted == 'yes':
        pass  # omit the filter to include deleted and active
    elif read_deleted == 'only':
        query = query.filter_by(deleted=True)
    else:
        raise Exception(
            _("Unrecognized read_deleted value '%s'") %
            read_deleted)
    if context:
        if project_only and project_only == 'yes' and is_user_context(context):
            query = query.filter_by(project_id=context.project_id)
        elif project_only and project_only == 'yes':
            query = query.filter_by(project_id=context.project_id)
    if 'get_hidden' in kwargs:
        if kwargs.get('get_hidden', False) == False:
            query = query.filter_by(hidden=False)

    return query


def exact_filter(query, model, filters, legal_keys):
    """Applies exact match filtering to a query.
    Returns the updated query.  Modifies filters argument to remove
    filters consumed.
    :param query: query to apply filters to
    :param model: model object the query applies to, for IN-style
                  filtering
    :param filters: dictionary of filters; values that are lists,
                    tuples, sets, or frozensets cause an 'IN' test to
                    be performed, while exact matching ('==' operator)
                    is used for other values
    :param legal_keys: list of keys to apply exact filtering to
    """

    filter_dict = {}

    # Walk through all the keys
    for key in legal_keys:
        # Skip ones we're not filtering on
        if key not in filters:
            continue

        # OK, filtering on this key; what value do we search for?
        value = filters.pop(key)

        if isinstance(value, (list, tuple, set, frozenset)):
            # Looking for values in a list; apply to query directly
            column_attr = getattr(model, key)
            query = query.filter(column_attr.in_(value))
        else:
            # OK, simple exact match; save for later
            filter_dict[key] = value

    # Apply simple exact matches
    if filter_dict:
        query = query.filter_by(**filter_dict)

    return query


#


@require_admin_context
def service_delete(context, service_id):
    session = get_session()
    with session.begin():
        session.query(models.Service).filter_by(id=service_id).delete()


@require_admin_context
def service_get(context, service_id):
    session = get_session()
    return _service_get(context, service_id, session)


@require_admin_context
def _service_get(context, service_id, session):
    result = model_query(
        context,
        models.Service,
        session=session). \
        filter_by(id=service_id). \
        first()
    if not result:
        raise exception.ServiceNotFound(service_id=service_id)

    return result


@require_admin_context
def service_get_all(context):
    session = get_session()
    return session.query(models.Service).all()


@require_admin_context
def service_get_all_by_topic(context, topic, read_disabled):
    session = get_session()
    if read_disabled:
        result_query = model_query(
            context, models.Service, session=session, read_deleted="no"). \
            filter_by(topic=topic)
    else:
        result_query = model_query(
            context, models.Service, session=session, read_deleted="no"). \
            filter_by(disabled=False). \
            filter_by(topic=topic)
    return result_query.all()


@require_admin_context
def service_get_by_host_and_topic(context, host, topic):
    session = get_session()
    result = model_query(
        context, models.Service, session=session, read_deleted="no"). \
        filter_by(host=host). \
        filter_by(topic=topic). \
        first()
    if not result:
        raise exception.ServiceNotFoundOnHost(topic=topic, host=host)
    return result


@require_admin_context
def service_get_all_by_host(context, host):
    session = get_session()
    return model_query(
        context, models.Service, session=session, read_deleted="no"). \
        filter_by(host=host). \
        all()


@require_admin_context
def _service_get_all_topic_subquery(context, session, topic, subq, label):
    sort_value = getattr(subq.c, label)
    return model_query(context, models.Service,
                       func.coalesce(sort_value, 0),
                       session=session, read_deleted="no"). \
        filter_by(topic=topic). \
        filter_by(disabled=False). \
        outerjoin((subq, models.Service.host == subq.c.host)). \
        order_by(sort_value). \
        all()


@require_admin_context
def service_get_by_args(context, host, binary):
    session = get_session()
    result = model_query(context, models.Service, session=session, ). \
        filter_by(host=host). \
        filter_by(binary=binary). \
        filter_by(version=models.DB_VERSION). \
        first()

    if not result:
        raise exception.HostBinaryNotFound(host=host, binary=binary)

    return result


@require_admin_context
def service_create(context, values):
    session = get_session()
    service_ref = models.Service()
    service_ref.update(values)
    if not FLAGS.enable_new_services:
        service_ref.disabled = True
    service_ref.save()
    return service_ref


@require_admin_context
def service_update(context, service_id, values):
    session = get_session()
    with session.begin():
        service_ref = _service_get(context, service_id, session=session)
        service_ref.update(values)
        service_ref.save(session=session)


# File search ###############

@require_context
def file_search_get_all(context, **kwargs):
    status = kwargs.get('status', None)
    time_in_minutes = kwargs.get('time_in_minutes', None)
    host = kwargs.get('host', None)
    vm_id = kwargs.get('vm_id', None)
    query = model_query(context, models.FileSearch, **kwargs)
    query = query.filter_by(project_id=context.project_id)
    if time_in_minutes is not None:
        now = timeutils.utcnow()
        minutes_ago = now - timedelta(minutes=int(time_in_minutes))
        query = query.filter(models.FileSearch.created_at < minutes_ago)
    if vm_id is not None:
        query = query.filter_by(vm_id=vm_id)
    if host is not None:
        query = query.filter(or_(models.FileSearch.host ==
                                 host, models.FileSearch.host is None))
    if status is not None:
        query = query.filter(
            and_(
                models.FileSearch.status != status,
                models.FileSearch.status != 'error'))
    return query.all()


@require_context
def file_search_delete(context, search_id):
    session = get_session()
    with session.begin():
        ref = _file_search_get(context, search_id, session=session)
        session.delete(ref)


@require_context
def file_search_get(context, search_id):
    session = get_session()
    return _file_search_get(context, search_id, session)


@require_context
def _file_search_get(context, search_id, session):
    result = model_query(
        context,
        models.FileSearch,
        session=session). \
        filter_by(project_id=context.project_id). \
        filter_by(id=search_id). \
        first()
    if not result:
        raise exception.FileSearchNotFound(search_id=search_id)

    return result


@require_context
def file_search_create(context, values):
    session = get_session()
    ref = models.FileSearch()
    ref.update(values)
    ref.save()
    return ref


@require_context
def file_search_update(context, search_id, values):
    session = get_session()
    with session.begin():
        ref = _file_search_get(context, search_id, session=session)
        ref.update(values)
        ref.save(session=session)


# Work load Types #################
""" workload_type functions """


@require_context
def _set_metadata_for_workload_type(context, workload_type_ref, metadata,
                                    purge_metadata, session):
    """
    Create or update a set of workload_type_metadata for a given workload_type
    :param context: Request context
    :param workload_type_ref: An workload_type object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in workload_type_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'workload_type_id': workload_type_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _workload_type_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _workload_type_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _workload_type_metadata_delete(context, metadata_ref, session)


@require_context
def _workload_type_metadata_create(context, values, session):
    """Create an WorkloadTypeMetadata object"""
    metadata_ref = models.WorkloadTypeMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _workload_type_metadata_update(
        context, metadata_ref, values, session)


@require_context
def workload_type_metadata_create(context, values):
    """Create an WorkloadTypeMetadata object"""
    session = get_session()
    return _workload_type_metadata_create(context, values, session)


@require_context
def _workload_type_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by workload_type_metadata_create and workload_type_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _workload_type_metadata_delete(context, metadata_ref, session):
    """
    Used internally by workload_type_metadata_create and workload_type_metadata_update
    """
    metadata_ref.delete(session=session)
    return metadata_ref


@require_context
def _workload_type_update(
        context, values, workload_type_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if workload_type_id:
        workload_type_ref = workload_type_get(
            context, workload_type_id, session)
    else:
        workload_type_ref = models.WorkloadTypes()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    workload_type_ref.update(values)
    workload_type_ref.save(session)

    _set_metadata_for_workload_type(
        context,
        workload_type_ref,
        metadata,
        purge_metadata,
        session)

    return workload_type_ref


@require_context
def workload_type_create(context, values):
    session = get_session()
    return _workload_type_update(context, values, None, False, session)


@require_context
def workload_type_update_all(context, values):
    session = get_session()
    if 'project_id' not in values:
        values['project_id'] = context.project_id
    if 'user_id' not in values:
        values['user_id'] = context.user_id
    with session.begin():
        session.query(models.WorkloadTypes). \
            filter_by(deleted=False). \
            update(values)


@require_context
def workload_type_update(context, id, values, purge_metadata=False):
    session = get_session()
    return _workload_type_update(context, values, id, purge_metadata, session)


@require_context
def workload_types_get(context):
    session = get_session()
    try:
        query = model_query(
            context,
            models.WorkloadTypes,
            session=session,
            read_deleted="no").options(
            sa_orm.joinedload(
                models.WorkloadTypes.metadata)).filter(
            (models.WorkloadTypes.project_id == context.project_id) | (
                models.WorkloadTypes.is_public))

        # TODO(gbasava): filter out deleted workload_types if context disallows
        # it
        workload_types = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.WorkloadTypesNotFound()

    return workload_types


@require_context
def workload_type_get(context, id):
    session = get_session()
    try:
        query = session.query(
            models.WorkloadTypes).options(
            sa_orm.joinedload(
                models.WorkloadTypes.metadata)).filter_by(
            id=id)

        # TODO(gbasava): filter out deleted workload_types if context disallows
        # it
        workload_types = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.WorkloadTypeNotFound(workload_type_id=id)

    if workload_types is None:
        raise exception.WorkloadTypeNotFound(workload_type_id=id)

    return workload_types


@require_context
def workload_type_delete(context, id):
    session = get_session()
    with session.begin():
        session.query(models.WorkloadTypes). \
            filter_by(id=id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


# Workloads ################################################################
""" workload functions """


def _set_metadata_for_workload(context, workload_ref, metadata,
                               purge_metadata, session):
    """
    Create or update a set of workload_metadata for a given workload
    :param context: Request context
    :param workload_ref: An workload object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in workload_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'workload_id': workload_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _workload_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _workload_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _workload_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _workload_metadata_create(context, values, session):
    """Create an WorkloadMetadata object"""
    metadata_ref = models.WorkloadMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _workload_metadata_update(context, metadata_ref, values, session)


@require_context
def workload_metadata_create(context, values, session):
    """Create an WorkloadMetadata object"""
    session = get_session()
    return _workload_metadata_create(context, values, session)


@require_context
def _workload_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by workload_metadata_create and workload_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _workload_metadata_delete(context, metadata_ref, session):
    """
    Used internally by workload_metadata_create and workload_metadata_update
    """
    metadata_ref.delete(session=session)


def _workload_update(context, values, workload_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if workload_id:
        workload_ref = _workload_get(context, workload_id, session)
    else:
        workload_ref = models.Workloads()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    workload_ref.update(values)
    workload_ref.save(session)

    if metadata:
        _set_metadata_for_workload(
            context,
            workload_ref,
            metadata,
            purge_metadata,
            session=session)

    return workload_ref


@require_context
def workload_create(context, values):
    session = get_session()
    return _workload_update(context, values, None, False, session)


@require_context
def workload_update(context, id, values, purge_metadata=False):
    session = get_session()
    return _workload_update(context, values, id, purge_metadata, session)


@require_context
def workload_get_all(context, **kwargs):
    qs = model_query(context, models.Workloads, **kwargs). \
        options(sa_orm.joinedload(models.Workloads.metadata))

    if is_admin_context(context):
        if 'nfs_share' in kwargs and kwargs['nfs_share'] is not None and kwargs['nfs_share'] != '':
            qs = qs.filter(
                and_(
                    models.Workloads.metadata.any(
                        models.WorkloadMetadata.key.in_(
                            ['backup_media_target'])), models.Workloads.metadata.any(
                        models.WorkloadMetadata.value.in_(
                            [
                                kwargs['nfs_share']]))))
        else:
            if 'dashboard_item' in kwargs:
                if kwargs.get('dashboard_item') == 'activities':
                    if 'time_in_minutes' in kwargs:
                        time_in_minutes = int(kwargs.get('time_in_minutes'))
                    else:
                        time_in_minutes = 0
                    time_delta = ((time_in_minutes / 60) / 24) * -1
                    qs = model_query(
                        context,
                        models.Workloads.id,
                        models.Workloads.deleted,
                        models.Workloads.deleted_at,
                        models.Workloads.display_name,
                        models.Workloads.status,
                        models.Workloads.created_at,
                        models.Workloads.user_id,
                        models.Workloads.project_id,
                        **kwargs).filter(
                        or_(
                            models.Workloads.created_at > func.adddate(
                                func.now(),
                                time_delta),
                            models.Workloads.deleted_at > func.adddate(
                                func.now(),
                                time_delta)))

        if 'project_id' in kwargs and kwargs['project_id'] is not None and kwargs['project_id'] != '':
            qs = qs.filter_by(project_id=kwargs['project_id'])
        elif 'all_workloads' in kwargs and kwargs['all_workloads'] is not True:
            qs = qs.filter_by(project_id=context.project_id)

    else:
        qs = qs.filter_by(project_id=context.project_id)
    if 'project_list' in kwargs and 'user_list' in kwargs:
        project_list = kwargs['project_list']
        user_list = kwargs['user_list']
        if isinstance(project_list, list) and isinstance(user_list, list):
            if 'exclude' in kwargs and kwargs['exclude'] is True:
                qs = qs.filter(models.Workloads.project_id.notin_(
                    project_list) | models.Workloads.user_id.notin_(user_list))
            else:
                qs = qs.filter(
                    models.Workloads.project_id.in_(project_list),
                    models.Workloads.user_id.in_(user_list))
        else:
            error = _('Project list and user list should be list')
            raise exception.ErrorOccurred(reason=error)

    if 'project_list' in kwargs and 'user_list' not in kwargs:
        project_list = kwargs['project_list']
        qs = model_query(context, models.Workloads, **kwargs)
        if isinstance(project_list, list):
            if 'exclude_project' in kwargs and kwargs['exclude_project'] is True:
                qs = qs.filter(
                    (models.Workloads.project_id.notin_(project_list)))
            else:
                qs = qs.filter((models.Workloads.project_id.in_(project_list)))
        else:
            error = _('Project list should be list')
            raise exception.ErrorOccurred(reason=error)

    if 'workload_list' in kwargs:
        workload_list = kwargs['workload_list']
        if isinstance(workload_list, list):
            if len(workload_list):
                if 'exclude_workload' in kwargs and kwargs['exclude_workload'] is True:
                    qs = qs.filter(and_(models.Workloads.id.notin_(workload_list)))
                else:
                    qs = qs.filter(and_(models.Workloads.id.in_(workload_list)))
        else:
            error = _('Workload list should be list')
            raise exception.ErrorOccurred(reason=error)

    qs = qs.order_by(models.Workloads.created_at.desc())

    if 'page_number' in kwargs and kwargs['page_number'] is not None and kwargs['page_number'] != '':
        page_size = setting_get(context, 'page_size')
        return qs.limit(int(page_size)).offset(
            int(page_size) * (int(kwargs['page_number']) - 1)).all()
    else:
        return qs.all()


@require_context
def _workload_get(context, wl_id, session, **kwargs):
    try:
        workload = model_query(
            context, models.Workloads, session=session, **kwargs). \
            options(sa_orm.joinedload(models.Workloads.metadata)). \
            filter_by(id=wl_id).all()

        # TODO(gbasava): filter out deleted workloads if context disallows it

        if not workload:
            raise exception.WorkloadNotFound(workload_id=wl_id)
        return workload[0]
    except sa_orm.exc.NoResultFound:
        raise exception.WorkloadNotFound(workload_id=wl_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.WorkloadNotFound(workload_id=wl_id)

@require_context
def _workload_import_get(context, jobid, wl_id, session, **kwargs):
    try:
        wl_import = model_query(
            context, models.WorkloadImport, session=session, **kwargs). \
            filter_by(workload_id=wl_id). \
            filter_by(jobid=jobid)

        # TODO(gbasava): filter out deleted workloads if context disallows it

        if not wl_import:
            raise exception.WorkloadNotFound(workload_id=wl_id)
        return wl_import
    except sa_orm.exc.NoResultFound:
        raise exception.WorkloadNotFound(workload_id=wl_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.WorkloadNotFound(workload_id=wl_id)


@require_admin_context
def get_workload_count(context, project_id):
    session = get_session()
    with session.begin():
        return session.query(models.Workloads).filter_by(
            project_id=project_id, deleted=False
        ).count()


@require_context
def workload_get(context, id, **kwargs):
    session = get_session()
    with session.begin():
        return _workload_get(context, id, session, **kwargs)


@require_context
def workload_delete(context, id):
    session = get_session()
    with session.begin():
        session.query(models.Workloads). \
            filter_by(id=id).delete()

# WorkloadVMs #########################################################
""" workload_vms functions """


def _set_metadata_for_workload_vms(context, workload_vm_ref, metadata,
                                   purge_metadata, session):
    """
    Create or update a set of workload_vms_metadata for a given workload_vm
    :param context: Request context
    :param workload_vm_ref: An workload_vm object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in workload_vm_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'workload_vm_id': workload_vm_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _workload_vms_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _workload_vms_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _workload_vms_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _workload_vms_metadata_create(context, values, session):
    """Create an WorkloadMetadata object"""
    metadata_ref = models.WorkloadVMMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _workload_vms_metadata_update(
        context, metadata_ref, values, session)


@require_context
def workload_vms_metadata_create(context, values, session):
    """Create an WorkloadMetadata object"""
    session = get_session()
    return _workload_vms_metadata_create(context, values, session)


@require_context
def _workload_vms_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by workload_vms_metadata_create and workload_vms_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _workload_vms_metadata_delete(context, metadata_ref, session):
    """
    Used internally by workload_vms_metadata_create and workload_vms_metadata_update
    """
    metadata_ref.delete(session=session)
    return metadata_ref


def _workload_vms_update(context, values, id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if id:
        workload_vm_ref = _workload_vm_get(context, id, session)
    else:
        workload_vm_ref = models.WorkloadVMs()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    workload_vm_ref.update(values)
    workload_vm_ref.save(session)

    if metadata:
        _set_metadata_for_workload_vms(
            context,
            workload_vm_ref,
            metadata,
            purge_metadata,
            session=session)

    return workload_vm_ref


@require_context
def workload_vms_create(context, values):
    session = get_session()
    return _workload_vms_update(context, values, None, False, session)


@require_context
def workload_vms_update(context, id, values, purge_metadata=False):
    session = get_session()
    return _workload_vms_update(context, values, id, purge_metadata, session)


@require_context
def workload_vms_get(context, workload_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        if workload_id:
            query = model_query(context, models.WorkloadVMs,
                                session=session, read_deleted="no") \
                .options(sa_orm.joinedload(models.WorkloadVMs.metadata)) \
                .filter_by(workload_id=workload_id) \
                .filter(models.WorkloadVMs.status is not None)
        else:
            query = model_query(context, models.WorkloadVMs,
                                session=session, read_deleted="no") \
                .options(sa_orm.joinedload(models.WorkloadVMs.metadata)) \
                .filter(models.WorkloadVMs.status is not None)

        if kwargs.get('workload_list'):
            query = query.filter(models.WorkloadVMs.workload_id.in_(kwargs['workload_list']))
        # TODO(gbasava): filter out deleted workload_vms if context disallows
        # it
        workload_vms = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.WorkloadVMsNotFound(workload_id=workload_id)

    return workload_vms


@require_context
def _import_job_get(context, jobid, session):
    session = get_session()
    with session.begin():
        res = session.query(models.Job). \
            filter_by(jobid=jobid)
    return res

@require_context
def import_job_create(context, values):
    session = get_session()
    with session.begin():
        job_ref = models.Job()
        job_ref.created_at = datetime.now()
        job_ref.progress = 0
        job_ref.status = values.get('status')
        job_ref.action = values.get('action')
        job_ref.save(session)
    return job_ref.jobid

@require_context
def import_job_update(context, jobid, values):
    session = get_session()
    if jobid:
        job_ref = _import_job_get(context, jobid, session)
        job_ref.update(values)

@require_context
def import_job_get(context, jobid):
    session = get_session()
    if jobid:
        return _import_job_get(context, jobid)


@require_context
def workload_import_get_all(context, jobid):
    return importable_workloads_get_all(context, jobid=jobid)


@require_context
def workload_import_update(context, job_data_id, values):
    session = get_session()
    job_data_ref =  model_query(
                       context, models.JobDetails, session=session).filter_by(id=job_data_id).first()
    if not job_data_ref:
        raise exception.JobDataNotFound(job_data_id=job_data_id)
    db_values = dict(json.loads(job_data_ref.data))
    db_values.update(values)
    job_data_ref.update({'jobid': job_data_ref.jobid, 'data':json.dumps(db_values)})
    job_data_ref.save(session)
    return job_data_ref


@require_context
def workload_import_create(context, values):
    session = get_session()
    jobid = values.get('jobid')
    return importable_workloads_create(context, jobid, {'workload_id': values.get('workload_id'), 'created_at': str(datetime.utcnow()), 'progress': 0, 'status': 'created'})


@require_context
def import_job_get(context, jobid, **kwargs):
    session = get_session()
    job = None
    with session.begin():
        job = model_query(
            context, models.Job, session=session, **kwargs). \
            filter_by(jobid=jobid).all()
    if not job:
        raise exception.JobNotFound(jobid=jobid)
    return job[0]


@require_admin_context
def get_vms_count_by_project_id(context, project_id):
    session = get_session()
    with session.begin():
        return session.query(
            models.Workloads.project_id, models.WorkloadVMs.vm_id).join(
            models.WorkloadVMs).filter(
            and_(models.Workloads.project_id == project_id,
                 models.Workloads.deleted == False,
                 models.WorkloadVMs.deleted == False)).count()


@require_context
def workload_vm_get_by_id(context, vm_id, **kwargs):
    session = kwargs.get('session') or get_session()
    read_deleted = 'no'
    if 'read_deleted' in kwargs:
        read_deleted = kwargs['read_deleted']
    try:
        query = model_query(context, models.WorkloadVMs,
                            session=session, read_deleted=read_deleted) \
            .options(sa_orm.joinedload(models.WorkloadVMs.metadata)) \
            .join(models.Workloads) \
            .filter(models.WorkloadVMs.status is not None) \
            .filter(models.WorkloadVMs.vm_id == vm_id) \
            .filter(models.Workloads.project_id == context.project_id)

        if 'workloads_filter' in kwargs:
            query = query.filter(
                and_(
                    models.Workloads.status != kwargs['workloads_filter'],
                    models.Workloads.status is not None))

        vm_found = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.WorkloadVMsNotFound(vm_id=vm_id)

    return vm_found


@require_context
def _workload_vm_get(context, id, session):
    try:
        query = session.query(
            models.WorkloadVMs).options(
            sa_orm.joinedload(
                models.WorkloadVMs.metadata)).filter_by(
            id=id)
        # TODO(gbasava): filter out deleted workload_vms if context disallows
        # it
        workload_vm = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.WorkloadVMNotFound(workload_vm_id=id)

    return workload_vm


@require_context
def workload_vm_get(context, id):
    session = get_session()
    return _workload_vm_get(context, id, session)


@require_context
def workload_vms_delete(context, vm_id, workload_id):
    session = get_session()
    with session.begin():
        session.query(models.WorkloadVMs). \
            filter_by(vm_id=vm_id). \
            filter_by(workload_id=workload_id).delete()


@require_context
def restored_instance_get(context, instance_id, **kwargs):
    """"
    Return list of models.WorkloadVMs which are using
    vms restored from given instance_id
    """
    session = get_session()
    query = session.query(models.RestoredVMs.vm_id).join(models.RestoredVMMetadata).filter(
        and_(
            models.RestoredVMMetadata.key == 'instance_id'), models.RestoredVMMetadata.value == instance_id,
                                                             models.RestoredVMs.deleted == False)
    restored_vms = [vm.vm_id for vm in query.all()]
    restored_vms.append(instance_id)
    query = session.query(models.WorkloadVMs).filter(
        and_(models.WorkloadVMs.vm_id.in_(restored_vms), models.WorkloadVMs.deleted == False))
    return query.all()


#


@require_admin_context
def unlock_workloads_for_host(context, host):
    session = get_session()
    workloads_query = model_query(context, models.Workloads, session=session)
    workloads_query = workloads_query.filter(and_(
        models.Workloads.status == 'locked',
        models.Workloads.host == host)
    )
    locked_workloads = workloads_query.all()
    for wrklod in locked_workloads:
        values = {'status': 'available'}
        wrklod.update(values)
        session.add(wrklod)
    session.flush()
    session.close()


@require_admin_context
def workloads_mark_deleting_as_error(context, host):
    session = get_session()
    workloads_query = model_query(context, models.Workloads, session=session)
    workloads_query = workloads_query.filter(and_(
        models.Workloads.status == 'deleting',
        models.Workloads.host == host)
    )
    deleting_workloads = workloads_query.all()
    for wrklod in deleting_workloads:
        values = {'status': 'error'}
        wrklod.update(values)
        session.add(wrklod)
    session.flush()
    session.close()


@require_admin_context
def snapshot_mark_incomplete_as_error(context, host):
    """
    mark the snapshots that are left hanging from previous run on host as 'error'
    """
    session = get_session()
    now = timeutils.utcnow()
    snapshots_query = model_query(context, models.Snapshots, session=session)
    snapshots_query = snapshots_query.filter(and_(
        models.Snapshots.status != 'available',
        models.Snapshots.status != 'error',
        models.Snapshots.status != 'mounted',
        models.Snapshots.status != 'cancelled'))

    snapshots_query = snapshots_query.filter(or_(
        models.Snapshots.host == host,
        models.Snapshots.host == ''))
    snapshots = snapshots_query.all()

    for snapshot in snapshots:
        if (snapshot.host == '' and
                now - snapshot.created_at <= timedelta(minutes=60)):
            continue

        if snapshot.status == 'restoring':
            values = {'status': 'available'}
        else:
            values = {'progress_percent': 100, 'progress_msg': '',
                      'error_msg': 'Snapshot did not finish successfully',
                      'status': 'error'}
        snapshot.update(values)
        session.add(snapshot)
    session.flush()
    session.close()


# Snapshot ################################################################
""" snapshot functions """


def _set_metadata_for_snapshot(context, snapshot_ref, metadata,
                               purge_metadata, session):
    """
    Create or update a set of snapshot_metadata for a given snapshot
    :param context: Request context
    :param snapshot_ref: A snapshot object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in snapshot_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'snapshot_id': snapshot_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _snapshot_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _snapshot_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _snapshot_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _snapshot_metadata_create(context, values, session):
    """Create a SnapshotMetadata object"""
    metadata_ref = models.SnapshotMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _snapshot_metadata_update(context, metadata_ref, values, session)


@require_context
def snapshot_metadata_create(context, values, session):
    """Create an SnapshotMetadata object"""
    session = get_session()
    return _snapshot_metadata_create(context, values, session)


@require_context
def _snapshot_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by snapshot_metadata_create and snapshot_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _snapshot_metadata_delete(context, metadata_ref, session):
    """
    Used internally by snapshot_metadata_create and snapshot_metadata_update
    """
    metadata_ref.delete(session=session)


def _snapshot_update(context, values, snapshot_id, purge_metadata, session):
    try:
        lock.acquire()
        metadata = values.pop('metadata', {})

        if snapshot_id:
            snapshot_ref = model_query(
                context,
                models.Snapshots,
                session=session,
                read_deleted="yes").filter_by(
                id=snapshot_id).first()
            if not snapshot_ref:
                lock.release()
                raise exception.SnapshotNotFound(snapshot_id=snapshot_id)

            if not values.get('uploaded_size'):
                if values.get('uploaded_size_incremental'):
                    values['uploaded_size'] = snapshot_ref.uploaded_size + \
                                              values.get('uploaded_size_incremental')
                    if not values.get(
                            'progress_percent') and snapshot_ref.size > 0:
                        values['progress_percent'] = min(
                            99, (100 * values.get('uploaded_size')) / snapshot_ref.size)
        else:
            snapshot_ref = models.Snapshots()
            if not values.get('id'):
                values['id'] = str(uuid.uuid4())
            if not values.get('size'):
                values['size'] = 0
            if not values.get('restore_size'):
                values['restore_size'] = 0
            if not values.get('uploaded_size'):
                values['uploaded_size'] = 0
            if not values.get('progress_percent'):
                values['progress_percent'] = 0
        snapshot_ref.update(values)
        snapshot_ref.save(session)

        if metadata:
            _set_metadata_for_snapshot(
                context,
                snapshot_ref,
                metadata,
                purge_metadata,
                session=session)

        return snapshot_ref
    finally:
        lock.release()
    return snapshot_ref


@require_context
def _snapshot_get(context, snapshot_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    result = model_query(context, models.Snapshots, **kwargs). \
        options(sa_orm.joinedload(models.Snapshots.metadata)). \
        filter_by(id=snapshot_id). \
        first()

    if not result:
        raise exception.SnapshotNotFound(snapshot_id=snapshot_id)

    return result


@require_admin_context
def get_active_snapshot_count(context, project_id):
    session = get_session()
    with session.begin():
        return session.query(models.Snapshots).filter(and_(
            models.Snapshots.project_id==project_id,
            models.Snapshots.deleted==False,
            models.Snapshots.status.notin_(['error', 'cancelled'])
        )).count()


@require_context
def snapshot_get(context, snapshot_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    return _snapshot_get(context, snapshot_id, **kwargs)


@require_context
def snapshot_get_metadata_cancel_flag(
        context, snapshot_id, return_val=0, process=None, **kwargs):
    flag = '0'
    snapshot_obj = snapshot_get(context, snapshot_id)
    for meta in snapshot_obj.metadata:
        if meta.key == 'cancel_requested':
            flag = meta.value

    if return_val == 1:
        return flag

    if flag == '1':
        if process:
            process.kill()
        error = _('Cancel requested for snapshot')
        raise exception.ErrorOccurred(reason=error)


@require_context
def snapshot_get_running_snapshots_by_host(context, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()

    result = model_query(context,
                         models.Snapshots.host,
                         func.count(models.Snapshots.host),
                         **kwargs).filter(
        and_(~models.Snapshots.status.in_(['available',
                                           'error',
                                           'deleted',
                                           'cancelled'])),
        models.Snapshots.host != '').group_by(models.Snapshots.host).all()
    return result


@require_context
def migration_get_running_migrations_by_host(context, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()

    result = model_query(context,
                         models.Migrations.host,
                         func.count(models.Migrations.host),
                         **kwargs).filter(
        and_(~models.Migrations.status.in_(['available',
                                           'error',
                                           'deleted',
                                           'cancelled'])),
        models.Migrations.host != '').group_by(models.Migrations.host).all()
    return result


@require_context
def snapshot_get_all(context, **kwargs):
    qs = model_query(context, models.Snapshots, **kwargs)
    if kwargs.get('read_metadata', True):
        qs = qs.options(sa_orm.joinedload(models.Snapshots.metadata))
    elif kwargs.get('read_restores', True):
        qs = qs.options(sa_orm.joinedload(models.Snapshots.restores))
    if 'workload_id' in kwargs and kwargs['workload_id'] is not None and kwargs['workload_id'] != '':
        qs = qs.filter_by(workload_id=kwargs['workload_id'])
    if kwargs.get('project_id'):
        qs = qs.filter_by(project_id=kwargs['project_id'])
    elif kwargs.get('project_list'):
        qs = qs.filter(models.Snapshots.project_id.in_(kwargs['project_list']))
    if 'host' in kwargs and kwargs['host'] is not None and kwargs['host'] != '':
        qs = qs.filter(models.Snapshots.host == kwargs['host'])
    if 'date_from' in kwargs and kwargs['date_from'] is not None and kwargs['date_from'] != '':
        if 'date_to' in kwargs and kwargs['date_to'] is not None and kwargs['date_to'] != '':
            date_to = kwargs['date_to']
        else:
            date_to = datetime.now()
        qs = qs.filter(
            and_(
                models.Snapshots.created_at >= func.date_format(
                    kwargs['date_from'],
                    '%y-%m-%dT%H:%i:%s'),
                models.Snapshots.created_at <= func.date_format(
                    date_to,
                    '%y-%m-%dT%H:%i:%s')))

    if not is_admin_context(context):
        qs = qs.filter_by(project_id=context.project_id)
    else:
        if 'get_all' in kwargs and kwargs['get_all'] is not True:
            qs = qs.filter_by(project_id=context.project_id)
    if 'status' in kwargs and kwargs['status'] is not None:
        if kwargs['status'] == 'available':
            qs = qs.filter(or_(models.Snapshots.status ==
                               'available', models.Snapshots.status == 'mounted'))
        elif kwargs['status'] == 'running':
            """ All other status except available/mounted/error/cancelled will be considered as Running snapshot by any of the Node."""
            qs = qs.filter(and_(models.Snapshots.status != 'available',
                                models.Snapshots.status != 'mounted', 
                                models.Snapshots.status != 'error',
                                models.Snapshots.status != 'cancelled',
                                models.Snapshots.status != 'deleting',
                                models.Snapshots.status != 'restoring',
                                models.Snapshots.status != 'mounting'))
        else:
            qs = qs.filter_by(status=kwargs['status'])

    qs = qs.order_by(models.Snapshots.created_at.desc())
    if 'end' in kwargs and kwargs['end'] != 0:
        qs = qs.limit(kwargs['end'])
    if 'start' in kwargs and kwargs['start'] != 0:
        qs = qs.offset(kwargs['start'])

    if 'get_instances' in kwargs and kwargs['get_instances'] is True:
        snapshots = qs.all()
        i = 0
        for snapshot in snapshots:
            instances = []
            snapshot_vms = snapshot_vms_get(context, snapshot.id)
            for snapshot_vm in snapshot_vms:
                instances.append(snapshot_vm)
            snapshots[i].instances = instances
            i = i + 1
        return snapshots
    return qs.all()


@require_context
def snapshot_get_all_by_workload(context, workload_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()

    return model_query(context, models.Snapshots, **kwargs). \
        options(sa_orm.joinedload(models.Snapshots.metadata)). \
        filter_by(workload_id=workload_id). \
        order_by(models.Snapshots.created_at.desc()).all()


@require_context
def snapshot_get_all_by_project(context, project_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    authorize_project_context(context, project_id)
    return model_query(context, models.Snapshots, **kwargs). \
        options(sa_orm.joinedload(models.Snapshots.metadata)). \
        filter_by(project_id=project_id).all()


@require_context
def snapshot_get_all_by_project_workload(
        context, project_id, workload_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    authorize_project_context(context, project_id)
    return model_query(context, models.Snapshots, **kwargs). \
        options(sa_orm.joinedload(models.Snapshots.metadata)). \
        filter_by(project_id=project_id). \
        filter_by(workload_id=workload_id). \
        order_by(models.Snapshots.created_at.desc()).all()


@require_context
def snapshot_show(context, snapshot_id, **kwargs):
    session = get_session()
    result = model_query(
        context,
        models.Snapshots,
        session=session,
        **kwargs).options(
        sa_orm.joinedload(
            models.Snapshots.metadata)).filter_by(
        id=snapshot_id).first()

    if not result:
        raise exception.SnapshotNotFound(snapshot_id=snapshot_id)

    return result


@require_context
def snapshot_create(context, values):
    session = get_session()
    return _snapshot_update(context, values, None, False, session)


@require_context
def snapshot_update(context, snapshot_id, values, purge_metadata=False):
    session = get_session()
    return _snapshot_update(context, values, snapshot_id,
                            purge_metadata, session)


def snapshot_next(context, snapshot_id, workload_id, **kwargs):
    """

    Args:
        context: HTTPRequest Object
        snapshot_id: int<snapshot_id>
        workload_id: int<snapshot_id>
        **kwargs: dict

    Returns: Quesryset Object.

    """
    session = get_session()

    qs_1 = session.query(models.Snapshots). \
        filter(models.Snapshots.workload_id == workload_id). \
        filter(models.Snapshots.created_at > (session.query(models.Snapshots).
                                              filter(models.Snapshots.workload_id == workload_id).
                                              filter(models.Snapshots.id == snapshot_id).
                                              order_by(asc(models.Snapshots.created_at)).first()).
               __dict__.get('created_at')). \
        order_by(asc(models.Snapshots.created_at)).limit(2)

    if not qs_1:
        raise exception.SnapshotNotFound(snapshot_id=snapshot_id)

    return qs_1


@require_context
def snapshot_prev(context, snapshot_id, workload_id, **kwargs):
    """

    Args:
        context: HTTPRequest Object.
        snapshot_id: int<snapshot_id>
        workload_id: int<snapshot_id>
        **kwargs: dict

    Returns: Queryset Object.

    """
    session = get_session()

    qs_1 = session.query(models.Snapshots). \
        filter(models.Snapshots.workload_id == workload_id). \
        filter(models.Snapshots.created_at < (session.query(models.Snapshots).
                                              filter(models.Snapshots.workload_id == workload_id).
                                              filter(models.Snapshots.id == snapshot_id).
                                              order_by(asc(models.Snapshots.created_at)).first()).
               __dict__.get('created_at')). \
        order_by(desc(models.Snapshots.created_at)).limit(2)

    if not qs_1:
        raise exception.SnapshotNotFound(snapshot_id=snapshot_id)

    return qs_1


@require_context
def snapshot_type_time_size_update(context, snapshot_id):
    snapshot = snapshot_get(context, snapshot_id, read_deleted='yes')
    workload = workload_get(context, snapshot['workload_id'])

    backup_endpoint = get_metadata_value(workload.metadata,
                                         'backup_media_target')

    backup_target = vault.get_backup_target(backup_endpoint)

    snapshot_type_full = False
    snapshot_type_incremental = False
    snapshot_vm_resources = snapshot_resources_get(context, snapshot_id)
    time_taken = 0
    snapshot_size = 0
    snapshot_restore_size = 0
    for snapshot_vm_resource in snapshot_vm_resources:
        if snapshot_vm_resource.resource_type != 'disk':
            continue
        if snapshot_vm_resource.snapshot_type == 'full' and \
                snapshot_vm_resource.resource_name != 'vda':
            snapshot_type_full = True
        if snapshot_vm_resource.snapshot_type == 'incremental':
            snapshot_type_incremental = True
        time_taken = time_taken + snapshot_vm_resource.time_taken

        # update size
        if snapshot_vm_resource.status != 'deleted':
            disk_type = get_metadata_value(
                snapshot_vm_resource.metadata, 'disk_type')
            vm_disk_resource_snaps = vm_disk_resource_snaps_get(
                context, snapshot_vm_resource.id)
            snapshot_vm_resource_size = 0
            for vm_disk_resource_snap in vm_disk_resource_snaps:
                vm_disk_resource_snap_restore_size = 0

                if vm_disk_resource_snap.vault_url is None:
                    continue

                resource_snap_path = os.path.join(
                    backup_target.mount_path,
                    vm_disk_resource_snap.vault_url.strip(
                        os.sep))
                vm_disk_resource_snap_size = backup_target.get_object_size(
                    resource_snap_path)
                if vm_disk_resource_snap_size == 0:
                    vm_disk_resource_snap_size = vm_disk_resource_snap.size

                disk_format = get_metadata_value(
                    vm_disk_resource_snap.metadata, 'disk_format')
                if disk_format == 'vmdk':
                    vault_path = os.path.join(
                        backup_target.mount_path,
                        vm_disk_resource_snap.vault_url.strip(
                            os.sep))
                    vm_disk_resource_snap_restore_size = vault.get_restore_size(
                        vault_path, disk_format, disk_type)
                else:
                    vm_disk_resource_snap_restore_size = qemuimages.get_effective_size(
                        resource_snap_path)

                # For vmdk
                if vm_disk_resource_snap_restore_size == 0:
                    vm_disk_resource_snap_restore_size = vm_disk_resource_snap_size
                    vm_disk_resource_snap_backing_id = vm_disk_resource_snap.vm_disk_resource_snap_backing_id
                    while vm_disk_resource_snap_backing_id:
                        vm_disk_resource_snap_backing = vm_disk_resource_snap_get(
                            context, vm_disk_resource_snap_backing_id)
                        if vm_disk_resource_snap_backing.vm_disk_resource_snap_backing_id:
                            vm_disk_resource_snap_restore_size = vm_disk_resource_snap_restore_size + \
                                                                 vm_disk_resource_snap_backing.size
                        else:
                            vm_disk_resource_snap_restore_size = vm_disk_resource_snap_restore_size + \
                                                                 vm_disk_resource_snap_backing.restore_size
                        vm_disk_resource_snap_backing_id = vm_disk_resource_snap_backing.vm_disk_resource_snap_backing_id

                vm_disk_resource_snap_update(
                    context, vm_disk_resource_snap.id, {
                        'size': vm_disk_resource_snap_size, 'restore_size': vm_disk_resource_snap_restore_size})
                snapshot_vm_resource_size = snapshot_vm_resource_size + \
                                            vm_disk_resource_snap_size

            vm_disk_resource_snap_top = vm_disk_resource_snap_get_top(
                context, snapshot_vm_resource.id)
            snapshot_vm_resource_restore_size = vm_disk_resource_snap_top.restore_size
            snapshot_vm_resource_update(
                context, snapshot_vm_resource.id, {
                    'size': snapshot_vm_resource_size, 'restore_size': snapshot_vm_resource_restore_size})
            snapshot_size = snapshot_size + snapshot_vm_resource_size
            snapshot_restore_size = snapshot_restore_size + \
                                    snapshot_vm_resource_restore_size

    snapshot_vms = snapshot_vms_get(context, snapshot_id)
    snapshot_data_transfer_time = 0
    snapshot_object_store_transfer_time = 0
    for snapshot_vm in snapshot_vms:
        snapshot_vm_size = 0
        snapshot_vm_restore_size = 0
        snapshot_vm_data_transfer_time = 0
        snapshot_vm_object_store_transfer_time = 0
        snapshot_vm_resources = snapshot_vm_resources_get(
            context, snapshot_vm.vm_id, snapshot_id)
        for snapshot_vm_resource in snapshot_vm_resources:
            if snapshot_vm_resource.resource_type != 'disk':
                continue
            snapshot_vm_size = snapshot_vm_size + snapshot_vm_resource.size
            snapshot_vm_restore_size = snapshot_vm_restore_size + \
                                       snapshot_vm_resource.restore_size
            snapshot_vm_data_transfer_time += snapshot_vm_resource.time_taken
            snapshot_vm_object_store_transfer_time += int(get_metadata_value(
                snapshot_vm_resource.metadata, 'object_store_transfer_time', '0'))
        snapshot_vm_update(
            context,
            snapshot_vm.vm_id,
            snapshot_id,
            {
                'size': snapshot_vm_size,
                'restore_size': snapshot_vm_restore_size,
                'metadata': {
                    'data_transfer_time': snapshot_vm_data_transfer_time,
                    'object_store_transfer_time': snapshot_vm_object_store_transfer_time,
                },
            })
        snapshot_data_transfer_time += snapshot_vm_data_transfer_time
        snapshot_object_store_transfer_time += snapshot_vm_object_store_transfer_time

    if snapshot.finished_at:
        time_taken = max(
            time_taken, int(
                (snapshot.finished_at - snapshot.created_at).total_seconds()))
    else:
        time_taken = max(
            time_taken, int(
                (timeutils.utcnow() - snapshot.created_at).total_seconds()))

    if snapshot_type_full and snapshot_type_incremental:
        snapshot_type = 'mixed'
    elif snapshot_type_incremental:
        snapshot_type = 'incremental'
    elif snapshot_type_full:
        snapshot_type = 'full'
    else:
        snapshot_type = 'full'

    return snapshot_update(
        context,
        snapshot_id,
        {
            'snapshot_type': snapshot_type,
            'time_taken': time_taken,
            'size': snapshot_size,
            'restore_size': snapshot_restore_size,
            'uploaded_size': snapshot_size,
            'metadata': {
                'data_transfer_time': snapshot_data_transfer_time,
                'object_store_transfer_time': snapshot_object_store_transfer_time,
            },
        })


@require_context
def snapshot_delete(context, snapshot_id, hard_delete=False):
    session = get_session()
    with session.begin():
        if hard_delete:
            session.query(models.Snapshots). \
                filter_by(id=snapshot_id).delete()
        else:
            session.query(models.Snapshots). \
                filter_by(id=snapshot_id). \
                update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


@require_context
def get_snapshot_children(context, snapshot_id, children):
    grand_children = set()
    snapshot_vm_resources = snapshot_resources_get(context, snapshot_id)
    for snapshot_vm_resource in snapshot_vm_resources:
        if snapshot_vm_resource.resource_type != 'disk':
            continue
        vm_disk_resource_snaps = vm_disk_resource_snaps_get(
            context, snapshot_vm_resource.id)
        for vm_disk_resource_snap in vm_disk_resource_snaps:
            if vm_disk_resource_snap.vm_disk_resource_snap_child_id:
                try:
                    vm_disk_resource_snap_child = vm_disk_resource_snap_get(
                        context, vm_disk_resource_snap.vm_disk_resource_snap_child_id)
                    snapshot_vm_resource_child = snapshot_vm_resource_get(
                        context, vm_disk_resource_snap_child.snapshot_vm_resource_id)
                    if snapshot_vm_resource_child.snapshot_id not in grand_children:
                        grand_children.add(
                            snapshot_vm_resource_child.snapshot_id)
                        if snapshot_vm_resource_child.snapshot_id != snapshot_id:
                            grand_children = get_snapshot_children(
                                context, snapshot_vm_resource_child.snapshot_id, grand_children)
                except Exception as ex:
                    LOG.exception(ex)
    if children:
        return grand_children.union(children)
    else:
        return grand_children


@require_context
def get_snapshot_parents(context, snapshot_id, parents):
    grand_parents = set()
    snapshot_vm_resources = snapshot_resources_get(context, snapshot_id)
    for snapshot_vm_resource in snapshot_vm_resources:
        if snapshot_vm_resource.resource_type != 'disk':
            continue
        vm_disk_resource_snaps = vm_disk_resource_snaps_get(
            context, snapshot_vm_resource.id)
        for vm_disk_resource_snap in vm_disk_resource_snaps:
            if vm_disk_resource_snap.vm_disk_resource_snap_backing_id:
                try:
                    vm_disk_resource_snap_parent = vm_disk_resource_snap_get(
                        context, vm_disk_resource_snap.vm_disk_resource_snap_backing_id)
                    snapshot_vm_resource_parent = snapshot_vm_resource_get(
                        context, vm_disk_resource_snap_parent.snapshot_vm_resource_id)
                    if snapshot_vm_resource_parent.snapshot_id not in grand_parents:
                        grand_parents.add(
                            snapshot_vm_resource_parent.snapshot_id)
                        if snapshot_vm_resource_parent.snapshot_id != snapshot_id:
                            grand_parents = get_snapshot_parents(
                                context, snapshot_vm_resource_parent.snapshot_id, grand_parents)
                except Exception as ex:
                    LOG.exception(ex)
    if parents:
        return grand_parents.union(parents)
    else:
        return grand_parents


# SnapshotVMs #########################################################
""" snapshot_vms functions """


def _set_metadata_for_snapshot_vms(context, snapshot_vm_ref, metadata,
                                   purge_metadata, session):
    """
    Create or update a set of snapshot_vms_metadata for a given snapshot_vm
    :param context: Request context
    :param snapshot_vm_ref: An snapshot_vm object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in snapshot_vm_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'snapshot_vm_id': snapshot_vm_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _snapshot_vms_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _snapshot_vms_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _snapshot_vms_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _snapshot_vms_metadata_create(context, values, session):
    """Create an SnapshotMetadata object"""
    metadata_ref = models.SnapshotVMMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _snapshot_vms_metadata_update(
        context, metadata_ref, values, session)


@require_context
def snapshot_vms_metadata_create(context, values, session):
    """Create an SnapshotMetadata object"""
    session = get_session()
    return _snapshot_vms_metadata_create(context, values, session)


@require_context
def _snapshot_vms_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by snapshot_vms_metadata_create and snapshot_vms_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _snapshot_vms_metadata_delete(context, metadata_ref, session):
    """
    Used internally by snapshot_vms_metadata_create and snapshot_vms_metadata_update
    """
    metadata_ref.delete(session=session)


def _snapshot_vm_update(context, values, vm_id,
                        snapshot_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if vm_id:
        snapshot_vm_ref = _snapshot_vm_get(
            context, vm_id, snapshot_id, session)
        if snapshot_vm_ref is None:
            return
    else:
        snapshot_vm_ref = models.SnapshotVMs()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())
        if not values.get('size'):
            values['size'] = 0
        if not values.get('restore_size'):
            values['restore_size'] = 0

    snapshot_vm_ref.update(values)
    snapshot_vm_ref.save(session)

    if metadata:
        _set_metadata_for_snapshot_vms(
            context,
            snapshot_vm_ref,
            metadata,
            purge_metadata,
            session=session)

    return snapshot_vm_ref


@require_context
def snapshot_vm_create(context, values):
    session = get_session()
    return _snapshot_vm_update(context, values, None, None, False, session)


@require_context
def snapshot_vm_update(context, vm_id, snapshot_id,
                       values, purge_metadata=False):
    session = get_session()
    return _snapshot_vm_update(
        context, values, vm_id, snapshot_id, purge_metadata, session)


@require_context
def snapshot_vms_get(context, snapshot_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.SnapshotVMs).options(
            sa_orm.joinedload(
                models.SnapshotVMs.metadata)).filter_by(
            snapshot_id=snapshot_id)
        snapshot_vms = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.SnapshotVMsNotFound(snapshot_id=snapshot_id)

    return snapshot_vms


@require_context
def _snapshot_vm_get(context, vm_id, snapshot_id, session):
    try:
        query = session.query(
            models.SnapshotVMs).options(
            sa_orm.joinedload(
                models.SnapshotVMs.metadata)).filter_by(
            vm_id=vm_id)

        if snapshot_id is not None:
            query = query.filter_by(snapshot_id=snapshot_id)

        # TODO(gbasava): filter out deleted snapshot_vm if context disallows it
        snapshot_vm = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.SnapshotVMsNotFound(snapshot_id=snapshot_id)

    return snapshot_vm


@require_context
def snapshot_vm_get(context, vm_id, snapshot_id):
    session = get_session()
    return _snapshot_vm_get(context, vm_id, snapshot_id, session)


@require_context
def snapshot_vm_delete(context, vm_id, snapshot_id):
    session = get_session()
    with session.begin():
        session.query(models.SnapshotVMs). \
            filter_by(vm_id=vm_id). \
            filter_by(snapshot_id=snapshot_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


#


@require_context
def vm_recent_snapshot_create(context, values):
    vm_recent_snapshot = models.VMRecentSnapshot()
    vm_recent_snapshot.update(values)
    vm_recent_snapshot.save()
    return vm_recent_snapshot


@require_context
def vm_recent_snapshot_get(context, vm_id, **kwargs):
    session = kwargs.get('session') or get_session()
    result = model_query(context, models.VMRecentSnapshot, session=session). \
        filter_by(vm_id=vm_id). \
        first()

    return result


@require_context
def vm_recent_snapshot_update(context, vm_id, values):
    session = get_session()
    with session.begin():
        vm_recent_snapshot = model_query(context, models.VMRecentSnapshot,
                                         session=session, read_deleted="yes"). \
            filter_by(vm_id=vm_id).first()

        if not vm_recent_snapshot:
            values['vm_id'] = vm_id
            vm_recent_snapshot = models.VMRecentSnapshot()

        vm_recent_snapshot.update(values)
        vm_recent_snapshot.save(session=session)

    return vm_recent_snapshot


@require_context
def vm_recent_snapshot_delete(context, vm_id):
    session = get_session()
    with session.begin():
        session.query(models.VMRecentSnapshot). \
            filter_by(vm_id=vm_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


""" snapshot vm resource functions """


@require_context
def _set_metadata_for_snapshot_vm_resource(
        context,
        snapshot_vm_resource_ref,
        metadata,
        purge_metadata,
        session):
    """
    Create or update a set of snapshot_vm_resource_metadata for a given snapshot resource
    :param context: Request context
    :param snapshot_vm_resource_ref: An snapshot_vm_resource object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in snapshot_vm_resource_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'snapshot_vm_resource_id': snapshot_vm_resource_ref.id,
            'key': key,
            'value': str(value) if isinstance(value, uuid.UUID) else value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _snapshot_vm_resource_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _snapshot_vm_resource_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _snapshot_vm_resource_metadata_delete(
                    context, metadata_ref, session)


@require_context
def _snapshot_vm_resource_metadata_create(context, values, session):
    """Create an SnapshotVMResourceMetadata object"""
    metadata_ref = models.SnapshotVMResourceMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _snapshot_vm_resource_metadata_update(
        context, metadata_ref, values, session)


@require_context
def snapshot_vm_resource_metadata_create(context, values):
    session = get_session()
    return _snapshot_vm_resource_metadata_create(context, values, session)


@require_context
def _snapshot_vm_resource_metadata_update(
        context, metadata_ref, values, session):
    """
    Used internally by snapshot_vm_resource_metadata_create and snapshot_vm_resource_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _snapshot_vm_resource_metadata_delete(context, metadata_ref, session):
    """
    Used internally by snapshot_vm_resource_metadata_create and snapshot_vm_resource_metadata_update
    """
    metadata_ref.delete(session=session)


@require_context
def _snapshot_vm_resource_update(
        context, values, snapshot_vm_resource_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if snapshot_vm_resource_id:
        snapshot_vm_resource_ref = _snapshot_vm_resource_get(
            context, snapshot_vm_resource_id, session=session)
    else:
        snapshot_vm_resource_ref = models.SnapshotVMResources()
        if not values.get('size'):
            values['size'] = 0
        if not values.get('restore_size'):
            values['restore_size'] = 0

    snapshot_vm_resource_ref.update(values)
    snapshot_vm_resource_ref.save(session)

    _set_metadata_for_snapshot_vm_resource(
        context,
        snapshot_vm_resource_ref,
        metadata,
        purge_metadata,
        session)

    return snapshot_vm_resource_ref


@require_context
def snapshot_vm_resource_create(context, values):
    session = get_session()
    return _snapshot_vm_resource_update(context, values, None, False, session)


@require_context
def snapshot_vm_resource_update(
        context, snapshot_vm_resource_id, values, purge_metadata=False):
    session = get_session()
    return _snapshot_vm_resource_update(
        context, values, snapshot_vm_resource_id, purge_metadata, session)


@require_context
def snapshot_vm_resources_get(context, vm_id, snapshot_id):
    session = get_session()
    try:
        query = session.query(
            models.SnapshotVMResources).options(
            sa_orm.joinedload(
                models.SnapshotVMResources.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            snapshot_id=snapshot_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        snapshot_vm_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.SnapshotVMResourcesNotFound(
            snapshot_vm_id=vm_id, snapshot_id=snapshot_id)

    return snapshot_vm_resources


@require_context
def snapshot_resources_get(context, snapshot_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.SnapshotVMResources).options(
            sa_orm.joinedload(
                models.SnapshotVMResources.metadata)).filter_by(
            snapshot_id=snapshot_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        snapshot_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.SnapshotResourcesNotFound(snapshot_id=snapshot_id)

    return snapshot_resources


@require_context
def snapshot_vm_resource_get_by_resource_name(
        context, vm_id, snapshot_id, resource_name):
    session = get_session()
    try:
        query = session.query(
            models.SnapshotVMResources).options(
            sa_orm.joinedload(
                models.SnapshotVMResources.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            snapshot_id=snapshot_id).filter_by(
            resource_name=resource_name)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        snapshot_vm_resource = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.SnapshotVMResourceWithNameNotFound(
            resource_name=resource_name, snapshot_vm_id=vm_id, snapshot_id=snapshot_id)

    return snapshot_vm_resource


@require_context
def snapshot_vm_resource_get_by_resource_pit_id(
        context, vm_id, snapshot_id, resource_pit_id):
    session = get_session()
    try:
        query = session.query(
            models.SnapshotVMResources).options(
            sa_orm.joinedload(
                models.SnapshotVMResources.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            snapshot_id=snapshot_id).filter_by(
            resource_pit_id=resource_pit_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        snapshot_vm_resource = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.SnapshotVMResourceWithNameNotFound(
            resource_pit_id=resource_pit_id, snapshot_vm_id=vm_id, snapshot_id=snapshot_id)

    return snapshot_vm_resource


@require_context
def _snapshot_vm_resource_get(context, id, session):
    try:
        query = session.query(
            models.SnapshotVMResources).options(
            sa_orm.joinedload(
                models.SnapshotVMResources.metadata)).filter_by(
            id=id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        snapshot_vm_resource = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.SnapshotVMResourceNotFound(snapshot_vm_resource_id=id)

    return snapshot_vm_resource


@require_context
def snapshot_vm_resource_get(context, id):
    session = get_session()
    return _snapshot_vm_resource_get(context, id, session)


@require_context
def snapshot_vm_resource_delete(context, id):
    session = get_session()
    with session.begin():
        session.query(models.SnapshotVMResources). \
            filter_by(id=id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


""" disk resource snapshot functions """


def _set_metadata_for_vm_disk_resource_snap(
        context,
        vm_disk_resource_snap_ref,
        metadata,
        purge_metadata,
        session):
    """
    Create or update a set of vm_disk_resource_snap_metadata for a given snapshot
    :param context: Request context
    :param image_ref: An vm_disk_resource_snap object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in vm_disk_resource_snap_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'vm_disk_resource_snap_id': vm_disk_resource_snap_ref.id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _vm_disk_resource_snap_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _vm_disk_resource_snap_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _vm_disk_resource_snap_metadata_delete(
                    context, metadata_ref, session)


@require_context
def _vm_disk_resource_snap_metadata_create(context, values, session):
    """Create an VMDiskResourceSnapMetadata object"""
    metadata_ref = models.VMDiskResourceSnapMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _vm_disk_resource_snap_metadata_update(
        context, metadata_ref, values, session)


@require_context
def vm_disk_resource_snap_metadata_create(context, values):
    """Create an VMDiskResourceSnapMetadata object"""
    session = get_session()
    return _vm_disk_resource_snap_metadata_create(context, values, session)


def _vm_disk_resource_snap_metadata_update(
        context, metadata_ref, values, session):
    """
    Used internally by vm_disk_resource_snap_metadata_create and vm_disk_resource_snap_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


def _vm_disk_resource_snap_metadata_delete(context, metadata_ref, session):
    """
    Used internally by vm_disk_resource_snap_metadata_create and vm_disk_resource_snap_metadata_update
    """
    metadata_ref.delete(session=session)


def _vm_disk_resource_snap_update(
        context, values, vm_disk_resource_snap_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if vm_disk_resource_snap_id:
        vm_disk_resource_snap_ref = _vm_disk_resource_snap_get(
            context, vm_disk_resource_snap_id, session)
    else:
        vm_disk_resource_snap_ref = models.VMDiskResourceSnaps()
        if not values.get('size'):
            values['size'] = 0
        if not values.get('restore_size'):
            values['restore_size'] = 0

    vm_disk_resource_snap_ref.update(values)
    vm_disk_resource_snap_ref.save(session)

    _set_metadata_for_vm_disk_resource_snap(
        context,
        vm_disk_resource_snap_ref,
        metadata,
        purge_metadata,
        session)

    return vm_disk_resource_snap_ref


@require_context
def vm_disk_resource_snap_create(context, values):
    session = get_session()
    return _vm_disk_resource_snap_update(context, values, None, False, session)


@require_context
def vm_disk_resource_snap_update(
        context, vm_disk_resource_snap_id, values, purge_metadata=False):
    session = get_session()
    return _vm_disk_resource_snap_update(
        context, values, vm_disk_resource_snap_id, purge_metadata, session)


@require_context
def vm_disk_resource_snaps_get(context, snapshot_vm_resource_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.VMDiskResourceSnaps).options(
            sa_orm.joinedload(
                models.VMDiskResourceSnaps.metadata)).filter_by(
            snapshot_vm_resource_id=snapshot_vm_resource_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        vm_disk_resource_snaps = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.VMDiskResourceSnapsNotFound(
            snapshot_vm_resource_id=snapshot_vm_resource_id)

    return vm_disk_resource_snaps


@require_context
def vm_disk_resource_snap_get_top(context, snapshot_vm_resource_id):
    session = get_session()
    try:
        query = session.query(
            models.VMDiskResourceSnaps).options(
            sa_orm.joinedload(
                models.VMDiskResourceSnaps.metadata)).filter_by(
            snapshot_vm_resource_id=snapshot_vm_resource_id).filter_by(
            top=True)

        # TODO(gbasava): filter out resource snapshots if context disallows it
        vm_disk_resource_snap = query.one()

    except sa_orm.exc.NoResultFound:
        raise exception.VMDiskResourceSnapTopNotFound(
            snapshot_vm_resource_id=snapshot_vm_resource_id)

    return vm_disk_resource_snap


@require_context
def vm_disk_resource_snap_get_bottom(context, snapshot_vm_resource_id):
    vm_disk_resource_snap = db.vm_disk_resource_snap_get_top(
        context, snapshot_vm_resource_id)
    while vm_disk_resource_snap and vm_disk_resource_snap.vm_disk_resource_snap_backing_id:
        vm_disk_resource_snap_backing = db.vm_disk_resource_snap_get(
            context, vm_disk_resource_snap.vm_disk_resource_snap_backing_id)
        if vm_disk_resource_snap_backing.snapshot_vm_resource_id == vm_disk_resource_snap.snapshot_vm_resource_id:
            vm_disk_resource_snap = vm_disk_resource_snap_backing
        else:
            break
    return vm_disk_resource_snap


@require_context
def _vm_disk_resource_snap_get(context, vm_disk_resource_snap_id, session):
    try:
        query = session.query(
            models.VMDiskResourceSnaps).options(
            sa_orm.joinedload(
                models.VMDiskResourceSnaps.metadata)).filter_by(
            id=vm_disk_resource_snap_id)

        # TODO(gbasava): filter out deleted resource snapshots if context
        # disallows it
        vm_disk_resource_snap = query.one()

    except sa_orm.exc.NoResultFound:
        raise exception.VMDiskResourceSnapNotFound(
            vm_disk_resource_snap_id=vm_disk_resource_snap_id)

    return vm_disk_resource_snap


@require_context
def vm_disk_resource_snap_get(context, vm_disk_resource_snap_id):
    session = get_session()
    return _vm_disk_resource_snap_get(
        context, vm_disk_resource_snap_id, session)


@require_context
def vm_disk_resource_snap_get_snapshot_vm_resource_id(
        context, vm_disk_resource_snap_id):
    vm_disk_resource_snap = vm_disk_resource_snap_get(
        context, vm_disk_resource_snap_id)
    return vm_disk_resource_snap.snapshot_vm_resource_id


@require_context
def vm_disk_resource_snap_delete(context, vm_disk_resource_snap_id):
    session = get_session()
    with session.begin():
        session.query(models.VMDiskResourceSnaps). \
            filter_by(id=vm_disk_resource_snap_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


""" network resource snapshot functions """


def _set_metadata_for_vm_network_resource_snap(
        context,
        vm_network_resource_snap_ref,
        metadata,
        purge_metadata,
        session):
    """
    Create or update a set of vm_network_resource_snap_metadata for a given snapshot
    :param context: Request context
    :param vm_network_resource_snap_ref: An vm_network_resource_snap object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in vm_network_resource_snap_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'vm_network_resource_snap_id': vm_network_resource_snap_ref.vm_network_resource_snap_id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _vm_network_resource_snap_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _vm_network_resource_snap_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _vm_network_resource_snap_metadata_delete(
                    context, metadata_ref, session)


@require_context
def _vm_network_resource_snap_metadata_create(context, values, session):
    """Create an VMNetworkResourceSnapMetadata object"""
    metadata_ref = models.VMNetworkResourceSnapMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _vm_network_resource_snap_metadata_update(
        context, metadata_ref, values, session)


@require_context
def vm_network_resource_snap_metadata_create(context, values):
    """Create an VMNetworkResourceSnapMetadata object"""
    session = get_session()
    return _vm_network_resource_snap_metadata_create(context, values, session)


def _vm_network_resource_snap_metadata_update(
        context, metadata_ref, values, session):
    """
    Used internally by vm_network_resource_snap_metadata_create and vm_network_resource_snap_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _vm_network_resource_snap_metadata_delete(context, metadata_ref, session):
    """
    Used internally by vm_network_resource_snap_metadata_create and vm_network_resource_snap_metadata_update
    """
    metadata_ref.delete(session=session)
    return metadata_ref


def _vm_network_resource_snap_update(
        context, vm_network_resource_snap_id, values, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if vm_network_resource_snap_id:
        vm_network_resource_snap_ref = vm_network_resource_snap_get(
            context, vm_network_resource_snap_id, session)
    else:
        vm_network_resource_snap_ref = models.VMNetworkResourceSnaps()

    vm_network_resource_snap_ref.update(values)
    vm_network_resource_snap_ref.save(session)

    _set_metadata_for_vm_network_resource_snap(
        context,
        vm_network_resource_snap_ref,
        metadata,
        purge_metadata,
        session=session)

    return vm_network_resource_snap_ref


@require_context
def vm_network_resource_snap_create(context, values):
    session = get_session()
    return _vm_network_resource_snap_update(
        context, None, values, False, session)


@require_context
def vm_network_resource_snap_update(
        context, vm_network_resource_snap_id, values, purge_metadata=False):
    session = get_session()
    return _vm_network_resource_snap_update(
        context, values, vm_network_resource_snap_id, purge_metadata, session)


@require_context
def vm_network_resource_snaps_get(context, snapshot_vm_resource_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.VMNetworkResourceSnaps).options(
            sa_orm.joinedload(
                models.VMNetworkResourceSnaps.metadata)).filter_by(
            vm_network_resource_snap_id=snapshot_vm_resource_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        vm_network_resource_snaps = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.VMNetworkResourceSnapsNotFound(
            snapshot_vm_resource_id=snapshot_vm_resource_id)

    return vm_network_resource_snaps


@require_context
def vm_network_resource_snap_get(context, vm_network_resource_snap_id):
    session = get_session()
    try:
        query = session.query(
            models.VMNetworkResourceSnaps).options(
            sa_orm.joinedload(
                models.VMNetworkResourceSnaps.metadata)).filter_by(
            vm_network_resource_snap_id=vm_network_resource_snap_id)

        # TODO(gbasava): filter out deleted resource snapshots if context
        # disallows it
        vm_network_resource_snap = query.one()

    except sa_orm.exc.NoResultFound:
        raise exception.VMNetworkResourceSnapNotFound(
            vm_network_resource_snap_id=vm_network_resource_snap_id)

    return vm_network_resource_snap


@require_context
def vm_network_resource_snap_delete(context, vm_network_resource_snap_id):
    session = get_session()
    with session.begin():
        session.query(models.VMNetworkResourceSnaps). \
            filter_by(id=vm_network_resource_snap_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


""" security group rule snapshot functions """


def _set_metadata_for_vm_security_group_rule_snap(
        context,
        vm_security_group_rule_snap_ref,
        metadata,
        purge_metadata,
        session):
    """
    Create or update a set of vm_security_group_rule_snap_metadata for a given snapshot
    :param context: Request context
    :param vm_security_group_rule_snap_ref: An vm_security_group_rule_snap object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in vm_security_group_rule_snap_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'vm_security_group_rule_snap_id': vm_security_group_rule_snap_ref.id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _vm_security_group_rule_snap_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _vm_security_group_rule_snap_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _vm_security_group_rule_snap_metadata_delete(
                    context, metadata_ref, session)


@require_context
def _vm_security_group_rule_snap_metadata_create(context, values, session):
    """Create an VMSecurityGroupRuleSnapMetadata object"""
    metadata_ref = models.VMSecurityGroupRuleSnapMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _vm_security_group_rule_snap_metadata_update(
        context, metadata_ref, values, session)


@require_context
def vm_security_group_rule_snap_metadata_create(context, values):
    """Create an VMSecurityGroupRuleSnapMetadata object"""
    session = get_session()
    return _vm_security_group_rule_snap_metadata_create(
        context, values, session)


def _vm_security_group_rule_snap_metadata_update(
        context, metadata_ref, values, session):
    """
    Used internally by vm_security_group_rule_snap_metadata_create and vm_security_group_rule_snap_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _vm_security_group_rule_snap_metadata_delete(
        context, metadata_ref, session):
    """
    Used internally by vm_security_group_rule_snap_metadata_create and vm_security_group_rule_snap_metadata_update
    """
    metadata_ref.delete(session=session)


def _vm_security_group_rule_snap_update(
        context,
        id,
        vm_security_group_snap_id,
        values,
        purge_metadata,
        session):
    metadata = values.pop('metadata', {})

    if id and vm_security_group_snap_id:
        vm_security_group_rule_snap_ref = vm_security_group_rule_snap_get(
            context, id, vm_security_group_snap_id, session)
    else:
        vm_security_group_rule_snap_ref = models.VMSecurityGroupRuleSnaps()

    vm_security_group_rule_snap_ref.update(values)
    vm_security_group_rule_snap_ref.save(session)

    _set_metadata_for_vm_security_group_rule_snap(
        context, vm_security_group_rule_snap_ref, metadata, purge_metadata, session)

    return vm_security_group_rule_snap_ref


@require_context
def vm_security_group_rule_snap_create(context, values):
    session = get_session()
    return _vm_security_group_rule_snap_update(
        context, None, None, values, False, session)


@require_context
def vm_security_group_rule_snap_update(
        context, id, vm_security_group_snap_id, values, purge_metadata=False):
    session = get_session()
    return _vm_security_group_rule_snap_update(
        context, id, vm_security_group_snap_id, values, purge_metadata, session)


@require_context
def vm_security_group_rule_snaps_get(
        context, vm_security_group_snap_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.VMSecurityGroupRuleSnaps).options(
            sa_orm.joinedload(
                models.VMSecurityGroupRuleSnaps.metadata)).filter_by(
            vm_security_group_snap_id=vm_security_group_snap_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        vm_security_group_rule_snaps = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.VMSecurityGroupRuleSnapsNotFound(
            vm_security_group_snap_id=vm_security_group_snap_id)

    return vm_security_group_rule_snaps


@require_context
def vm_security_group_rule_snap_get(context, id, vm_security_group_snap_id):
    session = get_session()
    try:
        query = session.query(
            models.VMSecurityGroupRuleSnaps).options(
            sa_orm.joinedload(
                models.VMSecurityGroupRuleSnaps.metadata)).filter_by(
            id=id).filter_by(
            vm_security_group_snap_id=vm_security_group_snap_id)

        # TODO(gbasava): filter out deleted resource snapshots if context
        # disallows it
        vm_security_group_rule_snap = query.one()

    except sa_orm.exc.NoResultFound:
        raise exception.VMSecurityGroupRuleSnapNotFound(
            vm_security_group_rule_snap_id=id,
            vm_security_group_snap_id=vm_security_group_snap_id)

    return vm_security_group_rule_snap


@require_context
def vm_security_group_rule_snap_delete(
        context, id, vm_security_group_rule_snap_id):
    session = get_session()
    with session.begin():
        session.query(models.VMSecurityGroupRuleSnaps). \
            filter_by(id=id). \
            filter_by(id=vm_security_group_rule_snap_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


def get_metadata_value(metadata, key, default=None):
    for kvpair in metadata:
        if kvpair['key'] == key:
            return kvpair['value']
    return default


# Restore ################################################################
""" restore functions """


@require_admin_context
def restore_mark_incomplete_as_error(context, host):
    """
    mark the restores that are left hanging from previous run on host as 'error'
    """
    session = get_session()
    restores_query = model_query(context, models.Restores, session=session). \
        filter_by(host=host)
    restores_query = restores_query.filter(and_(
        models.Restores.status != 'available',
        models.Restores.status != 'error',
        models.Restores.status != 'cancelled'))
    restores = restores_query.all()
    values = {'progress_percent': 100, 'progress_msg': '',
              'error_msg': 'Restore did not finish successfully',
              'status': 'error'}
    for restore in restores:
        restore.update(values)
        session.add(restore)
        snapshot_update(context, restore.snapshot_id, {
            'status': 'available'})

    session.flush()
    session.close()


def _set_metadata_for_restore(context, restore_ref, metadata,
                              purge_metadata, session):
    """
    Create or update a set of restore_metadata for a given restore
    :param context: Request context
    :param restore_ref: A restore object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in restore_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'restore_id': restore_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _restore_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _restore_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _restore_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _restore_metadata_create(context, values, session):
    """Create a RestoreMetadata object"""
    metadata_ref = models.RestoreMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _restore_metadata_update(context, metadata_ref, values, session)


@require_context
def restore_metadata_create(context, values, session):
    """Create an RestoreMetadata object"""
    session = get_session()
    return _restore_metadata_create(context, values, session)


@require_context
def _restore_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by restore_metadata_create and restore_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _restore_metadata_delete(context, metadata_ref, session):
    """
    Used internally by restore_metadata_create and restore_metadata_update
    """
    metadata_ref.delete(session=session)


def _restore_update(context, values, restore_id, purge_metadata, session):
    try:
        lock.acquire()
        metadata = values.pop('metadata', {})

        if restore_id:
            restore_ref = model_query(
                context,
                models.Restores,
                session=session,
                read_deleted="yes").filter_by(
                id=restore_id).first()
            if not restore_ref:
                lock.release()
                raise exception.RestoreNotFound(restore_id=restore_id)

            if not values.get('uploaded_size'):
                if values.get('uploaded_size_incremental'):
                    values['uploaded_size'] = restore_ref.uploaded_size + \
                                              values.get('uploaded_size_incremental')
                    if not values.get(
                            'progress_percent') and restore_ref.size > 0:
                        values['progress_percent'] = min(
                            99, (100 * values.get('uploaded_size')) / restore_ref.size)
        else:
            restore_ref = models.Restores()
            if not values.get('id'):
                values['id'] = str(uuid.uuid4())
            if not values.get('size'):
                values['size'] = 0
            if not values.get('uploaded_size'):
                values['uploaded_size'] = 0
            if not values.get('progress_percent'):
                values['progress_percent'] = 0

        restore_ref.update(values)
        restore_ref.save(session)

        if metadata:
            _set_metadata_for_restore(
                context,
                restore_ref,
                metadata,
                purge_metadata,
                session=session)

        return restore_ref
    finally:
        lock.release()
    return restore_ref


@require_context
def _restore_get(context, restore_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    result = model_query(context, models.Restores, **kwargs). \
        options(sa_orm.joinedload(models.Restores.metadata)). \
        filter_by(id=restore_id). \
        first()

    if not result:
        raise exception.RestoreNotFound(restore_id=restore_id)

    return result


@require_context
def restore_get(context, restore_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    return _restore_get(context, restore_id, **kwargs)


@require_context
def restore_get_metadata_cancel_flag(
        context, restore_id, return_val=0, process=None, **kwargs):
    flag = '0'
    restore_obj = restore_get(context, restore_id)
    for meta in restore_obj.metadata:
        if meta.key == 'cancel_requested':
            flag = meta.value

    if return_val == 1:
        return flag

    if flag == '1':
        if process:
            process.kill()
        error = _('Cancel requested for restore')
        raise exception.ErrorOccurred(reason=error)


@require_context
def restore_get_all(context, snapshot_id=None, **kwargs):
    if not is_admin_context(context):
        if snapshot_id:
            return restore_get_all_by_snapshot(context, snapshot_id, **kwargs)
        else:
            return restore_get_all_by_project(
                context, context.project_id, **kwargs)

    if snapshot_id is None:
        if 'dashboard_item' in kwargs:
            if kwargs.get('dashboard_item') == 'activities':
                if 'time_in_minutes' in kwargs:
                    time_in_minutes = int(kwargs.get('time_in_minutes'))
                else:
                    time_in_minutes = 0
                time_delta = ((time_in_minutes / 60) / 24) * -1
                result = \
                    model_query(context,
                                models.Restores.id,
                                models.Restores.deleted,
                                models.Restores.deleted_at,
                                models.Restores.display_name,
                                models.Restores.status,
                                models.Restores.created_at,
                                models.Restores.user_id,
                                models.Restores.project_id,
                                (models.Snapshots.display_name).label(
                                    'snapshot_name'),
                                (models.Snapshots.created_at).label(
                                    'snapshot_created_at'),
                                (models.Workloads.display_name).label(
                                    'workload_name'),
                                (models.Workloads.created_at).label(
                                    'workload_created_at'),
                                **kwargs). \
                        filter(
                        or_(models.Restores.created_at > func.adddate(func.now(), time_delta),
                            models.Restores.deleted_at > func.adddate(func.now(), time_delta))). \
                        outerjoin(models.Snapshots,
                                  models.Restores.snapshot_id == models.Snapshots.id). \
                        outerjoin(models.Workloads,
                                  models.Snapshots.workload_id == models.Workloads.id). \
                        order_by(models.Restores.created_at.desc()).all()
                return result
        else:
            return model_query(context, models.Restores, **kwargs). \
                options(sa_orm.joinedload(models.Restores.metadata)). \
                order_by(models.Restores.created_at.desc()).all()
    else:
        return model_query(context, models.Restores, **kwargs). \
            options(sa_orm.joinedload(models.Restores.metadata)). \
            filter_by(snapshot_id=snapshot_id). \
            order_by(models.Restores.created_at.desc()).all()


@require_context
def restore_get_all_by_snapshot(context, snapshot_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    return model_query(context, models.Restores, **kwargs). \
        options(sa_orm.joinedload(models.Restores.metadata)). \
        filter_by(snapshot_id=snapshot_id). \
        order_by(models.Restores.created_at.desc()).all()


@require_context
def restore_get_all_by_project(context, project_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    authorize_project_context(context, project_id)
    return model_query(context, models.Restores, **kwargs). \
        options(sa_orm.joinedload(models.Restores.metadata)). \
        filter_by(project_id=project_id).all()


@require_context
def restore_get_all_by_project_snapshot(
        context, project_id, snapshot_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    authorize_project_context(context, project_id)
    return model_query(context, models.Restores, **kwargs). \
        options(sa_orm.joinedload(models.Restores.metadata)). \
        filter_by(project_id=project_id). \
        filter_by(snapshot_id=snapshot_id). \
        order_by(models.Restores.created_at.desc()).all()


@require_context
def restore_show(context, restore_id):
    session = get_session()
    result = model_query(context, models.Restores, session=session). \
        options(sa_orm.joinedload(models.Restores.metadata)). \
        filter_by(id=restore_id). \
        first()

    if not result:
        raise exception.RestoreNotFound(restore_id=restore_id)

    return result


@require_context
def restore_create(context, values):
    session = get_session()
    return _restore_update(context, values, None, False, session)


@require_context
def restore_update(context, restore_id, values, purge_metadata=False):
    session = get_session()
    return _restore_update(context, values, restore_id,
                           purge_metadata, session)


@require_context
def restore_delete(context, restore_id):
    session = get_session()
    with session.begin():
        session.query(models.Restores). \
        filter_by(id=restore_id).delete()


# RestoredVMs #########################################################
""" restored_vms functions """


def _set_metadata_for_restored_vms(context, restored_vm_ref, metadata,
                                   purge_metadata, session):
    """
    Create or update a set of restored_vms_metadata for a given restored_vm
    :param context: Request context
    :param restored_vm_ref: An restored_vm object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in restored_vm_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'restored_vm_id': restored_vm_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _restored_vms_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _restored_vms_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _restored_vms_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _restored_vms_metadata_create(context, values, session):
    """Create an RestoredMetadata object"""
    metadata_ref = models.RestoredVMMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _restored_vms_metadata_update(
        context, metadata_ref, values, session)


@require_context
def restored_vms_metadata_create(context, values, session):
    """Create an RestoredMetadata object"""
    session = get_session()
    return _restored_vms_metadata_create(context, values, session)


@require_context
def _restored_vms_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by restored_vms_metadata_create and restored_vms_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _restored_vms_metadata_delete(context, metadata_ref, session):
    """
    Used internally by restored_vms_metadata_create and restored_vms_metadata_update
    """
    metadata_ref.delete(session=session)


def _restored_vm_update(context, values, vm_id,
                        restore_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if vm_id:
        restored_vm_ref = _restored_vm_get(context, vm_id, restore_id, session)
    else:
        restored_vm_ref = models.RestoredVMs()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())
        if not values.get('size'):
            values['size'] = 0

    restored_vm_ref.update(values)
    restored_vm_ref.save(session)

    if metadata:
        _set_metadata_for_restored_vms(
            context,
            restored_vm_ref,
            metadata,
            purge_metadata,
            session=session)

    return restored_vm_ref


@require_context
def restored_vm_create(context, values):
    session = get_session()
    return _restored_vm_update(context, values, None, None, False, session)


@require_context
def restored_vm_update(context, vm_id, restore_id,
                       values, purge_metadata=False):
    session = get_session()
    return _restored_vm_update(
        context, values, vm_id, restore_id, purge_metadata, session)


@require_context
def restored_vms_get(context, restore_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.RestoredVMs).options(
            sa_orm.joinedload(
                models.RestoredVMs.metadata)).filter_by(
            restore_id=restore_id)
        restored_vms = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.RestoredVMsNotFound(restore_id=restore_id)

    return restored_vms


@require_context
def _restored_vm_get(context, vm_id, restore_id, session):
    try:
        query = session.query(
            models.RestoredVMs).options(
            sa_orm.joinedload(
                models.RestoredVMs.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            restore_id=restore_id)

        # TODO(gbasava): filter out deleted restored_vm if context disallows it
        restored_vm = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.RestoredVMsNotFound(restore_id=restore_id)

    return restored_vm


@require_context
def restored_vm_get(context, vm_id, restore_id):
    session = get_session()
    return _restored_vm_get(context, vm_id, restore_id, session)


@require_context
def restored_vm_delete(context, vm_id, restore_id):
    session = get_session()
    with session.begin():
        session.query(models.RestoredVMs). \
            filter_by(vm_id=vm_id). \
            filter_by(restore_id=restore_id).delete()


""" restore vm resource functions """


def _set_metadata_for_restored_vm_resource(
        context,
        restored_vm_resource_ref,
        metadata,
        purge_metadata,
        session):
    """
    Create or update a set of restored_vm_resource_metadata for a given restored resource
    :param context: Request context
    :param restored_vm_resource_ref: An restored_vm_resource object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in restored_vm_resource_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'restored_vm_resource_id': restored_vm_resource_ref.id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _restored_vm_resource_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _restored_vm_resource_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                restored_vm_resource_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _restored_vm_resource_metadata_create(context, values, session):
    """Create an RestoredVMResourceMetadata object"""
    metadata_ref = models.RestoredVMResourceMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _restored_vm_resource_metadata_update(
        context, metadata_ref, values, session)


@require_context
def restored_vm_resource_metadata_create(context, values):
    """Create an RestoredVMResourceMetadata object"""
    session = get_session()
    return _restored_vm_resource_metadata_create(context, values, session)


def _restored_vm_resource_metadata_update(
        context, metadata_ref, values, session):
    """
    Used internally by restored_vm_resource_metadata_create and restored_vm_resource_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def restored_vm_resource_metadata_delete(context, metadata_ref):
    """
    Used internally by restored_vm_resource_metadata_create and restored_vm_resource_metadata_update
    """
    session = get_session()
    metadata_ref.delete(session=session)
    return metadata_ref


def _restored_vm_resource_update(
        context, values, restored_vm_resource_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if restored_vm_resource_id:
        restored_vm_resource_ref = restored_vm_resource_get(
            context, restored_vm_resource_id, session)
    else:
        restored_vm_resource_ref = models.RestoredVMResources()

    restored_vm_resource_ref.update(values)
    restored_vm_resource_ref.save(session)

    _set_metadata_for_restored_vm_resource(
        context,
        restored_vm_resource_ref,
        metadata,
        purge_metadata,
        session)

    return restored_vm_resource_ref


@require_context
def restored_vm_resource_create(context, values):
    session = get_session()
    return _restored_vm_resource_update(context, values, None, False, session)


@require_context
def restored_vm_resource_update(
        context, restored_vm_resource_id, values, purge_metadata=False):
    session = get_session()
    return _restored_vm_resource_update(
        context, values, restored_vm_resource_id, purge_metadata, session)


@require_context
def restored_vm_resources_get(context, vm_id, restore_id):
    session = get_session()
    try:
        query = session.query(
            models.RestoredVMResources).options(
            sa_orm.joinedload(
                models.RestoredVMResources.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            restore_id=restore_id)

        # TODO(gbasava): filter out deleted restores if context disallows it
        restored_vm_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.RestoredVMResourcesNotFound(
            restore_vm_id=vm_id, restore_id=restore_id)

    return restored_vm_resources


@require_context
def restored_vm_resource_get_by_resource_name(
        context, vm_id, restore_id, resource_name):
    session = get_session()
    try:
        query = session.query(
            models.RestoredVMResources).options(
            sa_orm.joinedload(
                models.RestoredVMResources.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            restore_id=restore_id).filter_by(
            resource_name=resource_name)

        # TODO(gbasava): filter out deleted restores if context disallows it
        restored_vm_resources = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.RestoredVMResourceWithNameNotFound(
            resource_name=resource_name, restore_vm_id=vm_id, restore_id=restore_id)

    return restored_vm_resources


@require_context
def restored_vm_resource_get(context, id):
    session = get_session()
    try:
        query = session.query(
            models.RestoredVMResources).options(
            sa_orm.joinedload(
                models.RestoredVMResources.metadata)).filter_by(
            id=id)

        # TODO(gbasava): filter out deleted restored if context disallows it
        restored_vm_resources = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.RestoredVMResourceWithIdNotFound(id=id)

    return restored_vm_resources


@require_context
def restored_vm_resource_delete(context, id, vm_id, restore_id):
    session = get_session()
    with session.begin():
        session.query(models.RestoredVMResources). \
            filter_by(id=id). \
            filter_by(vm_id=vm_id). \
            filter_by(restore_id=restore_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


# project quota types ####################################################


def get_all_project_quota_types(quota_type_id=None):
    session = get_session()
    with session.begin():
        if quota_type_id:
            return session.query(models.ProjectQuotaTypes).filter_by(
                id=quota_type_id
            ).all()
        else:
            return session.query(models.ProjectQuotaTypes).all()


def create_project_quota_type(quota_type_data):
    session = get_session()
    with session.begin():
        metadata_ref = models.ProjectQuotaTypes()
        metadata_ref.update(quota_type_data)
        metadata_ref.save(session=session)
        return metadata_ref


def update_project_quota_types(update_dict):
    session = get_session()
    with session.begin():
        session.query(models.ProjectQuotaTypes).update(update_dict)


# project allowed quotas ####################################################

@require_admin_context
def get_allowed_quotas(
        context, project_id=None, allowed_quota_id=None, quota_type_id=None
):
    session = get_session()
    params = {
        "deleted": False,
    }
    if project_id:
        params.update({"project_id": project_id})
    if allowed_quota_id:
        params.update({"id": allowed_quota_id})
    if quota_type_id:
        params.update({"quota_type_id": quota_type_id})

    with session.begin():
        return session.query(models.AllowedQuota).filter_by(**params).all()


@require_admin_context
def create_allowed_quotas(context, allowed_quota_data):
    session = get_session()
    with session.begin():
        res = models.AllowedQuota.save_multiple(
            allowed_quota_data, session=session
        )
        return res


@require_admin_context
def modify_allowed_quotas(context, id, allowed_quota_data):
    session = get_session()
    with session.begin():
        project_id = allowed_quota_data["project_id"]
        allowed_quota_data.pop("project_id", None)
        obj = session.query(models.AllowedQuota).filter_by(
            id=id, project_id=project_id, deleted=False
        )
        return obj.update(allowed_quota_data)


@require_admin_context
def delete_allowed_quota(context, allowed_quota_id):
    session = get_session()
    with session.begin():
        session.query(models.AllowedQuota). \
            filter_by(id=allowed_quota_id). \
            update({'deleted': True,
                    'allowed_value': -1,
                    'high_watermark': -1,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


# Setting ################################################################


""" setting functions """


def _set_metadata_for_setting(context, setting_ref, metadata,
                              purge_metadata, session):
    """
    Create or update a set of setting_metadata for a given setting
    :param context: Request context
    :param setting_ref: A setting object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in setting_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'settings_name': setting_ref.name,
                           'settings_project_id': setting_ref.project_id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _setting_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _setting_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _setting_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _setting_metadata_create(context, values, session):
    """Create a SettingMetadata object"""
    metadata_ref = models.SettingMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _setting_metadata_update(context, metadata_ref, values, session)


@require_context
def setting_metadata_create(context, values, session):
    """Create a SettingMetadata object"""
    session = get_session()
    return _setting_metadata_create(context, values, session)


@require_context
def _setting_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by setting_metadata_create and setting_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _setting_metadata_delete(context, metadata_ref, session):
    """
    Used internally by setting_metadata_create and setting_metadata_update
    """
    metadata_ref.delete(session=session)


def _setting_update(context, values, setting_name, purge_metadata, session, cloud_setting=False):
    try:
        lock.acquire()
        metadata = values.pop('metadata', {})

        if setting_name:
            if cloud_setting is True:
                setting_ref = model_query(context, models.Settings, session=session, read_deleted="yes"). \
                    filter_by(name=setting_name). \
                    first()
            else:
                setting_ref = model_query(context, models.Settings, session=session, read_deleted="yes"). \
                    filter_by(name=setting_name). \
                    filter_by(project_id=context.project_id). \
                    first()

            if not setting_ref:
                lock.release()
                raise exception.SettingNotFound(setting_name=setting_name)

        else:
            setting_ref = models.Settings()
            if not values.get('status'):
                values['status'] = 'available'
            if not values.get('project_id'):
                values['project_id'] = context.project_id
        if 'is_hidden' in values:
            values['hidden'] = int(values['is_hidden'])
        setting_ref.update(values)
        setting_ref.save(session)

        if metadata:
            _set_metadata_for_setting(
                context,
                setting_ref,
                metadata,
                purge_metadata,
                session=session)

        return setting_ref
    finally:
        lock.release()
    return setting_ref


@require_context
def _setting_get(context, setting_name, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    get_hidden = kwargs.get('get_hidden', False)
    cloud_setting = kwargs.get('cloud_setting', False)
    if cloud_setting is True:
        result = model_query(context, models.Settings, **kwargs). \
            filter_by(hidden=get_hidden). \
            options(sa_orm.joinedload(models.Settings.metadata)). \
            filter_by(name=setting_name). \
            first()
    else:
        result = model_query(context, models.Settings, **kwargs). \
            filter_by(hidden=get_hidden). \
            options(sa_orm.joinedload(models.Settings.metadata)). \
            filter_by(name=setting_name). \
            filter_by(project_id=context.project_id). \
            first()

    if not result:
        if setting_name == 'page_size':
            return 10
        else:
            raise exception.SettingNotFound(setting_name=setting_name)

    return result


@require_context
def setting_get(context, setting_name, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    return _setting_get(context, setting_name, **kwargs)


def setting_get_all(context, **kwargs):
    if context and not is_admin_context(context):
        return setting_get_all_by_project(
            context, context.project_id, **kwargs)

    get_hidden = kwargs.get('get_hidden', False)
    setting_type = kwargs.get('type', None)

    qs = model_query(context, models.Settings, **kwargs). \
        options(sa_orm.joinedload(models.Settings.metadata))

    if 'backup_settings' in kwargs:
        qs = qs.filter(models.Settings.project_id != 'Configurator')
    elif setting_type:
        qs = qs.filter_by(hidden=get_hidden, type=setting_type)
    else:
        qs = qs.filter_by(hidden=get_hidden)

    return qs.order_by(models.Settings.created_at.desc()).all()


@require_context
def setting_get_all_by_project(context, project_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    if kwargs.get('is_cloud_admin', False):
        kwargs['user_id'] = 'cloud_admin'
        kwargs.pop('is_cloud_admin', None)
    authorize_project_context(context, project_id)
    filter_options = {'project_id': project_id}
    valid_keys = ['type', 'user_id']
    filter_options.update({key: value for key, value in kwargs.items() if key in valid_keys})
    return model_query(context, models.Settings, **kwargs). \
        options(sa_orm.joinedload(models.Settings.metadata)). \
        filter_by(**filter_options).all()


@require_context
def setting_create(context, values):
    session = get_session()
    return _setting_update(context, values, None, False, session)


@require_context
def setting_bulk_create(context, setting_values, setting_metadata_values):
    session = get_session()
    try:
        lock.acquire()
        # insert all the settings and setting_metadata in bulk
        session.bulk_insert_mappings(models.Settings, setting_values)
        session.bulk_insert_mappings(models.SettingMetadata, setting_metadata_values)
        return True
    finally:
        lock.release()


@require_context
def setting_update(context, setting_name, values, purge_metadata=False, cloud_setting=False):
    session = get_session()
    return _setting_update(context, values, setting_name, purge_metadata, session, cloud_setting)


@require_context
def setting_delete(context, setting_name):
    session = get_session()

    try:
        setting = _setting_get(context, setting_name, session=session)
    except BaseException:
        setting = _setting_get(
            context,
            setting_name,
            session=session,
            get_hidden=True)

    for metadata_ref in setting.metadata:
        metadata_ref.purge(session=session)

    session.refresh(setting)
    setting.purge(session=session)


"""
Permanent Deletes
"""


@require_context
def purge_snapshot(context, id, session=None):
    if session is None:
        session = get_session()
    for snapshot_vm_resource in snapshot_resources_get(
            context, id, session=session):
        if snapshot_vm_resource.resource_type == 'disk':
            for vm_disk_resource_snap in vm_disk_resource_snaps_get(
                    context, snapshot_vm_resource.id, session=session):
                for metadata_ref in vm_disk_resource_snap.metadata:
                    metadata_ref.purge(session)
                session.refresh(vm_disk_resource_snap)
                vm_disk_resource_snap.purge(session)
        if snapshot_vm_resource.resource_type == 'network' or \
                snapshot_vm_resource.resource_type == 'subnet' or \
                snapshot_vm_resource.resource_type == 'router' or \
                snapshot_vm_resource.resource_type == 'nic':
            for vm_network_resource_snap in vm_network_resource_snaps_get(
                    context, snapshot_vm_resource.id, session=session):
                for metadata_ref in vm_network_resource_snap.metadata:
                    metadata_ref.purge(session)
                session.refresh(vm_network_resource_snap)
                vm_network_resource_snap.purge(session)
        if snapshot_vm_resource.resource_type == 'security_group':
            for vm_security_group_rule_snap in vm_security_group_rule_snaps_get(
                    context, snapshot_vm_resource.id, session=session):
                for metadata_ref in vm_network_resource_snap.metadata:
                    metadata_ref.purge(session)
                session.refresh(vm_security_group_rule_snap)
                vm_security_group_rule_snap.purge(session)

        for metadata_ref in snapshot_vm_resource.metadata:
            metadata_ref.purge(session)
        session.refresh(snapshot_vm_resource)
        snapshot_vm_resource.purge(session)

    for snapshot_vm in snapshot_vms_get(context, id, session=session):
        for metadata_ref in snapshot_vm.metadata:
            metadata_ref.purge(session)
        vm_recent_snapshot = vm_recent_snapshot_get(
            context, snapshot_vm.vm_id, session=session)
        if vm_recent_snapshot:
            vm_recent_snapshot.purge(session)
        session.refresh(snapshot_vm)
        snapshot_vm.purge(session)

    snapshot = snapshot_get(context, id, session=session, read_deleted='yes')
    if snapshot:
        snapshot.purge(session)


@require_admin_context
def purge_workload(context, id):
    try:
        session = get_session()
        for snapshot in snapshot_get_all(
                context, session=session, read_deleted='yes'):
            purge_snapshot(context, snapshot.id, session)
        for workload_vm in workload_vms_get(context, id, session=session):
            for metadata_ref in workload_vm.metadata:
                metadata_ref.purge(session)
            session.refresh(workload_vm)
            workload_vm.purge(session)
        workload = _workload_get(context, id, session=session)
        if workload:
            for metadata_ref in workload.metadata:
                metadata_ref.purge(session)
            session.refresh(workload)
            workload.purge(session)

    except Exception as ex:
        LOG.exception(ex)


@require_admin_context
def config_workload_update(context, values):
    session = get_session()
    return _config_workload_update(context, values, session)


@require_admin_context
def config_workload_get(context, **kwargs):
    session = get_session()
    return _config_workload_get(context, session, **kwargs)


def _config_workload_update(context, values, session):
    metadata = values.pop('metadata', {})
    try:
        config_workload_ref = _config_workload_get(context, session)
    except Exception as ex:
        config_workload_ref = models.ConfigWorkloads()
        if not values.get('id'):
            values['id'] = CONF.cloud_unique_id

    config_workload_ref.update(values)
    config_workload_ref.save(session)

    if metadata:
        _set_metadata_for_config_workload(
            context, config_workload_ref, metadata, session=session)

    return config_workload_ref


def _config_workload_get(context, session, **kwargs):
    try:
        config_workload = model_query(
            context, models.ConfigWorkloads, session=session, **kwargs). \
            options(sa_orm.joinedload(models.ConfigWorkloads.metadata)). \
            filter_by(id=CONF.cloud_unique_id).first()

        if config_workload is None:
            raise exception.ConfigWorkloadNotFound(id=CONF.cloud_unique_id)

    except sa_orm.exc.NoResultFound:
        raise exception.ConfigWorkloadNotFound(id=CONF.cloud_unique_id)

    return config_workload


def _set_metadata_for_config_workload(
        context, config_workload_ref, metadata, session):
    """
    Create or update a set of config_workload_metadata for a given config_workload
    """
    orig_metadata = {}
    for metadata_ref in config_workload_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'config_workload_id': config_workload_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _config_workload_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _config_workload_metadata_create(context, metadata_values, session)


def _config_workload_metadata_create(context, values, session):
    """Create an ConfigWorkloadMetadata object"""
    metadata_ref = models.ConfigWorkloadMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _config_workload_metadata_update(
        context, metadata_ref, values, session)


@require_admin_context
def config_workload_metadata_create(context, values, session):
    """Create an ConfigWorkloadMetadata object"""
    session = get_session()
    return _config_workload_metadata_create(context, values, session)


def _config_workload_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by config_workload_metadata_create and config_workload_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


def _config_workload_metadata_delete(context, metadata_ref, session):
    """
    Used internally by config_workload_metadata_create and config_workload_metadata_create
    """
    metadata_ref.delete(session=session)


# Workload Policy API's


@require_admin_context
def _policy_field_update(context, policy_field_id, values, session):
    if policy_field_id:
        policy_field_ref = _policy_field_get(context, policy_field_id, session)
    else:
        policy_field_ref = models.WorkloadPolicyFields()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    policy_field_ref.update(values)
    policy_field_ref.save(session)

    return policy_field_ref


@require_admin_context
def policy_field_create(context, values, **kwargs):
    session = get_session()
    return _policy_field_update(context, None, values, session)


@require_admin_context
def policy_field_update(context, id, values, **kwargs):
    session = get_session()
    return _policy_field_update(context, id, values, session)


@require_admin_context
def policy_fields_get_all(context, **kwargs):
    session = get_session()
    try:
        query = model_query(
            context, models.WorkloadPolicyFields, session=session,
            read_deleted="no")

        policy_fields = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.PolicyFieldNotFound()

    return policy_fields


@require_admin_context
def policy_field_get(context, id, **kwargs):
    session = get_session()
    return _policy_field_get(context, id, session)


@require_admin_context
def _policy_field_get(context, id, session):
    try:
        policy_field = model_query(
            context, models.WorkloadPolicyFields, session=session, read_deleted="no"). \
            filter_by(id=id).first()

    except sa_orm.exc.NoResultFound:
        raise exception.PolicyFieldNotFound(policy_field_id=id)

    if policy_field is None:
        raise exception.PolicyFieldNotFound(policy_field_id=id)

    return policy_field


@require_admin_context
def policy_field_delete(context, id, **kwargs):
    session = get_session()
    with session.begin():
        session.query(models.WorkloadPolicyFields). \
        filter_by(id=id).delete()


@require_admin_context
def _policy_update(context, values, policy_id, purge_metadata, session):
    metadata = values.pop('metadata', {})
    field_values = values.pop('field_values', {})
    if policy_id:
        policy_ref = _policy_get(context, policy_id, session)
    else:
        policy_ref = models.WorkloadPolicy()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    policy_ref.update(values)
    policy_ref.save(session)

    if metadata:
        _set_metadata_for_policy(
            context, policy_ref, metadata, purge_metadata, session)
    if field_values:
        field_values = _set_field_values_for_policy(
            context, policy_ref, field_values, session)
    return policy_ref


@require_admin_context
def policy_create(context, values, **kwargs):
    session = get_session()
    return _policy_update(context, values, None, False, session)


@require_admin_context
def policy_update(context, id, values, purge_metadata=False, **kwargs):
    session = get_session()
    return _policy_update(context, values, id, purge_metadata, session)


@require_admin_context
def policy_get_all(context, **kwargs):
    qs = model_query(context, models.WorkloadPolicy, read_deleted="no", **kwargs). \
        options(sa_orm.joinedload(models.WorkloadPolicy.metadata)). \
        options(sa_orm.joinedload(models.WorkloadPolicy.field_values))

    return qs.all()


def _policy_get(context, id, session, **kwargs):
    try:
        policy = model_query(
            context, models.WorkloadPolicy, session=session, read_deleted="no", **kwargs). \
            options(sa_orm.joinedload(models.WorkloadPolicy.metadata)). \
            options(sa_orm.joinedload(models.WorkloadPolicy.field_values)). \
            options(sa_orm.joinedload(models.WorkloadPolicy.policy_assignments)). \
            filter_by(id=id).first()

        if policy is None:
            raise exception.PolicyNotFound(policy_id=id)

    except sa_orm.exc.NoResultFound:
        raise exception.PolicyNotFound(policy_id=id)

    return policy


@require_context
def policy_get(context, id, **kwargs):
    session = get_session()
    return _policy_get(context, id, session, **kwargs)


@require_admin_context
def policy_delete(context, id, **kwargs):
    session = get_session()
    with session.begin():
        session.query(models.WorkloadPolicy). \
            filter_by(id=id).delete()


def _set_metadata_for_policy(context, policy_ref, metadata,
                             purge_metadata, session):
    """
    Create or update a set of policy_metadata for a given policy
    :param context: Request context
    :param policy_ref: An policy object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in policy_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'policy_id': policy_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _policy_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _policy_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _policy_metadata_delete(context, metadata_ref, session)


def _policy_metadata_create(context, values, session):
    """Create an WorkloadPolicyMetadata object"""
    metadata_ref = models.WorkloadPolicyMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _policy_metadata_update(context, metadata_ref, values, session)


@require_admin_context
def policy_metadata_create(context, values, **kwargs):
    """Create an WorkloadPolicyMetadata object"""
    session = get_session()
    return _policy_metadata_create(context, values, session)


def _policy_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by policy_metadata_create and policy_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


def _policy_metadata_delete(context, metadata_ref, session):
    """
    Used internally by policy_metadata_create and policy_metadata_update
    """
    metadata_ref.delete(session=session)
    return metadata_ref


def _set_field_values_for_policy(context, policy_ref, policy_field_values,
                                 session):
    """
    Create or update a set of policy_values for a given policy
    :param context: Request context
    :param policy_ref: An policy object
    :param policy_field_values: A dict of policy values to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_field_values = {}
    for field_value in policy_ref.field_values:
        orig_field_values[field_value.policy_field_name] = field_value

    for field_name, field_value in policy_field_values.items():
        field_values = {'policy_id': policy_ref.id,
                        'policy_field_name': field_name,
                        'value': field_value}
        if field_name in orig_field_values:
            field_value_ref = orig_field_values[field_name]
            _policy_value_update(context, field_value_ref,
                                 field_values, session)
        else:
            _policy_value_create(context, field_values, session)


def _policy_value_create(context, values, session):
    """Create an WorkloadPolicyValues object"""
    policy_value_ref = models.WorkloadPolicyValues()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _policy_value_update(context, policy_value_ref, values, session)


@require_admin_context
def policy_value_create(context, values, **kwargs):
    """Create an WorkloadPolicyValues object"""
    session = get_session()
    return _policy_value_create(context, values, session)


def _policy_value_update(context, policy_value_ref, values, session):
    """
    Used internally by policy_value_create and policy_value_update
    """
    values["deleted"] = False
    policy_value_ref.update(values)
    policy_value_ref.save(session=session)
    return policy_value_ref


@require_admin_context
def policy_values_get_all(context, **kwargs):
    session = get_session()
    try:
        query = model_query(
            context, models.WorkloadPolicyValues, session=session,
            read_deleted="no")

        policy_values = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.PolicyValueNotFound()

    return policy_values


@require_admin_context
def policy_value_get(context, id, **kwargs):
    session = get_session()
    try:
        query = session.query(models.WorkloadPolicyValues) \
            .filter_by(id=id)

        policy_values = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.PolicyValueNotFound(policy_value_id=id)

    if policy_values is None:
        raise exception.PolicyValueNotFound(policy_value_id=id)

    return policy_values


@require_admin_context
def policy_value_delete(context, id, **kwargs):
    session = get_session()
    with session.begin():
        session.query(models.WorkloadPolicyValues). \
            filter_by(id=id).delete()


def _policy_assignment_update(context, values, policy_assignment_id, session):
    if policy_assignment_id:
        policy_assignment_ref = _policy_assignment_get(
            context, policy_assignment_id, session)
    else:
        policy_assignment_ref = models.WorkloadPolicyAssignmnets()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    policy_assignment_ref.update(values)
    policy_assignment_ref.save(session)

    return policy_assignment_ref


@require_admin_context
def policy_assignment_create(context, values, **kwargs):
    session = get_session()
    return _policy_assignment_update(context, values, None, session)


@require_admin_context
def policy_assignment_update(context, id, values, **kwargs):
    session = get_session()
    return _policy_assignment_update(context, values, id, session)


@require_context
def policy_assignments_get_all(context, **kwargs):
    session = get_session()
    query = session.query(models.WorkloadPolicyAssignmnets)

    if 'policy_id' in kwargs and kwargs['policy_id'] is not None:
        query = query.filter_by(policy_id=kwargs['policy_id'])

    if 'project_id' in kwargs and kwargs['project_id'] is not None:
        query = query.filter_by(project_id=kwargs['project_id'])

    if (kwargs.get('workloads', False) is True) \
            and ('policy_id' in kwargs and kwargs['policy_id'] is not None):
        query = session.query(models.Workloads.id).join(models.WorkloadMetadata).filter(
            and_(models.WorkloadMetadata.key == 'policy_id'), models.WorkloadMetadata.value == kwargs['policy_id'],
                                                              models.Workloads.deleted == False)
        if 'project_id' in kwargs and kwargs['project_id'] is not None:
            query = query.filter(
                models.Workloads.project_id == kwargs['project_id'])

    policy_assignments = query.all()
    return policy_assignments


def _policy_assignment_get(context, id, session, **kwargs):
    try:
        policy_assignments = model_query(
            context, models.WorkloadPolicyAssignmnets, session=session). \
            filter_by(id=id).first()

    except sa_orm.exc.NoResultFound:
        raise exception.PolicyAssignmentNotFound(policy_assignment_id=id)

    if policy_assignments is None:
        raise exception.PolicyAssignmentNotFound(policy_assignment_id=id)

    return policy_assignments


@require_admin_context
def policy_assignment_get(context, id, **kwargs):
    session = get_session()
    return _policy_assignment_get(context, id, session)


@require_admin_context
def policy_assignment_delete(context, id, **kwargs):
    session = get_session()
    with session.begin():
        session.query(models.WorkloadPolicyAssignmnets). \
            filter_by(id=id).delete()


@require_admin_context
def get_tenants_usage(context, **kwargs):
    """Give storage used by tenant workloads"""
    try:
        tenant_chargeback = {}
        session = get_session()
        qry = session.query(models.Snapshots.project_id, func.sum(cast(models.Snapshots.size, Integer))). \
            group_by(models.Snapshots.project_id).filter_by(deleted=False)

        result = qry.all()
        for proj_id, storage_used in result:
            if proj_id not in tenant_chargeback:
                tenant_chargeback[proj_id] = {}
                tenant_chargeback[proj_id]['vms_protected'] = []
            tenant_chargeback[proj_id]['used_capacity'] = int(storage_used)

        qry = session.query(models.Workloads.project_id, models.WorkloadVMs.vm_id).join(
            models.WorkloadVMs).filter_by(deleted=False)
        result = qry.all()
        for proj_id, vm_protected in result:
            if proj_id not in tenant_chargeback:
                tenant_chargeback[proj_id] = {
                    'vms_protected': [], 'used_capacity': 0}
            tenant_chargeback[proj_id]['vms_protected'].append(vm_protected)

        return tenant_chargeback
    except Exception as ex:
        LOG.exception(ex)


""" snapshot network resource methods """


@require_context
def snapshot_network_resource_create(context, values):
    session = get_session()
    return _snapshot_network_resource_update(context, None, values, False, session)


@require_context
def snapshot_network_resource_update(context, id, values, purge_metadata=False):
    session = get_session()
    return _snapshot_network_resource_update(context, id, values,
                                             purge_metadata, session)


def _snapshot_network_resource_update(context, resource_id, values, purge_metadata, session):
    try:
        with lock:
            metadata = values.pop('metadata', {})

            if resource_id:
                resource_ref = model_query(
                    context,
                    models.SnapNetworkResources,
                    session=session).filter_by(
                    id=resource_id).first()
                if not resource_ref:
                    raise exception.NetworkResourceNotFound(
                        resource_id=resource_id)

            else:
                resource_ref = models.SnapNetworkResources()
                values['id'] = values.get('id', str(uuid.uuid4()))
                values['name'] = values.get('name', "")
                values['type'] = values.get('type', "")
                values['status'] = values.get('status', "available")
            resource_ref.update(values)
            resource_ref.save(session)

            if metadata:
                _set_metadata_for_snap_network_resource(
                    context,
                    resource_ref,
                    metadata,
                    purge_metadata,
                    session=session)

        return resource_ref
    except Exception as ex:
        LOG.exception(ex)
        raise ex


@require_context
def snapshot_network_resource_get(context, resource_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    return _snapshot_network_resource_get(context, resource_id, **kwargs)


def _snapshot_network_resource_get(context, resource_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    result = model_query(context, models.SnapNetworkResources, **kwargs). \
        options(sa_orm.joinedload(models.SnapNetworkResources.metadata)). \
        filter_by(id=resource_id). \
        first()

    if not result:
        raise exception.NetworkResourceNotFound(resource_id=resource_id)

    return result


@require_context
def snapshot_network_resources_get(context, snapshot_id, **kwargs):
    try:
        qs = model_query(context, models.SnapNetworkResources, **kwargs). \
            options(sa_orm.joinedload(models.SnapNetworkResources.metadata)). \
            filter_by(snapshot_id=snapshot_id)

        if 'type' in kwargs:
            qs = qs.filter(models.SnapNetworkResources.type == kwargs['type'])
    except sa_orm.exc.NoResultFound:
        raise exception.NetworkResourcesNotFound(snapshot_id=snapshot_id)

    return qs.all()


@require_context
def snapshot_network_resource_delete(context, resource_id):
    session = get_session()
    with session.begin():
        session.query(models.SnapNetworkResources). \
            filter_by(id=resource_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


# Metadata


@require_context
def snapshot_network_resource_metadata_create(context, values):
    session = get_session()
    return _snapshot_network_resource_metadata_create(context, values, session)


def _snapshot_network_resource_metadata_create(context, values, session):
    """Create an SnapshotVMResourceMetadata object"""
    metadata_ref = models.SnapNetworkResourceMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _snapshot_network_resource_metadata_update(
        context, metadata_ref, values, session)


def _snapshot_network_resource_metadata_update(context, metadata_ref, values, session):
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def snapshot_network_resource_metadata_delete(context, metadata_ref, session):
    metadata_ref.delete(session=session)


def _set_metadata_for_snap_network_resource(context, resource_ref, metadata,
                                            purge_metadata, session):
    orig_metadata = {}
    for metadata_ref in resource_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'snap_network_resource_id': resource_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _snapshot_network_resource_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            snapshot_network_resource_metadata_create(context, metadata_values)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                snapshot_network_resource_metadata_delete(
                    context, metadata_ref, session=session)


""" Scheduled job resource methods """


@require_context
def job_create(context, values, **kwargs):
    """ Create new scheduled job """
    session = get_session()
    return _job_update(context, values, session, None)


@require_context
def job_update(context, job_id, values, **kwargs):
    """ Update existing scheduled jobs """
    if not job_id:
        message = "Please provide job_id to update"
        LOG.exception(message)
        raise Exception(message)
    session = get_session()
    return _job_update(context, values, session, job_id)


def _job_update(context, values, session, job_id):
    job_ref = []
    if job_id:
        kwargs = {'id': job_id}
        job_ref = _job_get(context, session, **kwargs)
        if not job_ref:
            LOG.exception("No job found with job_id: %s for workload: %s" % (job_id, values['workload_id']))
            raise exception.ScheduledJobsNotFound(workload_id=values['workload_id'])
        else:
            job_ref = job_ref[0]
    else:
        job_ref = models.ScheduledJobs()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    if not job_ref:
        raise Exception("No job found to update")

    job_ref.update(values)
    job_ref.save(session)

    return job_ref


@require_context
def job_get_all(context, **kwargs):
    """List all scheduled jobs"""
    session = get_session()
    return _job_get(context, session, None, **kwargs)


@require_context
def job_get(context, workload_id, **kwargs):
    """Show given job"""
    session = get_session()
    jobs = _job_get(context, session, workload_id, **kwargs)
    if jobs:
        return jobs[0]


def _job_get(context, session, workload_id=None, **kwargs):
    try:
        if kwargs:
            filter_params = set(kwargs.keys())
            model_params = set(models.ScheduledJobs.__dict__.keys())
            invalid_params = list(filter_params - model_params)
            if invalid_params:
                raise Exception('Can not filter jobs with these keys: %s' \
                                % (str(invalid_params)))

        qs = model_query(context, models.ScheduledJobs, session=session, **kwargs)

        if workload_id:
            qs = qs.filter(models.ScheduledJobs.workload_id == workload_id)

        if 'id' in kwargs:
            qs = qs.filter(models.ScheduledJobs.id == kwargs['id'])

        return qs.all()
    except sa_orm.exc.NoResultFound:
        raise exception.ScheduledJobsNotFound(workload_id=workload_id)


@require_context
def job_delete(context, workload_id, **kwargs):
    """Delete given job"""
    session = get_session()
    with session.begin():
        session.query(models.ScheduledJobs). \
            filter_by(workload_id=workload_id). \
            delete()


@require_context
def job_multiple_delete(context, job_ids):
    """Delete given jobs"""
    session = get_session()
    with session.begin():
        session.query(models.ScheduledJobs). \
            filter(models.ScheduledJobs.id.in_(job_ids)).\
            delete(synchronize_session=False)


@require_context
def _migration_plan_metadata_create(context, values, session):
    """Create an WorkloadMetadata object"""
    metadata_ref = models.MigrationPlanMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_plan_metadata_update(context, metadata_ref, values, session)

@require_context
def migration_plan_metadata_create(context, values, session):
    """Create an MigrationPlanMetadata object"""
    session = get_session()
    return _migration_plan_metadata_create(context, values, session)


@require_context
def _migration_plan_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by migration_plan_metadata_create and migration_plan_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _migration_plan_metadata_delete(context, metadata_ref, session):
    """
    Used internally by migration_plan_metadata_create and migration_plan_metadata_update
    """
    metadata_ref.delete(session=session)


def _set_metadata_for_migration_plan(context, migration_plan_ref, metadata,
                                     purge_metadata, session):
    """
    Create or update a set of migration_plan_metadata for a given migration_plan
    :param context: Request context
    :param migration_plan_ref: An migration_plan object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in migration_plan_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'migration_plan_id': migration_plan_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_plan_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_plan_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_plan_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def migration_plan_create(context, values):
    session = get_session()
    return _migration_plan_update(context, values, None, False, session)


@require_context
def migration_plan_delete(context, id):
    session = get_session()
    with session.begin():
        session.query(models.MigrationPlans). \
            filter_by(id=id).delete()


@require_context
def migration_plan_update(context, id, values, purge_metadata=False):
    session = get_session()
    return _migration_plan_update(context, values, id, purge_metadata, session)


def _migration_plan_update(context, values, migration_plan_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if migration_plan_id:
        migration_plan_ref = _migration_plan_get(context, migration_plan_id, session)
    else:
        migration_plan_ref = models.MigrationPlans()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    migration_plan_ref.update(values)
    migration_plan_ref.save(session)

    if metadata:
        _set_metadata_for_migration_plan(
            context,
            migration_plan_ref,
            metadata,
            purge_metadata,
            session=session)

    return migration_plan_ref


@require_context
def _migration_plan_get(context, mp_id, session, **kwargs):
    try:
        migration_plan = model_query(
            context, models.MigrationPlans, session=session, **kwargs). \
            options(sa_orm.joinedload(models.MigrationPlans.metadata)). \
            filter_by(id=mp_id).all()

        if not migration_plan:
            raise exception.MigrationPlanNotFound(migration_plan_id=mp_id)
        return migration_plan[0]
    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanNotFound(migration_plan_id=mp_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.MigrationPlanNotFound(migration_plan_id=mp_id)


@require_context
def migration_plan_resources_get(context, migration_plan_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.MigrationPlanVMResources).options(
            sa_orm.joinedload(
                models.MigrationPlanVMResources.metadata)).filter_by(
            migration_plan_id=migration_plan_id)

        migration_plan_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanResourcesNotFound(migration_plan_id=migration_plan_id)

    return migration_plan_resources


@require_context
def migration_plan_vms_get(context, migration_plan_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        if migration_plan_id:
            query = model_query(context, models.MigrationPlanVMs,
                                session=session, read_deleted="no") \
                .options(sa_orm.joinedload(models.MigrationPlanVMs.metadata)) \
                .filter_by(migration_plan_id=migration_plan_id) \
                .filter(models.MigrationPlanVMs.status is not None)
        else:
            query = model_query(context, models.MigrationPlanVMs,
                                session=session, read_deleted="no") \
                .options(sa_orm.joinedload(models.MigrationPlanVMs.metadata)) \
                .filter(models.MigrationPlanVMs.status is not None)

        # TODO(gbasava): filter out deleted workload_vms if context disallows
        # it
        migration_plan_vms = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.WorkloadVMsNotFound(migration_plan_id=migration_plan_id)

    return migration_plan_vms


@require_context
def migration_plan_vms_delete(context, vm_id, migration_plan_id):
    session = get_session()
    with session.begin():
        session.query(models.MigrationPlanVMs). \
            filter_by(vm_id=vm_id). \
            filter_by(migration_plan_id=migration_plan_id).delete()


@require_context
def migration_plan_vm_get_by_id(context, vm_id, **kwargs):
    session = kwargs.get('session') or get_session()
    read_deleted = 'no'
    if 'read_deleted' in kwargs:
        read_deleted = kwargs['read_deleted']
    try:
        query = model_query(context, models.MigrationPlanVMs,
                            session=session, read_deleted=read_deleted) \
            .options(sa_orm.joinedload(models.MigrationPlanVMs.metadata)) \
            .join(models.MigrationPlans) \
            .filter(models.MigrationPlanVMs.status is not None) \
            .filter(models.MigrationPlanVMs.vm_id == vm_id) \
            .filter(models.MigrationPlans.project_id == context.project_id)

        if 'migration_plans_filter' in kwargs:
            query = query.filter(
                and_(
                    models.MigrationPlans.status != kwargs['migration_plans_filter'],
                    models.MigrationPlans.status is not None))
        if 'migration_plan_id' in kwargs:
            query = query.filter(
                and_(
                    models.MigrationPlans.id == kwargs['migration_plan_id']))

        vm_found = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMsNotFound(vm_id=vm_id)

    return vm_found


@require_context
def migration_plan_vms_create(context, values):
    session = get_session()
    return _migration_plan_vms_update(context, values, None, False, session)


@require_context
def migration_plan_vms_update(context, id, values, purge_metadata=False):
    session = get_session()
    return _migration_plan_vms_update(context, values, id, purge_metadata, session)


def _migration_plan_vms_update(context, values, id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if id:
        migration_plan_vm_ref = _migration_plan_vm_get_for_update(context, id, session)
    else:
        migration_plan_vm_ref = models.MigrationPlanVMs()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    migration_plan_vm_ref.update(values)
    migration_plan_vm_ref.save(session)

    if metadata:
        _set_metadata_for_migration_plan_vms(
            context,
            migration_plan_vm_ref,
            metadata,
            purge_metadata,
            session=session)

    return migration_plan_vm_ref


@require_context
def _migration_plan_vm_get_for_update(context, id, session):
    try:
        query = session.query(
            models.MigrationPlanVMs).options(
            sa_orm.joinedload(
                models.MigrationPlanVMs.metadata)).filter_by(
            id=id)
        # TODO(gbasava): filter out deleted migration_plan_vms if context disallows
        # it
        migration_plan_vm = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMNotFound(migration_plan_vm_id=id)

    return migration_plan_vm


@require_context
def _migration_plan_vms_metadata_update(context, metadata_ref, values, session):
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _migration_plan_vms_metadata_create(context, values, session):
    """Create an MigrationPlanMetadata object"""
    metadata_ref = models.MigrationPlanVMMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_plan_vms_metadata_update(
        context, metadata_ref, values, session)


def _set_metadata_for_migration_plan_vms(context, migration_plan_vm_ref, metadata,
                                         purge_metadata, session):
    """
    Create or update a set of workload_vms_metadata for a given workload_vm
    :param context: Request context
    :param workload_vm_ref: An workload_vm object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in migration_plan_vm_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'migration_plan_vm_id': migration_plan_vm_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_plan_vms_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_plan_vms_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_plan_vms_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def migration_plan_get_by_vmid(context, **kwargs):
    session = kwargs.get('session') or get_session()
    vms_obj = model_query(context, models.MigrationPlanVMs,
                                session=session, read_deleted="no") \
                .filter(models.MigrationPlanVMs.status is not None).filter_by(vm_id=kwargs.get('vm_id')).first()
    if vms_obj:
        qs = model_query(context, models.MigrationPlans, **kwargs). \
            options(sa_orm.joinedload(models.MigrationPlans.metadata)).filter_by(project_id=context.project_id, id=vms_obj.migration_plan_id)
        return qs.all()


@require_context
def migration_plan_get_all(context, **kwargs):
    qs = model_query(context, models.MigrationPlans, **kwargs). \
        options(sa_orm.joinedload(models.MigrationPlans.metadata))

    if is_admin_context(context):
        if 'project_id' in kwargs and kwargs['project_id'] is not None and kwargs['project_id'] != '':
            qs = qs.filter_by(project_id=kwargs['project_id'])
        elif 'all_migration_plans' in kwargs and kwargs['all_migration_plans'] is not True:
            qs = qs.filter_by(project_id=context.project_id)
    else:
        qs = qs.filter_by(project_id=context.project_id)

    if 'project_list' in kwargs and 'user_list' in kwargs:
        project_list = kwargs['project_list']
        user_list = kwargs['user_list']
        if isinstance(project_list, list) and isinstance(user_list, list):
            if 'exclude' in kwargs and kwargs['exclude'] is True:
                qs = qs.filter(models.MigrationPlans.project_id.notin_(
                    project_list) | models.MigrationPlans.user_id.notin_(user_list))
            else:
                qs = qs.filter(
                    models.MigrationPlans.project_id.in_(project_list),
                    models.MigrationPlans.user_id.in_(user_list))
        else:
            error = _('Project list and user list should be list')
            raise exception.ErrorOccurred(reason=error)

    if 'project_list' in kwargs and 'user_list' not in kwargs:
        project_list = kwargs['project_list']
        qs = model_query(context, models.MigrationPlans.id, **kwargs)
        if isinstance(project_list, list):
            if 'exclude_project' in kwargs and kwargs['exclude_project'] is True:
                qs = qs.filter(
                    (models.MigrationPlans.project_id.notin_(project_list)))
            else:
                qs = qs.filter((models.MigrationPlans.project_id.in_(project_list)))
        else:
            error = _('Project list should be list')
            raise exception.ErrorOccurred(reason=error)

    if 'migration_plan_list' in kwargs:
        migration_plan_list = kwargs['migration_plan_list']
        if isinstance(migration_plan_list, list):
            if len(migration_plan_list):
                if 'exclude_migration_plan' in kwargs and kwargs['exclude_migration_plan'] is True:
                    qs = qs.filter(and_(models.MigrationPlans.id.notin_(migration_plan_list)))
                else:
                    qs = qs.filter(and_(models.MigrationPlans.id.in_(migration_plan_list)))
        else:
            error = _('MigrationPlan list should be list')
            raise exception.ErrorOccurred(reason=error)

    qs = qs.order_by(models.MigrationPlans.created_at.desc())

    if 'page_number' in kwargs and kwargs['page_number'] is not None and kwargs['page_number'] != '':
        page_size = setting_get(context, 'page_size')
        return qs.limit(int(page_size)).offset(
            int(page_size) * (int(kwargs['page_number']) - 1)).all()
    else:
        return qs.all()


@require_context
def migration_get_all(context, migration_plan_id=None, **kwargs):
    if not is_admin_context(context):
        if migration_plan_id:
            return migration_get_all_by_migration_plan(
                context, migration_plan_id, **kwargs)
        else:
            return migration_get_all_by_project(
                context, context.project_id, **kwargs)

    if migration_plan_id is None:
        return model_query(context, models.Migrations, **kwargs). \
            options(sa_orm.joinedload(models.Migrations.metadata)). \
            order_by(models.Migrations.created_at.desc()).all()
    else:
        return model_query(context, models.Migrations, **kwargs). \
            options(sa_orm.joinedload(models.Migrations.metadata)). \
            filter_by(migration_plan_id=migration_plan_id). \
            order_by(models.Migrations.created_at.desc()).all()


def migration_get_all_by_migration_plan(context, migration_plan_id, **kwargs):
    return model_query(context, models.Migrations, **kwargs). \
            options(sa_orm.joinedload(models.Migrations.metadata)). \
            filter_by(migration_plan_id=migration_plan_id). \
            order_by(models.Migrations.created_at.desc()).all()


def migration_get_all_by_project(context, project_id, **kwargs):
    return model_query(context, models.Migrations, **kwargs). \
            options(sa_orm.joinedload(models.Migrations.metadata)). \
            filter_by(project_id=project_id). \
            order_by(models.Migrations.created_at.desc()).all()


@require_context
def migration_plan_get(context, id, **kwargs):
    session = get_session()
    with session.begin():
        return _migration_plan_get(context, id, session, **kwargs)

@require_context
def _migration_plan_get(context, migration_plan_id, session, **kwargs):
    try:
        migration_plan = model_query(
            context, models.MigrationPlans, session=session, **kwargs). \
            options(sa_orm.joinedload(models.MigrationPlans.metadata)). \
            filter_by(id=migration_plan_id).all()

        if not migration_plan:
            raise exception.MigrationPlanNotFound(migration_plan_id=migration_plan_id)
        return migration_plan[0]
    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanNotFound(migration_plan_id=migration_plan_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.MigrationPlanNotFound(migration_plan_id=migration_plan_id)


def _set_metadata_for_migration_plan_vms(context, migration_plan_vm_ref, metadata,
                                         purge_metadata, session):
    """
    :param context: Request context
    :param snapshot_vm_ref: An migration_plan_vm object
    :param metadata: A dict of metadata to set
    :param session: A SQLAlchemy session to use (if present)
    """
    orig_metadata = {}
    for metadata_ref in migration_plan_vm_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'migration_plan_vm_id': migration_plan_vm_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_plan_vms_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_plan_vms_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_plan_vms_metadata_delete(
                    context, metadata_ref, session=session)


def _migration_plan_vms_metadata_delete(context, metadata_ref, session):
    metadata_ref.delete(session=session)


@require_context
def _migration_plan_vm_resource_get(context, id, session):
    try:
        query = session.query(
            models.MigrationPlanVMResources).options(
            sa_orm.joinedload(
                models.MigrationPlanVMResources.metadata)).filter_by(
            id=id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        migration_plan_vm_resource = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMResourceNotFound(migration_plan_vm_resource_id=id)

    return migration_plan_vm_resource


@require_context
def migration_plan_vm_resource_get(context, id):
    session = get_session()
    return _migration_plan_vm_resource_get(context, id, session)


@require_context
def _migration_plan_vm_resource_update(
        context, values, migration_plan_vm_resource_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if migration_plan_vm_resource_id:
        migration_plan_vm_resource_ref = _migration_plan_vm_resource_get(
            context, snapshot_vm_resource_id, session=session)
    else:
        migration_plan_vm_resource_ref = models.MigrationPlanVMResources()
        if not values.get('size'):
            values['size'] = 0
        if not values.get('restore_size'):
            values['restore_size'] = 0

    migration_plan_vm_resource_ref.update(values)
    migration_plan_vm_resource_ref.save(session)

    _set_metadata_for_migration_plan_vm_resource(
        context,
        migration_plan_vm_resource_ref,
        metadata,
        purge_metadata,
        session)

    return migration_plan_vm_resource_ref


@require_context
def migration_plan_vm_resource_create(context, values):
    session = get_session()
    return _migration_plan_vm_resource_update(context, values, None, False, session)


@require_context
def migration_plan_vm_resource_delete(context, id):
    session = get_session()
    with session.begin():
        session.query(models.MigrationPlanVMResources). \
            filter_by(id=id). \
            delete()


@require_context
def migration_plan_vm_resource_update(
        context, migration_plan_vm_resource_id, values, purge_metadata=False):
    session = get_session()
    return _migration_plan_vm_resource_update(
        context, values, migration_plan_vm_resource_id, purge_metadata, session)


@require_context
def migration_plan_resources_get(context, migration_plan_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.MigrationPlanVMResources).options(
            sa_orm.joinedload(
                models.MigrationPlanVMResources.metadata)).filter_by(
            migration_plan_id=migration_plan_id)

        migration_plan_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanResourcesNotFound(migration_plan_id=migration_plan_id)

    return migration_plan_resources


@require_context
def migration_plan_get_metadata_cancel_flag(
        context, migration_plan_id, return_val=0, process=None, **kwargs):
    flag = '0'
    migration_plan_obj = migration_plan_get(context, migration_plan_id)
    for meta in migration_plan_obj.metadata:
        if meta.key == 'cancel_requested':
            flag = meta.value

    if return_val == 1:
        return flag

    if flag == '1':
        if process:
            process.kill()
        error = _('Cancel requested for snapshot')
        raise exception.ErrorOccurred(reason=error)

@require_context
def _set_metadata_for_migration_plan_vm_resource(
        context,
        migration_plan_vm_resource_ref,
        metadata,
        purge_metadata,
        session):

    orig_metadata = {}
    for metadata_ref in migration_plan_vm_resource_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'migration_plan_vm_resource_id': migration_plan_vm_resource_ref.id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_plan_vm_resource_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_plan_vm_resource_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_plan_vm_resource_metadata_delete(
                    context, metadata_ref, session)


@require_context
def _migration_plan_vm_resource_metadata_create(context, values, session):
    """Create an MigrationPlanVMResourceMetadata object"""
    metadata_ref = models.MigrationPlanVMResourceMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_plan_vm_resource_metadata_update(
        context, metadata_ref, values, session)


@require_context
def migration_plan_vm_resource_metadata_create(context, values):
    session = get_session()
    return _migration_plan_vm_resource_metadata_create(context, values, session)


@require_context
def _migration_plan_vm_resource_metadata_update(
        context, metadata_ref, values, session):
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _migration_plan_vm_resource_metadata_delete(context, metadata_ref, session):
    metadata_ref.delete(session=session)


@require_context
def _migration_plan_vm_resource_update(
        context, values, migration_plan_vm_resource_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if migration_plan_vm_resource_id:
        migration_plan_vm_resource_ref = _migration_plan_vm_resource_get(
            context, migration_plan_vm_resource_id, session=session)
    else:
        migration_plan_vm_resource_ref = models.MigrationPlanVMResources()
        if not values.get('size'):
            values['size'] = 0

    migration_plan_vm_resource_ref.update(values)
    migration_plan_vm_resource_ref.save(session)

    _set_metadata_for_migration_plan_vm_resource(
        context,
        migration_plan_vm_resource_ref,
        metadata,
        purge_metadata,
        session)

    return migration_plan_vm_resource_ref


@require_context
def migration_plan_vm_resource_create(context, values):
    session = get_session()
    return _migration_plan_vm_resource_update(context, values, None, False, session)


@require_context
def migration_plan_vm_resource_update(
        context, migration_plan_vm_resource_id, values, purge_metadata=False):
    session = get_session()
    return _migration_plan_vm_resource_update(
        context, values, migration_plan_vm_resource_id, purge_metadata, session)


@require_context
def migration_plan_vm_resources_get(context, vm_id, migration_plan_id):
    session = get_session()
    try:
        query = session.query(
            models.MigrationPlanVMResources).options(
            sa_orm.joinedload(
                models.MigrationPlanVMResources.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            migration_plan_id=migration_plan_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        migration_plan_vm_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMResourcesNotFound(
            migration_plan_vm_id=vm_id, migration_plan_id=migration_plan_id)

    return migration_plan_vm_resources

def _migration_plan_set_metadata_for_vm_network_resource(
        context,
        vm_network_resource_ref,
        metadata,
        purge_metadata,
        session):

    orig_metadata = {}
    for metadata_ref in vm_network_resource_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'migration_plan_vm_network_resource_id': vm_network_resource_ref.migration_plan_vm_network_resource_id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_plan_vm_network_resource_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_plan_vm_network_resource_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_plan_vm_network_resource_metadata_delete(
                    context, metadata_ref, session)


def _set_metadata_for_vm_network_resource(
        context,
        vm_network_resource_ref,
        metadata,
        purge_metadata,
        session):
    orig_metadata = {}
    for metadata_ref in vm_network_resource_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'migration_plan_vm_network_resource_id': vm_network_resource_ref.vm_network_resource_id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _vm_network_resource_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _vm_network_resource_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _vm_network_resource_metadata_delete(
                    context, metadata_ref, session)


@require_context
def _migration_plan_vm_network_resource_metadata_create(context, values, session):
    """Create an MigrationPlanVMNetworkResourceMetadata object"""
    metadata_ref = models.MigrationPlanVMNetworkResourceMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_plan_vm_network_resource_metadata_update(
        context, metadata_ref, values, session)


@require_context
def migration_plan_vm_network_resource_metadata_create(context, values):
    """Create an MigrationPlanVMNetworkResourceMetadata object"""
    session = get_session()
    return _migration_plan_vm_network_resource_metadata_create(context, values, session)


def _migration_plan_vm_network_resource_metadata_update(
        context, metadata_ref, values, session):
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _migration_plan_vm_network_resource_metadata_delete(context, metadata_ref, session):
    metadata_ref.delete(session=session)
    return metadata_ref


def _migration_plan_vm_network_resource_update(
        context, migration_plan_vm_network_resource_id, values, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if migration_plan_vm_network_resource_id:
        migration_plan_vm_network_resource_ref = migration_plan_vm_network_resource_get(
            context, migration_plan_vm_network_resource_id, session)
    else:
        migration_plan_vm_network_resource_ref = models.MigrationPlanVMNetworkResource()

    migration_plan_vm_network_resource_ref.update(values)
    migration_plan_vm_network_resource_ref.save(session)

    _migration_plan_set_metadata_for_vm_network_resource(
        context,
        migration_plan_vm_network_resource_ref,
        metadata,
        purge_metadata,
        session=session)

    return migration_plan_vm_network_resource_ref


@require_context
def migration_plan_vm_network_resource_create(context, values):
    session = get_session()
    return _migration_plan_vm_network_resource_update(
        context, None, values, False, session)


@require_context
def migration_plan_vm_network_resource_update(
        context, migration_plan_vm_network_resource_id, values, purge_metadata=False):
    session = get_session()
    return _migration_plan_vm_network_resource_update(
        context, values, vm_network_resource_id, purge_metadata, session)


@require_context
def migration_plan_vm_network_resources_get(context, migration_plan_vm_resource_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.MigrationPlanVMNetworkResource).options(
            sa_orm.joinedload(
                models.MigrationPlanVMNetworkResource.metadata)).filter_by(
            migration_plan_vm_network_resource_id=migration_plan_vm_resource_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        migration_plan_vm_network_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMNetworkResourceNotFound(
            migration_plan_vm_resource_id=migration_plan_vm_resource_id)

    return migration_plan_vm_network_resources


@require_context
def migration_plan_vm_network_resource_get(context, vm_network_resource_id):
    session = get_session()
    try:
        query = session.query(
            models.MigrationPlanVMNetworkResource).options(
            sa_orm.joinedload(
                models.MigrationPlanVMNetworkResource.metadata)).filter_by(
            migration_plan_vm_network_resource_id=vm_network_resource_id)

        migration_plan_vm_network_resource = query.one()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMNetworkResourceNotFound(
            migration_plan_vm_network_resource_id=vm_network_resource_id)

    return migration_plan_vm_network_resource


@require_context
def migration_plan_vm_network_resource_delete(context, vm_network_resource_id):
    session = get_session()
    with session.begin():
        session.query(models.MigrationPlanVMNetworkResource). \
            filter_by(migration_plan_vm_network_resource_id=vm_network_resource_id). \
            delete()


def _migration_plan_vm_disk_resource_metadata_update(
        context, metadata_ref, values, session):
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


def _migration_plan_vm_disk_resource_metadata_delete(context, metadata_ref, session):
    metadata_ref.delete(session=session)


@require_context
def _migration_plan_vm_disk_resource_metadata_create(context, values, session):
    metadata_ref = models.MigrationPlanVMDiskResourceMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_plan_vm_disk_resource_metadata_update(
        context, metadata_ref, values, session)


def _migration_plan_set_metadata_for_vm_disk_resource(
        context,
        vm_disk_resource_ref,
        metadata,
        purge_metadata,
        session):
    orig_metadata = {}
    for metadata_ref in vm_disk_resource_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'migration_plan_vm_disk_resource_id': vm_disk_resource_ref.id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_plan_vm_disk_resource_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_plan_vm_disk_resource_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_plan_vm_disk_resource_metadata_delete(
                    context, metadata_ref, session)


def _migration_plan_vm_disk_resource_update(
        context, values, vm_disk_resource_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if vm_disk_resource_id:
        vm_disk_resource_ref = _migration_plan_vm_disk_resource_get(
            context, vm_disk_resource_id, session)
    else:
        vm_disk_resource_ref = models.MigrationPlanVMDiskResource()
        if not values.get('size'):
            values['size'] = 0
        if not values.get('restore_size'):
            values['restore_size'] = 0

    vm_disk_resource_ref.update(values)
    vm_disk_resource_ref.save(session)

    _migration_plan_set_metadata_for_vm_disk_resource(
        context,
        vm_disk_resource_ref,
        metadata,
        purge_metadata,
        session)

    return vm_disk_resource_ref


@require_context
def migration_plan_vm_disk_resource_create(context, values):
    session = get_session()
    return _migration_plan_vm_disk_resource_update(context, values, None, False, session)


@require_context
def migration_plan_vm_disk_resource_update(
        context, vm_disk_resource_id, values, purge_metadata=False):
    session = get_session()
    return _migration_plan_vm_disk_resource_update(
        context, values, vm_disk_resource_id, purge_metadata, session)


@require_context
def migration_plan_vm_disk_resources_get(context, migration_plan_vm_resource_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.MigrationPlanVMDiskResource).options(
            sa_orm.joinedload(
                models.MigrationPlanVMDiskResource.metadata)).filter_by(
            migration_plan_vm_resource_id=migration_plan_vm_resource_id)

        vm_disk_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.VMDiskResourceNotFound(
            migration_plan__vm_resource_id=migration_plan__vm_resource_id)

    return vm_disk_resources


@require_context
def _migration_plan_vm_disk_resource_get(context, vm_disk_resource_id, session):
    try:
        query = session.query(
            models.MigrationPlanVMDiskResource).options(
            sa_orm.joinedload(
                models.MigrationPlanVMDiskResource.metadata)).filter_by(
            id=vm_disk_resource_id)

        # TODO(gbasava): filter out deleted resource snapshots if context
        # disallows it
        vm_disk_resource = query.one()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMDiskResourceNotFound(
            migration_plan_vm_disk_resource_id=vm_disk_resource_id)

    return vm_disk_resource_snap


@require_context
def migration_plan_vm_disk_resource_get(context, vm_disk_resource_id):
    session = get_session()
    return _migration_plan_vm_disk_resource_get(
        context, vm_disk_resource_id, session)


@require_context
def migration_plan_vm_disk_resource_delete(context, vm_disk_resource_id):
    session = get_session()
    with session.begin():
        session.query(models.MigrationPlanVMDiskResource). \
            filter_by(id=vm_disk_resource_id). \
            delete()


def _migration_plan_set_metadata_for_vm_security_group_rule(
        context,
        vm_security_group_rule_ref,
        metadata,
        purge_metadata,
        session):
    orig_metadata = {}
    for metadata_ref in vm_security_group_rule_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'migration_plan_vm_security_group_rule_id': vm_security_group_rule_ref.id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_plan_vm_security_group_rule_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_plan_vm_security_group_rule_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_plan_vm_security_group_rule_metadata_delete(
                    context, metadata_ref, session)


@require_context
def _migration_plan_vm_security_group_rule_metadata_create(context, values, session):
    """Create an VMSecurityGroupRuleSnapMetadata object"""
    metadata_ref = models.MigrationPlanVMSecurityGroupRuleMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_plan_vm_security_group_rule_metadata_update(
        context, metadata_ref, values, session)


@require_context
def migration_plan_vm_security_group_rule_metadata_create(context, values):
    session = get_session()
    return _migration_plan_vm_security_group_rule_metadata_create(
        context, values, session)


def _migration_plan_vm_security_group_rule_metadata_update(
        context, metadata_ref, values, session):
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _migration_plan_vm_security_group_rule_metadata_delete(
        context, metadata_ref, session):
    metadata_ref.delete(session=session)


def _migration_plan_vm_security_group_rule_update(
        context,
        id,
        vm_security_group_id,
        values,
        purge_metadata,
        session):
    metadata = values.pop('metadata', {})

    if id and vm_security_group_id:
        vm_security_group_rule_ref = migration_plan_vm_security_group_rule_get(
            context, id, vm_security_group_id, session)
    else:
        vm_security_group_rule_ref = models.MigrationPlanVMSecurityGroupRule()

    vm_security_group_rule_ref.update(values)
    vm_security_group_rule_ref.save(session)

    _migration_plan_set_metadata_for_vm_security_group_rule(
        context, vm_security_group_rule_ref, metadata, purge_metadata, session)

    return vm_security_group_rule_ref


@require_context
def migration_plan_vm_security_group_rule_create(context, values):
    session = get_session()
    return _migration_plan_vm_security_group_rule_update(
        context, None, None, values, False, session)


@require_context
def migration_plan_vm_security_group_rule_update(
        context, id, vm_security_group_id, values, purge_metadata=False):
    session = get_session()
    return _migration_plan_vm_security_group_rule_update(
        context, id, vm_security_group_id, values, purge_metadata, session)


@require_context
def migration_plan_vm_security_group_rules_get(
        context, vm_security_group_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.MigrationPlanVMSecurityGroupRule).options(
            sa_orm.joinedload(
                models.MigrationPlanVMSecurityGroupRule.metadata)).filter_by(
            migration_plan_vm_security_group_id=vm_security_group_id)

        # TODO(gbasava): filter out deleted snapshots if context disallows it
        vm_security_group_rules = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMSecurityGroupRuleNotFound(
            migration_plan_vm_security_group_id=vm_security_group_id)

    return vm_security_group_rules


@require_context
def migration_plan_vm_security_group_rule_get(context, id, vm_security_group_id):
    session = get_session()
    try:
        query = session.query(
            models.MigrationPlanVMSecurityGroupRule).options(
            sa_orm.joinedload(
                models.MigrationPlanVMSecurityGroupRule.metadata)).filter_by(
            id=id).filter_by(
            migration_plan_vm_security_group_id=vm_security_group_id)

        vm_security_group_rule = query.one()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMSecurityGroupRuleNotFound(
            migration_plan_vm_security_group_rule_snap_id=id)

    return vm_security_group_rule


@require_context
def migration_plan_vm_security_group_rule_delete(
        context, id):
    session = get_session()
    with session.begin():
        session.query(models.MigrationPlanVMSecurityGroupRule). \
            filter_by(id=id). \
            delete()


@require_context
def _migration_plan_vm_get(context, vm_id, migration_plan_id, session):
    try:
        query = session.query(
            models.MigrationPlanVMs).options(
            sa_orm.joinedload(
                models.MigrationPlanVMs.metadata)).filter_by(
            vm_id=vm_id)

        if migration_plan_id is not None:
            query = query.filter_by(migration_plan_id=migration_plan_id)

        migration_plan_vm = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationPlanVMsNotFound(migration_plan_id=migration_plant_id)

    return migration_plan_vm


@require_context
def migration_plan_vm_get(context, vm_id, migration_plan_id):
    session = get_session()
    return _migration_plan_vm_get(context, vm_id, migration_plan_id, session)


@require_context
def _migration_metadata_create(context, values, session):
    metadata_ref = models.MigrationMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_metadata_update(context, metadata_ref, values, session)


@require_context
def migration_metadata_create(context, values, session):
    session = get_session()
    return _migration_metadata_create(context, values, session)


@require_context
def _migration_metadata_update(context, metadata_ref, values, session):
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _migration_metadata_delete(context, metadata_ref, session):
    metadata_ref.delete(session=session)


def _set_metadata_for_migration(context, migration_ref, metadata,
                               purge_metadata, session):
    orig_metadata = {}
    for metadata_ref in migration_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'migration_id': migration_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_metadata_delete(
                    context, metadata_ref, session=session)


def _migration_update(context, values, migration_id, purge_metadata, session):
    try:
        lock.acquire()
        metadata = values.pop('metadata', {})

        if not values.get('uploaded_size'):
            values['uploaded_size'] = 0
            if not values.get('progress_percent'):
                values['progress_percent'] = 0
        if migration_id:
            migration_ref = model_query(
                context,
                models.Migrations,
                session=session,
                read_deleted="yes").filter_by(
                id=migration_id).first()
            if not migration_ref:
                lock.release()
                raise exception.MigrationNotFound(migration_id=migration_id)
            if migration_ref.uploaded_size:
                values['uploaded_size'] = migration_ref.uploaded_size
            if migration_ref.progress_percent:
                values['progress_percent'] = migration_ref.progress_percent
        else:
            migration_ref = models.Migrations()
            if not values.get('id'):
                values['id'] = str(uuid.uuid4())
            if not values.get('size'):
                values['size'] = 0
            if not values.get('progress_percent'):
                values['progress_percent'] = 0
        migration_ref.update(values)
        migration_ref.save(session)

        if metadata:
            _set_metadata_for_migration(
                context,
                migration_ref,
                metadata,
                purge_metadata,
                session=session)

        return migration_ref
    finally:
        lock.release()
    return migration_ref

@require_context
def migration_create(context, values):
    session = get_session()
    return _migration_update(context, values, None, False, session)


@require_context
def migration_update(context, migration_id, values, purge_metadata=False):
    session = get_session()
    return _migration_update(context, values, migration_id,
                            purge_metadata, session)


@require_context
def _migration_get(context, migration_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    result = model_query(context, models.Migrations, **kwargs). \
        options(sa_orm.joinedload(models.Migrations.metadata)). \
        filter_by(id=migration_id). \
        first()

    if not result:
        raise exception.MigrationNotFound(migration_id=migration_id)

    return result


@require_context
def migration_get(context, migration_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    return _migration_get(context, migration_id, **kwargs)


@require_context
def migration_vms_get(context, migration_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.MigrationVMs).options(
            sa_orm.joinedload(
                models.MigrationVMs.metadata)).filter_by(
            migration_id=migration_id)
        migration_vms = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationVMsNotFound(migration_id=migration_id)

    return migration_vms


@require_context
def migration_get_all_by_project_migration_plan(
        context, project_id, migration_plan_id, **kwargs):
    if kwargs.get('session') is None:
        kwargs['session'] = get_session()
    authorize_project_context(context, project_id)
    return model_query(context, models.Migrations, **kwargs). \
        options(sa_orm.joinedload(models.Migrations.metadata)). \
        filter_by(project_id=project_id). \
        filter_by(migration_plan_id=migration_plan_id). \
        order_by(models.Migrations.created_at.desc()).all()


@require_context
def migration_get_metadata_cancel_flag(
        context, migration_id, return_val=0, process=None, **kwargs):
    flag = '0'
    migration_obj = migration_get(context, migration_id)
    for meta in migration_obj.metadata:
        if meta.key == 'cancel_requested':
            flag = meta.value

    if return_val == 1:
        return flag

    if flag == '1':
        if process:
            process.kill()
        error = _('Cancel requested for migration')
        raise exception.ErrorOccurred(reason=error)


@require_context
def migration_delete(context, migration_id):
    session = get_session()
    with session.begin():
            session.query(models.Migrations). \
                filter_by(id=migration_id).delete()


def _set_metadata_for_migration_vms(context, migration_vm_ref, metadata,
                                   purge_metadata, session):
    orig_metadata = {}
    for metadata_ref in migration_vm_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'migrated_vm_id': migration_vm_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_vms_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_vms_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                _migration_vms_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _migration_vms_metadata_create(context, values, session):
    """Create an MigrationMetadata object"""
    metadata_ref = models.MigrationVMMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_vms_metadata_update(
        context, metadata_ref, values, session)


@require_context
def migration_vms_metadata_create(context, values, session):
    """Create an MigrationMetadata object"""
    session = get_session()
    return _migration_vms_metadata_create(context, values, session)


@require_context
def _migration_vms_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by migration_vms_metadata_create and migration_vms_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def _migration_vms_metadata_delete(context, metadata_ref, session):
    """
    Used internally by migration_vms_metadata_create and migration_vms_metadata_update
    """
    metadata_ref.delete(session=session)


def _migration_vm_update(context, values, vm_id,
                        migration_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if vm_id:
        migration_vm_ref = _migration_vm_get(context, vm_id, migration_id, session)
    else:
        migration_vm_ref = models.MigrationVMs()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())
        if not values.get('size'):
            values['size'] = 0

    migration_vm_ref.update(values)
    migration_vm_ref.save(session)

    if metadata:
        _set_metadata_for_migration_vms(
            context,
            migration_vm_ref,
            metadata,
            purge_metadata,
            session=session)

    return migration_vm_ref


@require_context
def migration_vm_create(context, values):
    session = get_session()
    return _migration_vm_update(context, values, None, None, False, session)


@require_context
def migration_vm_update(context, vm_id, migration_id,
                       values, purge_metadata=False):
    session = get_session()
    return _migration_vm_update(
        context, values, vm_id, migration_id, purge_metadata, session)


@require_context
def migration_vms_get(context, migration_id, **kwargs):
    session = kwargs.get('session') or get_session()
    try:
        query = session.query(
            models.MigrationVMs).options(
            sa_orm.joinedload(
                models.MigrationVMs.metadata)).filter_by(
            migration_id=migration_id)
        migration_vms = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationVMsNotFound(migration_id=migration_id)

    return migration_vms


@require_context
def _migration_vm_get(context, vm_id, migration_id, session):
    try:
        query = session.query(
            models.MigrationVMs).options(
            sa_orm.joinedload(
                models.MigrationVMs.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            migration_id=migration_id)

        migration_vm = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationVMsNotFound(migration_id=migration_id)

    return migration_vm


@require_context
def migration_vm_get(context, vm_id, migration_id):
    session = get_session()
    return _migration_vm_get(context, vm_id, migration_id, session)


@require_context
def migration_vm_delete(context, vm_id, migration_id):
    session = get_session()
    with session.begin():
        session.query(models.MigrationVMs). \
            filter_by(vm_id=vm_id). \
            filter_by(migration_id=migration_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


def _set_metadata_for_migration_vm_resource(
        context,
        migration_vm_resource_ref,
        metadata,
        purge_metadata,
        session):
    orig_metadata = {}
    for metadata_ref in migration_vm_resource_ref.metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {
            'migration_vm_resource_id': migration_vm_resource_ref.id,
            'key': key,
            'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _migration_vm_resource_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _migration_vm_resource_metadata_create(
                context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                migration_vm_resource_metadata_delete(
                    context, metadata_ref, session=session)


@require_context
def _migration_vm_resource_metadata_create(context, values, session):
    metadata_ref = models.MigrationVMResourceMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _migration_vm_resource_metadata_update(
        context, metadata_ref, values, session)


@require_context
def migration_vm_resource_metadata_create(context, values):
    session = get_session()
    return _migration_vm_resource_metadata_create(context, values, session)


def _migration_vm_resource_metadata_update(
        context, metadata_ref, values, session):
    """
    Used internally by migration_vm_resource_metadata_create and migration_vm_resource_metadata_update
    """
    values["deleted"] = False
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


@require_context
def migration_vm_resource_metadata_delete(context, metadata_ref):
    """
    Used internally by migration_vm_resource_metadata_create and migration_vm_resource_metadata_update
    """
    session = get_session()
    metadata_ref.delete(session=session)
    return metadata_ref


def _migration_vm_resource_update(
        context, values, migration_vm_resource_id, purge_metadata, session):
    metadata = values.pop('metadata', {})

    if migration_vm_resource_id:
        migration_vm_resource_ref = migration_vm_resource_get(
            context, migration_vm_resource_id, session)
    else:
        migration_vm_resource_ref = models.MigrationVMResources()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    migration_vm_resource_ref.update(values)
    migration_vm_resource_ref.save(session)

    _set_metadata_for_migration_vm_resource(
        context,
        migration_vm_resource_ref,
        metadata,
        purge_metadata,
        session)

    return migration_vm_resource_ref


@require_context
def migration_vm_resource_create(context, values):
    session = get_session()
    return _migration_vm_resource_update(context, values, None, False, session)


@require_context
def migration_vm_resource_update(
        context, migration_vm_resource_id, values, purge_metadata=False):
    session = get_session()
    return _migration_vm_resource_update(
        context, values, migration_vm_resource_id, purge_metadata, session)


@require_context
def migration_vm_resources_get(context, vm_id, migration_id):
    session = get_session()
    try:
        query = session.query(
            models.MigrationVMResources).options(
            sa_orm.joinedload(
                models.MigrationVMResources.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            migration_id=migration_id)

        migrationd_vm_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationVMResourcesNotFound(
            migration_vm_id=vm_id, migration_id=migration_id)

    return migration_vm_resources


@require_context
def migration_vm_resource_get_by_resource_name(
        context, vm_id, migration_id, resource_name):
    session = get_session()
    try:
        query = session.query(
            models.MigrationVMResources).options(
            sa_orm.joinedload(
                models.MigrationVMResources.metadata)).filter_by(
            vm_id=vm_id).filter_by(
            migration_id=migration_id).filter_by(
            resource_name=resource_name)

        migration_vm_resources = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationVMResourceWithNameNotFound(
            resource_name=resource_name, migration_vm_id=vm_id, migration_id=migration_id)

    return migration_vm_resources


@require_context
def migration_vm_resource_get(context, id):
    session = get_session()
    try:
        query = session.query(
            models.MigrationVMResources).options(
            sa_orm.joinedload(
                models.MigrationVMResources.metadata)).filter_by(
            id=id)

        migration_vm_resources = query.first()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationVMResourceWithIdNotFound(id=id)

    return migration_vm_resources


@require_context
def migration_vm_resource_delete(context, id, vm_id, migration_id):
    session = get_session()
    with session.begin():
        session.query(models.MigrationVMResources). \
            filter_by(id=id). \
            filter_by(vm_id=vm_id). \
            filter_by(migration_id=migration_id). \
            update({'status': 'deleted',
                    'deleted': True,
                    'deleted_at': timeutils.utcnow(),
                    'updated_at': literal_column('updated_at')})


@require_context
def migration_vm_resources_get_all(context, migration_id, **kwargs):
    session = get_session()
    try:
        query = session.query(
            models.MigrationVMResources).options(
            sa_orm.joinedload(
                models.MigrationVMResources.metadata)).filter_by(
            migration_id=migration_id)

        if kwargs.get('resource_type'):
            query = query.filter_by(resource_type=kwargs['resource_type'])

        migrated_vm_resources = query.all()

    except sa_orm.exc.NoResultFound:
        raise exception.MigrationVMResourcesNotFound(migration_id=migration_id)

    return migrated_vm_resources


@require_context
def migration_type_time_size_update(context, migration_id):
    migration = migration_get(context, migration_id)
    migration_vm_resources = migration_vm_resources_get_all(context, migration_id, resource_type='disk')
    migration_size = 0
    for migration_vm_resource in migration_vm_resources:
        if migration_vm_resource.status != 'deleted':
            migration_size += migration_vm_resource.size
    return migration_update(
        context,
        migration_id,
        {
            'time_taken': int((timeutils.utcnow() - migration.created_at).total_seconds()),
            'uploaded_size': migration_size,
        })


@require_admin_context
def unlock_migration_plans_for_host(context, host):
    session = get_session()
    migration_plans_query = model_query(context, models.MigrationPlans, session=session)
    migration_plans_query = migration_plans_query.filter(and_(
        models.MigrationPlans.status != 'available',
        models.MigrationPlans.status != 'error',
        models.MigrationPlans.host == host)
    )
    locked_migration_plans = migration_plans_query.all()
    for mig_plan in locked_migration_plans:
        values = {'status': 'available'}
        mig_plan.update(values)
        session.add(mig_plan)
    session.flush()
    session.close()


@require_admin_context
def migration_plans_mark_deleting_as_error(context, host):
    session = get_session()
    migration_plans_query = model_query(context, models.MigrationPlans, session=session)
    migration_plans_query = migration_plans_query.filter(and_(
        models.MigrationPlans.status == 'deleting',
        models.MigrationPlans.host == host)
    )
    deleting_migration_plans = migration_plans_query.all()
    for mig_plan in deleting_migration_plans:
        values = {'status': 'error'}
        mig_plan.update(values)
        session.add(wrklod)
    session.flush()
    session.close()


@require_admin_context
def migration_mark_incomplete_as_error(context, host):
    """
    mark the migrations that are left hanging from previous run on host as 'error'
    """
    session = get_session()
    now = timeutils.utcnow()
    migrations_query = model_query(context, models.Migrations, session=session)
    migrations_query = migrations_query.filter(and_(
        models.Migrations.status != 'available',
        models.Migrations.status != 'error',
        models.Migrations.status != 'mounted',
        models.Migrations.status != 'cancelled'))

    migrations_query = migrations_query.filter(or_(
        models.Migrations.host == host,
        models.Migrations.host == ''))
    migrations = migrations_query.all()

    for migration in migrations:
        if (migration.host == '' and
                now - migration.created_at <= timedelta(minutes=60)):
            continue

        if migration.status == 'migrating':
            values = {'status': 'available'}
        else:
            values = {'progress_percent': 100, 'progress_msg': '',
                      'error_msg': 'Migration did not finish successfully',
                      'status': 'error'}
        migration.update(values)
        session.add(migration)
    session.flush()
    session.close()


""" Backup target APIs """
def _backup_target_get(context, backup_target_id, session):
    try:
        backup_target_ref = model_query(
            context, models.BackupTargets, session=session). \
            filter_by(id=backup_target_id).first()

        if not backup_target_ref:
            raise exception.BackupTargetNotFound(backup_target_id=backup_target_id)
        return backup_target_ref
    except sa_orm.exc.NoResultFound:
        raise exception.BackupTargetNotFound(backup_target_id=backup_target_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.BackupTargetNotFound(backup_target_id=backup_target_id)

def _backup_target_update(context, backup_target_id, values, session):
    if backup_target_id:
        bt_ref = _backup_target_get(context, backup_target_id, session)
    else:
        bt_ref = models.BackupTargets()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())

    bt_ref.update(values)
    bt_ref.save(session)

    return bt_ref


@require_context
def backup_target_create(context, values):
    btt_name = values.pop('backup_target_type_name', None)
    session = get_session()
    bt_ref =_backup_target_update(context, None, values, session)
    values.update({'backup_targets_id': bt_ref.id, 'id': None})
    backup_target_metadata_create(context, values, session)
    backup_target_type_create(context, bt_ref.id, values={'name': btt_name or bt_ref.filesystem_export, 'user_id': context.user_id, 'is_default': bt_ref.is_default}, project_list=[])
    return bt_ref


@require_context
def backup_target_update(context, backup_target_id, values):
    session = get_session()
    return _backup_target_update(context, backup_target_id, values, session)


@require_context
def backup_target_delete(context, backup_target_id):
    session = get_session()
    session.query(models.BackupTargets). \
                    filter_by(id=backup_target_id).delete()


@require_context
def backup_target_show(context, backup_target_id):
    session = get_session()
    try:
        bt_ref = model_query(
            context, models.BackupTargets, session=session). \
            options(sa_orm.joinedload(models.BackupTargets.backup_target_types)). \
            filter_by(id=backup_target_id).first()
        if not bt_ref:
            raise exception.BackupTargetNotFound(backup_target_id=backup_target_id)
        return bt_ref
    except sa_orm.exc.NoResultFound:
        raise exception.BackupTargetNotFound(backup_target_id=backup_target_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.BackupTargetNotFound(backup_target_id=backup_target_id)


@require_context
def get_backup_target_by_backend_endpoint(context, backend_endpoint):
    session = get_session()
    try:
        bt_ref = model_query(
            context, models.BackupTargets, session=session). \
            options(sa_orm.joinedload(models.BackupTargets.backup_target_types)). \
            filter_by(filesystem_export=backend_endpoint).first()
        if not bt_ref:
            raise exception.BackupTargetNotFound(backup_target_id=backend_endpoint)
        return bt_ref
    except sa_orm.exc.NoResultFound:
        raise exception.BackupTargetNotFound(backup_target_id=backend_endpoint)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.BackupTargetNotFound(backup_target_id=backend_endpoint)


@require_context
def backup_target_get_all(context):
    session = get_session()
    #return session.query(models.BackupTargets).all()
    return model_query(context, models.BackupTargets). \
            options(sa_orm.joinedload(models.BackupTargets.backup_target_types)). \
            order_by(models.BackupTargets.created_at.desc()).all()


@require_context
def get_backup_target_by_btt(context, backup_target_type):
    session = get_session()
    try:
        uuid.UUID(str(backup_target_type))
        bt_ref = session.query(models.BackupTargets).join(models.BackupTargetTypes).filter(models.BackupTargetTypes.id == backup_target_type).first()
    except ValueError:
        bt_ref = session.query(models.BackupTargets).join(models.BackupTargetTypes).filter(models.BackupTargetTypes.name == backup_target_type).first()
    return bt_ref

@require_context
def _backup_target_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by backup_target_metadata_create and backup_target_metadata_update
    """
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref

@require_context
def _backup_target_metadata_create(context, values, session):
    """Create an BackupTargetMetadata object"""
    metadata_ref = models.BackupTargetsMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    values['key']='immutable'
    values['value']=values.get('immutable', 0)
    return _backup_target_type_metadata_update(context, metadata_ref, values, session)

@require_context
def backup_target_metadata_create(context, values, session):
    """Create an BackupTargetMetadata object"""
    session = get_session()
    return _backup_target_metadata_create(context, values, session)


""" BTT metadata """
@require_context
def _backup_target_type_metadata_create(context, values, session):
    """Create an BackupTargetTypeMetadata object"""
    metadata_ref = models.BackupTargetTypeMetadata()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _backup_target_type_metadata_update(context, metadata_ref, values, session)


@require_context
def backup_target_type_metadata_create(context, values, session):
    """Create an BackupTargetTypeMetadata object"""
    session = get_session()
    return _backup_target_type_metadata_create(context, values, session)


@require_context
def _backup_target_type_metadata_update(context, metadata_ref, values, session):
    """
    Used internally by backup_target_type_metadata_create and backup_target_type_metadata_update
    """
    metadata_ref.update(values)
    metadata_ref.save(session=session)
    return metadata_ref


def _set_metadata_for_backup_target_type(context, btt_ref, metadata, purge_metadata, session):
    orig_metadata = {}
    for metadata_ref in btt_ref.backup_target_type_metadata:
        orig_metadata[metadata_ref.key] = metadata_ref

    for key, value in metadata.items():
        metadata_values = {'backup_target_type_id': btt_ref.id,
                           'key': key,
                           'value': value}
        if key in orig_metadata:
            metadata_ref = orig_metadata[key]
            _backup_target_type_metadata_update(
                context, metadata_ref, metadata_values, session)
        else:
            _backup_target_type_metadata_create(context, metadata_values, session)

    if purge_metadata:
        for key in list(orig_metadata.keys()):
            if key not in metadata:
                metadata_ref = orig_metadata[key]
                session.query(models.BackupTargetTypeMetadata). \
                    filter_by(id=metadata_ref.id).delete()


""" BTT projects """
@require_context
def _backup_target_type_project_create(context, values, session):
    """Create an BackupTargetTypeProjects object"""
    project_ref = models.BackupTargetTypeProjects()
    if not values.get('id'):
        values['id'] = str(uuid.uuid4())
    return _backup_target_type_project_update(context, project_ref, values, session)


@require_context
def backup_target_type_project_create(context, values, session):
    """Create an BackupTargetTypeProjects object"""
    session = get_session()
    return _backup_target_type_project_create(context, values, session)


@require_context
def _backup_target_type_project_update(context, project_ref, values, session):
    """
    Used internally by backup_target_type_project_create and backup_target_type_project_update
    """
    project_ref.update(values)
    project_ref.save(session=session)
    return project_ref


def _set_projects_for_backup_target_type(context, btt_ref, project_list, purge_projects, session):
    orig_projects = {}
    for btt_project_ref in btt_ref.backup_target_type_projects:
        orig_projects[btt_project_ref.project_id] = btt_project_ref

    for project_id in project_list:
        project_values = {'backup_target_types_id': btt_ref.id,
                           'project_id': project_id,
                          }
        if project_id not in orig_projects:
            _backup_target_type_project_create(context, project_values, session)

    if purge_projects:
        for key in list(orig_projects.keys()):
            if key not in project_list:
                project_ref = orig_projects[key]
                session.query(models.BackupTargetTypeProjects). \
                    filter_by(id=project_ref.id).delete()

@require_context
def backup_target_type_remove_projects(context, backup_target_type_id):
    session = get_session()
    session.query(models.BackupTargetTypeProjects). \
                filter_by(backup_target_types_id=backup_target_type_id).delete()


""" BTT APIs """
def _backup_target_type_get(context, backup_target_type_id, session):
    try:
        backup_target_type_ref = model_query(
            context, models.BackupTargetTypes, session=session). \
            filter_by(id=backup_target_type_id).first()

        if not backup_target_type_ref:
            raise exception.BackupTargetTypeNotFound(backup_target_type_id=backup_target_type_id)
        return backup_target_type_ref
    except sa_orm.exc.NoResultFound:
        raise exception.BackupTargetTypeNotFound(backup_target_type_id=backup_target_type_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.BackupTargetTypeNotFound(backup_target_type_id=backup_target_type_id)

def backup_target_type_get_by_name(context, backup_target_type_name, session):
    try:
        backup_target_type_ref = model_query(
            context, models.BackupTargetTypes, session=session). \
            filter_by(name=backup_target_type_name).first()

        if not backup_target_type_ref:
            raise exception.BackupTargetTypeByNameNotFound(backup_target_type_name=backup_target_type_name)
        return backup_target_type_ref
    except sa_orm.exc.NoResultFound:
        raise exception.BackupTargetTypeByNameNotFound(backup_target_type_name=backup_target_type_name)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.BackupTargetTypeByNameNotFound(backup_target_type_name=backup_target_type_name)


def _backup_target_type_update(context, backup_target_id, backup_target_type_id, values, project_list, purge_metadata, purge_projects, session):
    metadata = values.pop('metadata', {})
    new_btt_flag = False
    btt_remove_projects = False
    if backup_target_type_id:
        btt_ref = _backup_target_type_get(context, backup_target_type_id, session)

        # user not allowed to change BT of BTT
        if values.get('backup_targets_id') and values['backup_targets_id'] != btt_ref.backup_targets_id:
            raise exception.BackupTargetTypeUpdateNotAllowed(backup_target_type_id=backup_target_type_id)

        # user not allowed to change default BTT
        if btt_ref.is_default:
            raise exception.BackupTargetTypeDefaultUpdateNotAllowed(backup_target_type_id=backup_target_type_id)

        # when user wishes to change private BTT to public then remove all it's assigned projects and make it public
        if values.get('is_public') and btt_ref.is_public != values['is_public']:
            btt_remove_projects = True

        # when user passess project list then make BTT as private
        if values.get('project_list'):
            values['is_public'] = 0
    else:
        new_btt_flag = True
        btt_ref = models.BackupTargetTypes()
        if not values.get('id'):
            values['id'] = str(uuid.uuid4())
        values['backup_targets_id'] = backup_target_id

    btt_ref.update(values)
    btt_ref.save(session)

    if btt_remove_projects:
        backup_target_type_remove_projects(context, btt_ref.id)

    try:
        if metadata:
            _set_metadata_for_backup_target_type(
                context,
                btt_ref,
                metadata,
                purge_metadata,
                session=session)

        if project_list:
            _set_projects_for_backup_target_type(
                context,
                btt_ref,
                project_list,
                purge_projects,
                session=session)
    except Exception as ex:
        if new_btt_flag:
            backup_target_type_delete(context, btt_ref.id)
        raise ex
    return btt_ref


@require_context
def backup_target_type_create(context, backup_target_id, values, project_list=[], purge_metadata=False, purge_projects=False):
    session = get_session()
    backup_target_type_ref = model_query(
        context, models.BackupTargetTypes, session=session). \
        filter_by(name=values.get('name')).first()
    if backup_target_type_ref:
        raise exception.BackupTargetTypeNameDuplicateNotAllowed(backup_target_type_name=values.get('name'))
    if not values.get('user_id'):
        values['user_id'] = context.user_id
    return _backup_target_type_update(context, backup_target_id, None, values, project_list, purge_metadata, purge_projects, session)


@require_context
def backup_target_type_update(context, backup_target_type_id, values, project_list=[], purge_metadata=False, purge_projects=False):
    session = get_session()
    return _backup_target_type_update(context, None, backup_target_type_id, values, project_list, purge_metadata, purge_projects, session)


@require_context
def backup_target_type_add_projects(context, backup_target_type_id, project_list=[]):
    session = get_session()
    btt_ref = _backup_target_type_get(context, backup_target_type_id, session)
    _set_projects_for_backup_target_type(
            context,
            btt_ref,
            project_list,
            purge_projects=False,
            session=session)
    return _backup_target_type_get(context, backup_target_type_id, session)


@require_context
def backup_target_type_remove_projects_api(context, backup_target_type_id, project_list=[]):
    session = get_session()
    for project_id in project_list:
        session.query(models.BackupTargetTypeProjects). \
                filter_by(backup_target_types_id=backup_target_type_id, project_id=project_id).delete()


@require_context
def backup_target_type_add_metadata(context, backup_target_type_id, metadata={}):
    session = get_session()
    btt_ref = _backup_target_type_get(context, backup_target_type_id, session)
    _set_metadata_for_backup_target_type(
                context,
                btt_ref,
                metadata,
                purge_metadata=False,
                session=session)
    return _backup_target_type_get(context, backup_target_type_id, session)


@require_context
def backup_target_type_remove_metadata(context, backup_target_type_id, metadata=[]):
    session = get_session()
    for metadata_key in metadata:
        session.query(models.BackupTargetTypeMetadata). \
                filter_by(backup_target_type_id=backup_target_type_id, key=metadata_key).delete()


@require_context
def backup_target_type_show(context, backup_target_type_id):
    session = get_session()
    try:
        try:
            uuid.UUID(backup_target_type_id)
            btt_ref = model_query(
                context, models.BackupTargetTypes, session=session). \
                options(sa_orm.joinedload(models.BackupTargetTypes.backup_target_type_metadata)). \
                options(sa_orm.joinedload(models.BackupTargetTypes.backup_target_type_projects)). \
                filter_by(id=backup_target_type_id).first()
        except ValueError:
            # if user provides name then get BTT data by name
            btt_ref = model_query(
                context, models.BackupTargetTypes, session=session). \
                options(sa_orm.joinedload(models.BackupTargetTypes.backup_target_type_metadata)). \
                options(sa_orm.joinedload(models.BackupTargetTypes.backup_target_type_projects)). \
                filter_by(name=backup_target_type_id).first()
        if not btt_ref:
            raise exception.BackupTargetTypeNotFound(backup_target_type_id=backup_target_type_id)
        return btt_ref
    except sa_orm.exc.NoResultFound:
        raise exception.BackupTargetTypeNotFound(backup_target_type_id=backup_target_type_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.BackupTargetTypeNotFound(backup_target_type_id=backup_target_type_id)


@require_context
def backup_target_type_get_all(context, **kwargs):
    session = get_session()
    return session.query(models.BackupTargetTypes).all()


@require_context
def backup_target_type_get_all_public_and_by_project_id(context, project_id=None, **kwargs):
    session = get_session()
    btt_public = session.query(models.BackupTargetTypes).filter_by(is_public=1).order_by(models.BackupTargetTypes.is_default.desc()).all()
    btt_private = session.query(models.BackupTargetTypes).join(models.BackupTargetTypeProjects).filter(and_(models.BackupTargetTypes.is_public == 0, models.BackupTargetTypeProjects.project_id == project_id)).all()
    btt_ref = list(set(btt_public + btt_private))
    return btt_ref


@require_context
def backup_target_type_get_all_by_backup_target(context, backup_target_id, **kwargs):
    session = get_session()
    return session.query(models.BackupTargetTypes).filter_by(backup_targets_id=backup_target_id).all()


@require_context
def backup_target_type_delete(context, backup_target_type_id):
    session = get_session()
    btt_ref = _backup_target_type_get(context, backup_target_type_id, session)
    if btt_ref.is_default:
        raise exception.BackupTargetTypeDefaultDeleteNotAllowed(backup_target_type_id=backup_target_type_id)
    session.query(models.BackupTargetTypes). \
                filter_by(id=backup_target_type_id).delete()


def set_all_backup_target_type_to_non_default(context, backup_target_id):
    session = get_session()
    session.query(models.BackupTargetTypes).filter_by(backup_targets_id=backup_target_id).update({'is_default': 0})


def get_default_backup_target_type(context):
    session = get_session()
    return session.query(models.BackupTargetTypes).filter_by(is_default=1).all()


""" Get importable workloads list APIs """
def _importable_workload_get(context, job_data_id, session):
    try:
        job_data_ref = model_query(
            context, models.JobDetails, session=session). \
            filter_by(id=job_data_id).first()
        return job_data_ref
    except sa_orm.exc.NoResultFound:
        raise exception.JobDataNotFound(job_data_id=job_data_id)
    except Exception as ex:
        LOG.exception(ex)
        raise exception.JobdataNotFound(job_data_id=job_data_id)


@require_context
def importable_workloads_create(context, jobid, values):
    session = get_session()
    job_data_ref = models.JobDetails()
    job_data_ref.update({'jobid': jobid, 'data': json.dumps(values)})
    job_data_ref.save(session)
    return job_data_ref


@require_context
def importable_workloads_update(context, job_data_id, values):
    session = get_session()
    job_data_ref =  model_query(
                       context, models.JobDetails, session=session).filter_by(id=job_data_id).first()
    if not job_data_ref:
        raise exception.JobDataNotFound(job_data_id=job_data_id)
    db_values = dict(json.loads(job_data_ref.data))
    db_values.update(values)
    job_data_ref.update({'jobid': job_data_ref.jobid, 'data':json.dumps(db_values)})
    job_data_ref.save(session)
    return job_data_ref


@require_context
def importable_workload_get(context, job_data_id):
    session = get_session()
    job_data_ref =  model_query(
                       context, models.JobDetails, session=session).filter_by(id=job_data_id).first()
    if not job_data_ref:
        raise exception.JobDataNotFound(job_data_id=job_data_id)
    return job_data_ref


@require_context
def importable_workloads_get_all(context, **kwargs):
    session = get_session()
    if kwargs.get('jobid'):
        response = session.query(models.JobDetails).filter_by(jobid=kwargs['jobid']).order_by(asc(models.JobDetails.created_at)).all()
    else:
        response = session.query(models.JobDetails).order_by(asc(models.JobDetails.created_at)).all()

    if kwargs.get('workload_id'):
        response = [resp for resp in response if json.loads(resp.data)['workload_id'] == kwargs['workload_id']]

    if kwargs.get('status'):
        response = [resp for resp in response if json.loads(resp.data)['status'] == kwargs['status']]

    if kwargs.get('backup_target'):
        response = [resp for resp in response if json.loads(resp.data)['backup_target'] == kwargs['backup_target']]
    return response


@require_context
def workload_delete_all(context, **kwargs):
    deleted_workload_list = []
    qs_wl = model_query(context, models.Workloads)
    if kwargs.get('workload_ids', None):
        qs_wl = qs_wl.filter(
                    models.Workloads.id.in_(kwargs['workload_ids']))
    if kwargs.get('project_list', None):
        qs_wl = qs_wl.filter(
                    models.Workloads.project_id.in_(kwargs['project_list']))
    if kwargs.get('user_list', None):
        qs_wl = qs_wl.filter(
                    models.Workloads.user_id.in_(kwargs['user_list']))
    if kwargs.get('workload_ids', None) or kwargs.get('all_workloads', None):
        deleted_workload_list = qs_wl.all()
        qs_wl.delete(synchronize_session=False)

    return deleted_workload_list

@require_context
def policy_delete_all(context, **kwargs):
    deleted_policy_list = []
    qs_policy = model_query(context, models.WorkloadPolicy)
    if kwargs.get('policy_ids', None):
        qs_policy = qs_policy.filter(
                    models.WorkloadPolicy.id.in_(kwargs['policy_ids']))
    if kwargs.get('project_list'):
        qs_policy = qs_policy.filter(
                    models.WorkloadPolicy.project_id.in_(kwargs['project_list']))
    if kwargs.get('user_list', None):
        qs_policy = qs_policy.filter(
                    models.WorkloadPolicy.user_id.in_(kwargs['user_list']))
    if kwargs.get('policy_ids', None) or kwargs.get('all_policies', None):
        deleted_policy_list = qs_policy.all()
        for policy in deleted_policy_list:
            # removing the policy assignments before deletion
            model_query(context, models.WorkloadPolicyAssignmnets).filter_by(policy_id=policy.id).delete()

        qs_policy.delete(synchronize_session=False)
    return deleted_policy_list

@require_context
def abandon_resources(context, **kwargs):
    deleted_workload_list = workload_delete_all(context, **kwargs)
    deleted_policy_list = policy_delete_all(context, **kwargs)
    return {'abandon_workload_list': deleted_workload_list, 'abandon_policy_list': deleted_policy_list}