#!/usr/bin/python

# (c) 2020, NetApp, Inc
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import absolute_import, division, print_function
__metaclass__ = type


DOCUMENTATION = """
---
module: na_santricity_firmware
short_description: NetApp E-Series manage firmware.
description:
    - Ensure specific firmware versions are activated on E-Series storage system.
author:
    - Nathan Swartz (@ndswartz)
extends_documentation_fragment:
    - netapp_eseries.santricity.santricity.santricity_doc
options:
    nvsram:
        description:
            - Path to the NVSRAM file.
            - NetApp recommends upgrading the NVSRAM when upgrading firmware.
            - Due to concurrency issues, use M(na_santricity_proxy_firmware_upload) to upload firmware and nvsram to SANtricity Web Services Proxy when
              upgrading multiple systems at the same time on the same instance of the proxy.
        type: str
        required: false
    firmware:
        description:
            - Path to the firmware file.
            - Due to concurrency issues, use M(na_santricity_proxy_firmware_upload) to upload firmware and nvsram to SANtricity Web Services Proxy when
              upgrading multiple systems at the same time on the same instance of the proxy.
        type: str
        required: True
    wait_for_completion:
        description:
            - This flag will cause module to wait for any upgrade actions to complete.
            - When changes are required to both firmware and nvsram and task is executed against SANtricity Web Services Proxy,
              the firmware will have to complete before nvsram can be installed.
        type: bool
        default: false
    clear_mel_events:
        description:
            - This flag will force firmware to be activated in spite of the storage system mel-event issues.
            - Warning! This will clear all storage system mel-events. Use at your own risk!
        type: bool
        default: false
"""
EXAMPLES = """
- name: Ensure correct firmware versions
  na_santricity_firmware:
    ssid: "1"
    api_url: "https://192.168.1.100:8443/devmgr/v2"
    api_username: "admin"
    api_password: "adminpass"
    validate_certs: true
    nvsram: "path/to/nvsram"
    firmware: "path/to/bundle"
    wait_for_completion: true
    clear_mel_events: true
- name: Ensure correct firmware versions
  na_santricity_firmware:
    ssid: "1"
    api_url: "https://192.168.1.100:8443/devmgr/v2"
    api_username: "admin"
    api_password: "adminpass"
    validate_certs: true
    nvsram: "path/to/nvsram"
    firmware: "path/to/firmware"
"""
RETURN = """
msg:
    description: Status and version of firmware and NVSRAM.
    type: str
    returned: always
    sample:
"""
import os
import multiprocessing
import threading

from time import sleep
from ansible.module_utils import six
from ansible_collections.netapp_eseries.santricity.plugins.module_utils.santricity import NetAppESeriesModule, create_multipart_formdata, request
from ansible.module_utils._text import to_native


class NetAppESeriesFirmware(NetAppESeriesModule):
    COMPATIBILITY_CHECK_TIMEOUT_SEC = 60
    REBOOT_TIMEOUT_SEC = 30 * 60
    MINIMUM_PROXY_VERSION = "04.10.00.0000"

    def __init__(self):
        ansible_options = dict(
            nvsram=dict(type="str", required=False),
            firmware=dict(type="str", required=True),
            wait_for_completion=dict(type="bool", default=False),
            clear_mel_events=dict(type="bool", default=False))

        super(NetAppESeriesFirmware, self).__init__(ansible_options=ansible_options,
                                                    web_services_version="02.00.0000.0000",
                                                    supports_check_mode=True)

        args = self.module.params
        self.nvsram = args["nvsram"]
        self.firmware = args["firmware"]
        self.wait_for_completion = args["wait_for_completion"]
        self.clear_mel_events = args["clear_mel_events"]

        self.nvsram_name = None
        self.firmware_name = None
        self.is_bundle_cache = None
        self.firmware_version_cache = None
        self.nvsram_version_cache = None
        self.upgrade_required = False
        self.upgrade_in_progress = False
        self.module_info = dict()

        if self.nvsram:
            self.nvsram_name = os.path.basename(self.nvsram)
        if self.firmware:
            self.firmware_name = os.path.basename(self.firmware)

        self.last_known_event = -1
        self.is_firmware_activation_started_mel_event_count = 1
        self.is_nvsram_download_completed_mel_event_count = 1
        self.proxy_wait_for_upgrade_mel_event_count = 1

    def is_upgrade_in_progress(self):
        """Determine whether an upgrade is already in progress."""
        in_progress = False

        if self.is_proxy():
            try:
                rc, status = self.request("storage-systems/%s/cfw-upgrade" % self.ssid)
                in_progress = status["running"]
            except Exception as error:
                if "errorMessage" in to_native(error):
                    self.module.warn("Failed to retrieve upgrade status. Array [%s]. Error [%s]." % (self.ssid, error))
                    in_progress = False
                else:
                    self.module.fail_json(msg="Failed to retrieve upgrade status. Array [%s]. Error [%s]." % (self.ssid, error))
        else:
            in_progress = False

        return in_progress

    def is_firmware_bundled(self):
        """Determine whether supplied firmware is bundle."""
        if self.is_bundle_cache is None:
            with open(self.firmware, "rb") as fh:
                signature = fh.read(16).lower()

                if b"firmware" in signature:
                    self.is_bundle_cache = False
                elif b"combined_content" in signature:
                    self.is_bundle_cache = True
                else:
                    self.module.fail_json(msg="Firmware file is invalid. File [%s]. Array [%s]" % (self.firmware, self.ssid))

        return self.is_bundle_cache

    def firmware_version(self):
        """Retrieve firmware version of the firmware file. Return: bytes string"""
        if self.firmware_version_cache is None:

            # Search firmware file for bundle or firmware version
            with open(self.firmware, "rb") as fh:
                line = fh.readline()
                while line:
                    if self.is_firmware_bundled():
                        if b'displayableAttributeList=' in line:
                            for item in line[25:].split(b','):
                                key, value = item.split(b"|")
                                if key == b'VERSION':
                                    self.firmware_version_cache = value.strip(b"\n")
                            break
                    elif b"Version:" in line:
                        self.firmware_version_cache = line.split()[-1].strip(b"\n")
                        break
                    line = fh.readline()
                else:
                    self.module.fail_json(msg="Failed to determine firmware version. File [%s]. Array [%s]." % (self.firmware, self.ssid))
        return self.firmware_version_cache

    def nvsram_version(self):
        """Retrieve NVSRAM version of the NVSRAM file. Return: byte string"""
        if self.nvsram_version_cache is None:

            with open(self.nvsram, "rb") as fh:
                line = fh.readline()
                while line:
                    if b".NVSRAM Configuration Number" in line:
                        self.nvsram_version_cache = line.split(b'"')[-2]
                        break
                    line = fh.readline()
                else:
                    self.module.fail_json(msg="Failed to determine NVSRAM file version. File [%s]. Array [%s]." % (self.nvsram, self.ssid))
        return self.nvsram_version_cache

    def check_system_health(self):
        """Ensure E-Series storage system is healthy. Works for both embedded and proxy web services."""
        try:
            rc, response = self.request("storage-systems/%s/health-check" % self.ssid, method="POST")
            return response["successful"]
        except Exception as error:
            self.module.fail_json(msg="Health check failed! Array Id [%s]. Error[%s]." % (self.ssid, to_native(error)))

    def embedded_check_compatibility(self):
        """Verify files are compatible with E-Series storage system."""
        if self.nvsram:
            self.embedded_check_nvsram_compatibility()
        if self.firmware:
            self.embedded_check_bundle_compatibility()

    def embedded_check_nvsram_compatibility(self):
        """Verify the provided NVSRAM is compatible with E-Series storage system."""
        files = [("nvsramimage", self.nvsram_name, self.nvsram)]
        headers, data = create_multipart_formdata(files=files)
        compatible = {}
        try:
            rc, compatible = self.request("firmware/embedded-firmware/%s/nvsram-compatibility-check" % self.ssid, method="POST", data=data, headers=headers)
        except Exception as error:
            self.module.fail_json(msg="Failed to retrieve NVSRAM compatibility results. Array Id [%s]. Error[%s]." % (self.ssid, to_native(error)))

        if not compatible["signatureTestingPassed"]:
            self.module.fail_json(msg="Invalid NVSRAM file. File [%s]." % self.nvsram)
        if not compatible["fileCompatible"]:
            self.module.fail_json(msg="Incompatible NVSRAM file. File [%s]." % self.nvsram)

        # Determine whether nvsram upgrade is required
        for module in compatible["versionContents"]:
            if module["bundledVersion"] != module["onboardVersion"]:
                self.upgrade_required = True

            # Update bundle info
            self.module_info.update({module["module"]: {"onboard_version": module["onboardVersion"], "bundled_version": module["bundledVersion"]}})

    def embedded_check_bundle_compatibility(self):
        """Verify the provided firmware bundle is compatible with E-Series storage system."""
        files = [("files[]", "blob", self.firmware)]
        headers, data = create_multipart_formdata(files=files, send_8kb=True)
        compatible = {}
        try:
            rc, compatible = self.request("firmware/embedded-firmware/%s/bundle-compatibility-check" % self.ssid, method="POST", data=data, headers=headers)
        except Exception as error:
            self.module.fail_json(msg="Failed to retrieve bundle compatibility results. Array Id [%s]. Error[%s]." % (self.ssid, to_native(error)))

        # Determine whether valid and compatible firmware
        if not compatible["signatureTestingPassed"]:
            self.module.fail_json(msg="Invalid firmware bundle file. File [%s]." % self.firmware)
        if not compatible["fileCompatible"]:
            self.module.fail_json(msg="Incompatible firmware bundle file. File [%s]." % self.firmware)

        # Determine whether bundle upgrade is required
        for module in compatible["versionContents"]:
            bundle_module_version = module["bundledVersion"].split(".")
            onboard_module_version = module["onboardVersion"].split(".")
            version_minimum_length = min(len(bundle_module_version), len(onboard_module_version))

            if bundle_module_version[:version_minimum_length] != onboard_module_version[:version_minimum_length]:
                self.upgrade_required = True

            # Build the modules information for logging purposes
            self.module_info.update({module["module"]: {"onboard_version": module["onboardVersion"], "bundled_version": module["bundledVersion"]}})

    def embedded_firmware_activate(self):
        """Activate firmware."""
        rc, response = self.request("firmware/embedded-firmware/activate", method="POST", ignore_errors=True, timeout=10)
        if rc == "422":
            self.module.fail_json(msg="Failed to activate the staged firmware. Array Id [%s]. Error [%s]" % (self.ssid, response))

    def embedded_firmware_download(self):
        """Execute the firmware download."""
        if self.nvsram:
            firmware_url = "firmware/embedded-firmware?nvsram=true&staged=true"
            headers, data = create_multipart_formdata(files=[("nvsramfile", self.nvsram_name, self.nvsram),
                                                             ("dlpfile", self.firmware_name, self.firmware)])
        else:
            firmware_url = "firmware/embedded-firmware?nvsram=false&staged=true"
            headers, data = create_multipart_formdata(files=[("dlpfile", self.firmware_name, self.firmware)])

        # Stage firmware and nvsram
        try:

            rc, response = self.request(firmware_url, method="POST", data=data, headers=headers, timeout=(30 * 60))
        except Exception as error:
            self.module.fail_json(msg="Failed to stage firmware. Array Id [%s]. Error[%s]." % (self.ssid, to_native(error)))

        # Activate firmware
        activate_thread = threading.Thread(target=self.embedded_firmware_activate)
        activate_thread.start()
        self.wait_for_reboot()

    def wait_for_reboot(self):
        """Wait for controller A to fully reboot and web services running"""
        reboot_started = False
        reboot_completed = False
        self.module.log("Controller firmware: Reboot commencing. Array Id [%s]." % self.ssid)
        while self.wait_for_completion and not (reboot_started and reboot_completed):
            try:
                rc, response = self.request("storage-systems/%s/symbol/pingController?controller=a&verboseErrorResponse=true"
                                            % self.ssid, method="POST", timeout=10, log_request=False)

                if reboot_started and response == "ok":
                    self.module.log("Controller firmware: Reboot completed. Array Id [%s]." % self.ssid)
                    reboot_completed = True
                sleep(2)
            except Exception as error:
                if not reboot_started:
                    self.module.log("Controller firmware: Reboot started. Array Id [%s]." % self.ssid)
                    reboot_started = True
                continue

    def firmware_event_logger(self):
        """Determine if firmware activation has started."""
        # Determine the last known event
        try:
            rc, events = self.request("storage-systems/%s/events" % self.ssid)
            for event in events:
                if int(event["eventNumber"]) > int(self.last_known_event):
                    self.last_known_event = event["eventNumber"]
        except Exception as error:
            self.module.fail_json(msg="Failed to determine last known event. Array Id [%s]. Error[%s]." % (self.ssid, to_native(error)))

        while True:
            try:
                rc, events = self.request("storage-systems/%s/events?lastKnown=%s&wait=1" % (self.ssid, self.last_known_event), log_request=False)
                for event in events:
                    if int(event["eventNumber"]) > int(self.last_known_event):
                        self.last_known_event = event["eventNumber"]

                    # Log firmware events
                    if event["eventType"] == "firmwareDownloadEvent":
                        self.module.log("%s" % event["status"])
                        if event["status"] == "informational" and event["statusMessage"]:
                            self.module.log("Controller firmware: %s Array Id [%s]." % (event["statusMessage"], self.ssid))

                        # When activation is successful, finish thread
                        if event["status"] == "activate_success":
                            self.module.log("Controller firmware activated. Array Id [%s]." % self.ssid)
                            return
            except Exception as error:
                pass

    def wait_for_web_services(self):
        """Wait for web services to report firmware and nvsram upgrade."""
        # Wait for system to reflect changes
        for count in range(int(self.REBOOT_TIMEOUT_SEC / 5)):
            try:
                if self.is_firmware_bundled():
                    firmware_rc, firmware_version = self.request("storage-systems/%s/graph/xpath-filter?query=/controller/"
                                                                 "codeVersions[codeModule='bundleDisplay']" % self.ssid, log_request=False)
                    current_firmware_version = six.b(firmware_version[0]["versionString"])
                else:
                    firmware_rc, firmware_version = self.request("storage-systems/%s/graph/xpath-filter?query=/sa/saData/fwVersion"
                                                                 % self.ssid, log_request=False)
                    current_firmware_version = six.b(firmware_version[0])

                nvsram_rc, nvsram_version = self.request("storage-systems/%s/graph/xpath-filter?query=/sa/saData/nvsramVersion" % self.ssid, log_request=False)
                current_nvsram_version = six.b(nvsram_version[0])

                if current_firmware_version == self.firmware_version() and (not self.nvsram or current_nvsram_version == self.nvsram_version()):
                    break
            except Exception as error:
                pass
            sleep(5)
        else:
            self.module.fail_json(msg="Timeout waiting for Santricity Web Services. Array [%s]" % self.ssid)

        # Wait for system to be optimal
        for count in range(int(self.REBOOT_TIMEOUT_SEC / 5)):
            try:
                rc, response = self.request("storage-systems/%s" % self.ssid, log_request=False)

                if response["status"] == "optimal":
                    self.upgrade_in_progress = False
                    break
            except Exception as error:
                pass
            sleep(5)
        else:
            self.module.fail_json(msg="Timeout waiting for storage system to return to optimal status. Array [%s]" % self.ssid)

    def embedded_upgrade(self):
        """Upload and activate both firmware and NVSRAM."""
        download_thread = threading.Thread(target=self.embedded_firmware_download)
        event_thread = threading.Thread(target=self.firmware_event_logger)
        download_thread.start()
        event_thread.start()
        download_thread.join()
        event_thread.join()

    def proxy_check_nvsram_compatibility(self, retries=10):
        """Verify nvsram is compatible with E-Series storage system."""
        self.module.log("Checking nvsram compatibility...")
        data = {"storageDeviceIds": [self.ssid]}
        try:
            rc, check = self.request("firmware/compatibility-check", method="POST", data=data)
        except Exception as error:
            if retries:
                sleep(1)
                self.proxy_check_nvsram_compatibility(retries - 1)
            else:
                self.module.fail_json(msg="Failed to receive NVSRAM compatibility information. Array [%s]. Error [%s]." % (self.ssid, to_native(error)))

        for count in range(int(self.COMPATIBILITY_CHECK_TIMEOUT_SEC / 5)):
            try:
                rc, response = self.request("firmware/compatibility-check?requestId=%s" % check["requestId"])
            except Exception as error:
                continue

            if not response["checkRunning"]:
                for result in response["results"][0]["nvsramFiles"]:
                    if result["filename"] == self.nvsram_name:
                        return
                self.module.fail_json(msg="NVSRAM is not compatible. NVSRAM [%s]. Array [%s]." % (self.nvsram_name, self.ssid))
            sleep(5)

        self.module.fail_json(msg="Failed to retrieve NVSRAM status update from proxy. Array [%s]." % self.ssid)

    def proxy_check_firmware_compatibility(self, retries=10):
        """Verify firmware is compatible with E-Series storage system."""
        check = {}
        try:
            rc, check = self.request("firmware/compatibility-check", method="POST", data={"storageDeviceIds": [self.ssid]})
        except Exception as error:
            if retries:
                sleep(1)
                self.proxy_check_firmware_compatibility(retries - 1)
            else:
                self.module.fail_json(msg="Failed to receive firmware compatibility information. Array [%s]. Error [%s]." % (self.ssid, to_native(error)))

        for count in range(int(self.COMPATIBILITY_CHECK_TIMEOUT_SEC / 5)):
            try:
                rc, response = self.request("firmware/compatibility-check?requestId=%s" % check["requestId"])
            except Exception as error:
                continue

            if not response["checkRunning"]:
                for result in response["results"][0]["cfwFiles"]:
                    if result["filename"] == self.firmware_name:
                        return
                self.module.fail_json(msg="Firmware bundle is not compatible. firmware [%s]. Array [%s]." % (self.firmware_name, self.ssid))
            sleep(5)

        self.module.fail_json(msg="Failed to retrieve firmware status update from proxy. Array [%s]." % self.ssid)

    def proxy_upload_and_check_compatibility(self):
        """Ensure firmware/nvsram file is uploaded and verify compatibility."""
        uploaded_files = []
        try:
            rc, uploaded_files = self.request("firmware/cfw-files")
        except Exception as error:
            self.module.fail_json(msg="Failed to retrieve uploaded firmware and nvsram files. Error [%s]" % to_native(error))

        if self.firmware:
            for uploaded_file in uploaded_files:
                if uploaded_file["filename"] == self.firmware_name:
                    break
            else:
                fields = [("validate", "true")]
                files = [("firmwareFile", self.firmware_name, self.firmware)]
                headers, data = create_multipart_formdata(files=files, fields=fields)
                try:
                    rc, response = self.request("firmware/upload", method="POST", data=data, headers=headers)
                except Exception as error:
                    self.module.fail_json(msg="Failed to upload firmware bundle file. File [%s]. Array [%s]. Error [%s]."
                                              % (self.firmware_name, self.ssid, to_native(error)))
            self.proxy_check_firmware_compatibility()

        if self.nvsram:
            for uploaded_file in uploaded_files:
                if uploaded_file["filename"] == self.nvsram_name:
                    break
            else:
                fields = [("validate", "true")]
                files = [("firmwareFile", self.nvsram_name, self.nvsram)]
                headers, data = create_multipart_formdata(files=files, fields=fields)
                try:
                    rc, response = self.request("firmware/upload", method="POST", data=data, headers=headers)
                except Exception as error:
                    self.module.fail_json(msg="Failed to upload NVSRAM file. File [%s]. Array [%s]. Error [%s]."
                                              % (self.nvsram_name, self.ssid, to_native(error)))
            self.proxy_check_nvsram_compatibility()

    def proxy_check_upgrade_required(self):
        """Determine whether the onboard firmware/nvsram version is the same as the file"""
        # Verify controller consistency and get firmware versions
        if self.firmware:
            current_firmware_version = b""
            try:
                # Retrieve current bundle version
                if self.is_firmware_bundled():
                    rc, response = self.request("storage-systems/%s/graph/xpath-filter?query=/controller/codeVersions[codeModule='bundleDisplay']" % self.ssid)
                    current_firmware_version = six.b(response[0]["versionString"])
                else:
                    rc, response = self.request("storage-systems/%s/graph/xpath-filter?query=/sa/saData/fwVersion" % self.ssid)
                    current_firmware_version = six.b(response[0])
            except Exception as error:
                self.module.fail_json(msg="Failed to retrieve controller firmware information. Array [%s]. Error [%s]" % (self.ssid, to_native(error)))

            # Determine whether the current firmware version is the same as the file
            new_firmware_version = self.firmware_version()
            if current_firmware_version != new_firmware_version:
                self.upgrade_required = True
                
            # Build the modules information for logging purposes
            self.module_info.update({"bundleDisplay": {"onboard_version": current_firmware_version, "bundled_version": new_firmware_version}})

        # Determine current NVSRAM version and whether change is required
        if self.nvsram:
            try:
                rc, response = self.request("storage-systems/%s/graph/xpath-filter?query=/sa/saData/nvsramVersion" % self.ssid)

                if six.b(response[0]) != self.nvsram_version():
                    self.upgrade_required = True
            except Exception as error:
                self.module.fail_json(msg="Failed to retrieve storage system's NVSRAM version. Array [%s]. Error [%s]" % (self.ssid, to_native(error)))

    def proxy_wait_for_upgrade(self):
        """Wait for SANtricity Web Services Proxy to report upgrade complete"""
        self.module.log("(Proxy) Waiting for upgrade to complete...")

        status = {}
        while True:
            try:
                rc, status = self.request("storage-systems/%s/cfw-upgrade" % self.ssid, log_request=False, ignore_errors=True)
            except Exception as error:
                self.module.fail_json(msg="Failed to retrieve firmware upgrade status! Array [%s]. Error[%s]." % (self.ssid, to_native(error)))

            if "errorMessage" in status:
                self.module.warn("Proxy reported an error. Checking whether upgrade completed. Array [%s]. Error [%s]." % (self.ssid, status["errorMessage"]))
                self.wait_for_web_services()
                break

            if not status["running"]:
                if status["activationCompletionTime"]:
                    self.upgrade_in_progress = False
                    break
                else:
                    self.module.fail_json(msg="Failed to complete upgrade. Array [%s]." % self.ssid)
            sleep(5)

    def delete_mel_events(self):
        """Clear all mel-events."""
        try:
            rc, response = self.request("storage-systems/%s/mel-events?clearCache=true&resetMel=true" % self.ssid, method="DELETE")
        except Exception as error:
            self.module.fail_json(msg="Failed to clear mel-events. Array [%s]. Error [%s]." % (self.ssid, to_native(error)))

    def proxy_upgrade(self):
        """Activate previously uploaded firmware related files."""
        self.module.log("(Proxy) Firmware upgrade commencing...")
        body = {"stageFirmware": False, "skipMelCheck": self.clear_mel_events, "cfwFile": self.firmware_name}
        if self.nvsram:
            body.update({"nvsramFile": self.nvsram_name})

        try:
            rc, response = self.request("storage-systems/%s/cfw-upgrade" % self.ssid, method="POST", data=body)
        except Exception as error:
            self.module.fail_json(msg="Failed to initiate firmware upgrade. Array [%s]. Error [%s]." % (self.ssid, to_native(error)))

        self.upgrade_in_progress = True
        if self.wait_for_completion:
            self.proxy_wait_for_upgrade()

    def apply(self):
        """Upgrade controller firmware."""
        if self.is_upgrade_in_progress():
            self.module.fail_json(msg="Upgrade is already is progress. Array [%s]." % self.ssid)

        if self.is_embedded():
            self.embedded_check_compatibility()
        else:
            if not self.is_web_services_version_met(self.MINIMUM_PROXY_VERSION):
                self.module.fail_json(msg="Minimum proxy version %s required!")
            self.proxy_check_upgrade_required()

            # This will upload the firmware files to the web services proxy but not to the controller
            if self.upgrade_required:
                self.proxy_upload_and_check_compatibility()

        # Perform upgrade
        if self.upgrade_required and not self.module.check_mode:

            if self.clear_mel_events:
                self.delete_mel_events()

            if self.is_embedded():
                self.embedded_upgrade()
            else:
                self.proxy_upgrade()

        self.module.exit_json(changed=self.upgrade_required, upgrade_in_process=self.upgrade_in_progress, modules_info=self.module_info)


def main():
    firmware = NetAppESeriesFirmware()
    firmware.apply()


if __name__ == "__main__":
    main()