22

✏️ Editing: system_upgrade.py

# -*- coding: utf-8 -*-
#
# Copyright (c) 2015-2020 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Author(s): Will Woods <wwoods@redhat.com>

"""system_upgrade.py - DNF plugin to handle major-version system upgrades."""

from subprocess import call, Popen, check_output, CalledProcessError
import json
import os
import os.path
import re
import sys
import uuid

from systemd import journal

from dnfpluginscore import _, logger

import dnf
import dnf.cli
from dnf.cli import CliError
from dnf.i18n import ucd
import dnf.transaction
from dnf.transaction_sr import serialize_transaction, TransactionReplay

import libdnf.conf

# Translators: This string is only used in unit tests.
_("the color of the sky")

DOWNLOAD_FINISHED_ID = uuid.UUID('9348174c5cc74001a71ef26bd79d302e')
REBOOT_REQUESTED_ID = uuid.UUID('fef1cc509d5047268b83a3a553f54b43')
UPGRADE_STARTED_ID = uuid.UUID('3e0a5636d16b4ca4bbe5321d06c6aa62')
UPGRADE_FINISHED_ID = uuid.UUID('8cec00a1566f4d3594f116450395f06c')

ID_TO_IDENTIFY_BOOTS = UPGRADE_STARTED_ID

PLYMOUTH = '/usr/bin/plymouth'

RELEASEVER_MSG = _(
    "Need a --releasever greater than the current system version.")
DOWNLOAD_FINISHED_MSG = _(  # Translators: do not change "reboot" here
    "Download complete! Use 'dnf {command} reboot' to start the upgrade.\n"
    "To remove cached metadata and transaction use 'dnf {command} clean'")
CANT_RESET_RELEASEVER = _(
    "Sorry, you need to use 'download --releasever' instead of '--network'")

STATE_VERSION = 2

# --- Miscellaneous helper functions ------------------------------------------

def reboot():
    if os.getenv("DNF_SYSTEM_UPGRADE_NO_REBOOT", default=False):
        logger.info(_("Reboot turned off, not rebooting."))
    else:
        Popen(["systemctl", "reboot"])

def get_url_from_os_release():
    key = "UPGRADE_GUIDE_URL="
    for path in ["/etc/os-release", "/usr/lib/os-release"]:
        try:
            with open(path) as release_file:
                for line in release_file:
                    line = line.strip()
                    if line.startswith(key):
                        return line[len(key):].strip('"')
        except IOError:
            continue
    return None

# DNF-FIXME: dnf.util.clear_dir() doesn't delete regular files :/
def clear_dir(path, ignore=[]):
    if not os.path.isdir(path):
        return

for entry in os.listdir(path):
        fullpath = os.path.join(path, entry)
        if fullpath in ignore:
            continue
        try:
            if os.path.isdir(fullpath):
                dnf.util.rm_rf(fullpath)
            else:
                os.unlink(fullpath)
        except OSError:
            pass

def check_release_ver(conf, target=None):
    if dnf.rpm.detect_releasever(conf.installroot) == conf.releasever:
        raise CliError(RELEASEVER_MSG)
    if target and target != conf.releasever:
        # it's too late to set releasever here, so this can't work.
        # (see https://bugzilla.redhat.com/show_bug.cgi?id=1212341)
        raise CliError(CANT_RESET_RELEASEVER)

def disable_blanking():
    try:
        tty = open('/dev/tty0', 'wb')
        tty.write(b'\33[9;0]')
    except Exception as e:
        print(_("Screen blanking can't be disabled: %s") % e)

# --- State object - for tracking upgrade state between runs ------------------

# DNF-INTEGRATION-NOTE: basically the same thing as dnf.persistor.JSONDB
class State(object):
    def __init__(self, statefile):
        self.statefile = statefile
        self._data = {}
        self._read()

def _read(self):
        try:
            with open(self.statefile) as fp:
                self._data = json.load(fp)
        except IOError:
            self._data = {}
        except ValueError:
            self._data = {}
            logger.warning(_("Failed loading state file: %s, continuing with "
                             "empty state."), self.statefile)

def write(self):
        dnf.util.ensure_dir(os.path.dirname(self.statefile))
        with open(self.statefile, 'w') as outf:
            json.dump(self._data, outf, indent=4, sort_keys=True)

def clear(self):
        if os.path.exists(self.statefile):
            os.unlink(self.statefile)
        self._read()

def __enter__(self):
        return self

def __exit__(self, exc_type, exc_value, traceback):
        if exc_type is None:
            self.write()

# helper function for creating properties. pylint: disable=protected-access
    def _prop(option):  # pylint: disable=no-self-argument
        def setprop(self, value):
            self._data[option] = value

def getprop(self):
            return self._data.get(option)
        return property(getprop, setprop)

#  !!! Increase STATE_VERSION for any changes in data structure like a new property or a new
    #  data structure !!!
    state_version = _prop("state_version")
    download_status = _prop("download_status")
    destdir = _prop("destdir")
    target_releasever = _prop("target_releasever")
    system_releasever = _prop("system_releasever")
    gpgcheck = _prop("gpgcheck")
    # list of repos with gpgcheck=True
    gpgcheck_repos = _prop("gpgcheck_repos")
    # list of repos with repo_gpgcheck=True
    repo_gpgcheck_repos = _prop("repo_gpgcheck_repos")
    upgrade_status = _prop("upgrade_status")
    upgrade_command = _prop("upgrade_command")
    distro_sync = _prop("distro_sync")
    enable_disable_repos = _prop("enable_disable_repos")
    module_platform_id = _prop("module_platform_id")

# --- Plymouth output helpers -------------------------------------------------

class PlymouthOutput(object):
    """A plymouth output helper class.

Filters duplicate calls, and stops calling the plymouth binary if we
    fail to contact it.
    """

def __init__(self):
        self.alive = True
        self._last_args = dict()
        self._last_msg = None

def _plymouth(self, cmd, *args):
        dupe_cmd = (args == self._last_args.get(cmd))
        if (self.alive and not dupe_cmd) or cmd == '--ping':
            try:
                self.alive = (call((PLYMOUTH, cmd) + args) == 0)
            except OSError:
                self.alive = False
            self._last_args[cmd] = args
        return self.alive

def ping(self):
        return self._plymouth("--ping")

def message(self, msg):
        if self._last_msg and self._last_msg != msg:
            self._plymouth("hide-message", "--text", self._last_msg)
        self._last_msg = msg
        return self._plymouth("display-message", "--text", msg)

def set_mode(self):
        mode = 'updates'
        try:
            s = check_output([PLYMOUTH, '--help'])
            if re.search('--system-upgrade', ucd(s)):
                mode = 'system-upgrade'
        except (CalledProcessError, OSError):
            pass
        return self._plymouth("change-mode", "--" + mode)

def progress(self, percent):
        return self._plymouth("system-update", "--progress", str(percent))

# A single PlymouthOutput instance for us to use within this module
Plymouth = PlymouthOutput()

# A TransactionProgress class that updates plymouth for us.
class PlymouthTransactionProgress(dnf.callback.TransactionProgress):

# pylint: disable=too-many-arguments
    def progress(self, package, action, ti_done, ti_total, ts_done, ts_total):
        self._update_plymouth(package, action, ts_done, ts_total)

def _update_plymouth(self, package, action, current, total):
        # Prevents quick jumps of progressbar when pretrans scriptlets
        # and TRANS_PREPARATION are reported as 1/1
        if total == 1:
            return
        # Verification goes through all the packages again,
        # which resets the "current" param value, this prevents
        # resetting of the progress bar as well. (Rhbug:1809096)
        if action != dnf.callback.PKG_VERIFY:
            Plymouth.progress(int(90.0 * current / total))
        else:
            Plymouth.progress(90 + int(10.0 * current / total))

Plymouth.message(self._fmt_event(package, action, current, total))

def _fmt_event(self, package, action, current, total):
        action = dnf.transaction.ACTIONS.get(action, action)
        return "[%d/%d] %s %s..." % (current, total, action, package)

# --- journal helpers -------------------------------------------------

def find_boots(message_id):
    """Find all boots with this message id.

Returns the entries of all found boots.
    """
    j = journal.Reader()
    j.add_match(MESSAGE_ID=message_id.hex,  # identify the message
                _UID=0)                     # prevent spoofing of logs

oldboot = None
    for entry in j:
        boot = entry['_BOOT_ID']
        if boot == oldboot:
            continue
        oldboot = boot
        yield entry

def list_logs():
    print(_('The following boots appear to contain upgrade logs:'))
    n = -1
    for n, entry in enumerate(find_boots(ID_TO_IDENTIFY_BOOTS)):
        print('{} / {.hex}: {:%Y-%m-%d %H:%M:%S} {}→{}'.format(
            n + 1,
            entry['_BOOT_ID'],
            entry['__REALTIME_TIMESTAMP'],
            entry.get('SYSTEM_RELEASEVER', '??'),
            entry.get('TARGET_RELEASEVER', '??')))
    if n == -1:
        print(_('-- no logs were found --'))

def pick_boot(message_id, n):
    boots = list(find_boots(message_id))
    # Positive indices index all found boots starting with 1 and going forward,
    # zero is the current boot, and -1, -2, -3 are previous going backwards.
    # This is the same as journalctl.
    try:
        if n == 0:
            raise IndexError
        if n > 0:
            n -= 1
        return boots[n]['_BOOT_ID']
    except IndexError:
        raise CliError(_("Cannot find logs with this index."))

def show_log(n):
    boot_id = pick_boot(ID_TO_IDENTIFY_BOOTS, n)
    process = Popen(['journalctl', '--boot', boot_id.hex])
    process.wait()
    rc = process.returncode
    if rc == 1:
        raise dnf.exceptions.Error(_("Unable to match systemd journal entry"))

CMDS = ['download', 'clean', 'reboot', 'upgrade', 'log']

# --- The actual Plugin and Command objects! ----------------------------------

class SystemUpgradePlugin(dnf.Plugin):
    name = 'system-upgrade'

def __init__(self, base, cli):
        super(SystemUpgradePlugin, self).__init__(base, cli)
        if cli:
            cli.register_command(SystemUpgradeCommand)
            cli.register_command(OfflineUpgradeCommand)
            cli.register_command(OfflineDistrosyncCommand)

class SystemUpgradeCommand(dnf.cli.Command):
    aliases = ('system-upgrade', 'fedup',)
    summary = _("Prepare system for upgrade to a new release")

DATADIR = 'var/lib/dnf/system-upgrade'

def __init__(self, cli):
        super(SystemUpgradeCommand, self).__init__(cli)
        self.datadir = os.path.join(cli.base.conf.installroot, self.DATADIR)
        self.transaction_file = os.path.join(self.datadir, 'system-upgrade-transaction.json')
        self.magic_symlink = os.path.join(cli.base.conf.installroot, 'system-update')

self.state = State(os.path.join(self.datadir, 'system-upgrade-state.json'))

@staticmethod
    def set_argparser(parser):
        parser.add_argument("--no-downgrade", dest='distro_sync',
                            action='store_false',
                            help=_("keep installed packages if the new "
                                   "release's version is older"))
        parser.add_argument('tid', nargs=1, choices=CMDS,
                            metavar="[%s]" % "|".join(CMDS))
        parser.add_argument('--number', type=int, help=_('which logs to show'))

def log_status(self, message, message_id):
        """Log directly to the journal."""
        journal.send(message,
                     MESSAGE_ID=message_id,
                     PRIORITY=journal.LOG_NOTICE,
                     SYSTEM_RELEASEVER=self.state.system_releasever,
                     TARGET_RELEASEVER=self.state.target_releasever,
                     DNF_VERSION=dnf.const.VERSION)

def pre_configure(self):
        self._call_sub("check")
        self._call_sub("pre_configure")

def configure(self):
        self._call_sub("configure")

def run(self):
        self._call_sub("run")

def run_transaction(self):
        self._call_sub("transaction")

def run_resolved(self):
        self._call_sub("resolved")

def _call_sub(self, name):
        subfunc = getattr(self, name + '_' + self.opts.tid[0], None)
        if callable(subfunc):
            subfunc()

def _check_state_version(self, command):
        if self.state.state_version != STATE_VERSION:
            msg = _("Incompatible version of data. Rerun 'dnf {command} download [OPTIONS]'"
                    "").format(command=command)
            raise CliError(msg)

def _set_cachedir(self):
        # set download directories from json state file
        self.base.conf.cachedir = self.datadir
        self.base.conf.destdir = self.state.destdir if self.state.destdir else None

def _get_forward_reverse_pkg_reason_pairs(self):
        """
        forward = {repoid:{pkg_nevra: {tsi.action: tsi.reason}}
        reverse = {pkg_nevra: {tsi.action: tsi.reason}}
        :return: forward, reverse
        """
        backward_action = set(dnf.transaction.BACKWARD_ACTIONS + [libdnf.transaction.TransactionItemAction_REINSTALLED])
        forward_actions = set(dnf.transaction.FORWARD_ACTIONS)

forward = {}
        reverse = {}
        for tsi in self.cli.base.transaction:
            if tsi.action in forward_actions:
                pkg = tsi.pkg
                forward.setdefault(pkg.repo.id, {}).setdefault(
                    str(pkg), {})[tsi.action] = tsi.reason
            elif tsi.action in backward_action:
                reverse.setdefault(str(tsi.pkg), {})[tsi.action] = tsi.reason
        return forward, reverse

# == pre_configure_*: set up action-specific demands ==========================
    def pre_configure_download(self):
        # only download subcommand accepts --destdir command line option
        self.base.conf.cachedir = self.datadir
        self.base.conf.destdir = self.opts.destdir if self.opts.destdir else None
        if 'offline-distrosync' == self.opts.command and not self.opts.distro_sync:
            raise CliError(
                _("Command 'offline-distrosync' cannot be used with --no-downgrade option"))
        elif 'offline-upgrade' == self.opts.command:
            self.opts.distro_sync = False

def pre_configure_reboot(self):
        self._set_cachedir()

def pre_configure_upgrade(self):
        self._set_cachedir()
        if self.state.enable_disable_repos:
            self.opts.repos_ed = self.state.enable_disable_repos
        self.base.conf.releasever = self.state.target_releasever

def pre_configure_clean(self):
        self._set_cachedir()

# == configure_*: set up action-specific demands ==========================

def configure_download(self):
        if 'system-upgrade' == self.opts.command or 'fedup' == self.opts.command:
            logger.warning(_('WARNING: this operation is not supported on the RHEL distribution. '
                             'Proceed at your own risk.'))
            help_url = get_url_from_os_release()
            if help_url:
                msg = _('Additional information for System Upgrade: {}')
                logger.info(msg.format(ucd(help_url)))
            if self.base._promptWanted():
                msg = _('Before you continue ensure that your system is fully upgraded by running '
                        '"dnf --refresh upgrade". Do you want to continue')
                if self.base.conf.assumeno or not self.base.output.userconfirm(
                        msg='{} [y/N]: '.format(msg), defaultyes_msg='{} [Y/n]: '.format(msg)):
                    logger.error(_("Operation aborted."))
                    sys.exit(1)
            check_release_ver(self.base.conf, target=self.opts.releasever)
        elif 'offline-upgrade' == self.opts.command:
            self.cli._populate_update_security_filter(self.opts)

self.cli.demands.root_user = True
        self.cli.demands.resolving = True
        self.cli.demands.available_repos = True
        self.cli.demands.sack_activation = True
        self.cli.demands.freshest_metadata = True
        # We want to do the depsolve / download / transaction-test, but *not*
        # run the actual RPM transaction to install the downloaded packages.
        # Setting the "test" flag makes the RPM transaction a test transaction,
        # so nothing actually gets installed.
        # (It also means that we run two test transactions in a row, which is
        # kind of silly, but that's something for DNF to fix...)
        self.base.conf.tsflags += ["test"]

def configure_reboot(self):
        # FUTURE: add a --debug-shell option to enable debug shell:
        # systemctl add-wants system-update.target debug-shell.service
        self.cli.demands.root_user = True

def configure_upgrade(self):
        # same as the download, but offline and non-interactive. so...
        self.cli.demands.root_user = True
        self.cli.demands.resolving = True
        self.cli.demands.available_repos = True
        self.cli.demands.sack_activation = True
        # use the saved value for --allowerasing, etc.
        self.opts.distro_sync = self.state.distro_sync
        if self.state.gpgcheck is not None:
            self.base.conf.gpgcheck = self.state.gpgcheck
        if self.state.gpgcheck_repos is not None:
            for repo in self.base.repos.values():
                repo.gpgcheck = repo.id in self.state.gpgcheck_repos
        if self.state.repo_gpgcheck_repos is not None:
            for repo in self.base.repos.values():
                repo.repo_gpgcheck = repo.id in self.state.repo_gpgcheck_repos
        self.base.conf.module_platform_id = self.state.module_platform_id
        # don't try to get new metadata, 'cuz we're offline
        self.cli.demands.cacheonly = True
        # and don't ask any questions (we confirmed all this beforehand)
        self.base.conf.assumeyes = True
        self.cli.demands.transaction_display = PlymouthTransactionProgress()
        # upgrade operation already removes all element that must be removed. Additional removal
        # could trigger unwanted changes in transaction.
        self.base.conf.clean_requirements_on_remove = False
        self.base.conf.install_weak_deps = False

def configure_clean(self):
        self.cli.demands.root_user = True

def configure_log(self):
        pass

# == check_*: do any action-specific checks ===============================

def check_reboot(self):
        if not self.state.download_status == 'complete':
            raise CliError(_("system is not ready for upgrade"))
        self._check_state_version(self.opts.command)
        if self.state.upgrade_command != self.opts.command:
            msg = _("the transaction was not prepared for '{command}'. "
                    "Rerun 'dnf {command} download [OPTIONS]'").format(command=self.opts.command)
            raise CliError(msg)
        if os.path.lexists(self.magic_symlink):
            raise CliError(_("upgrade is already scheduled"))
        dnf.util.ensure_dir(self.datadir)
        # FUTURE: checkRPMDBStatus(self.state.download_transaction_id)

def check_upgrade(self):
        if not os.path.lexists(self.magic_symlink):
            logger.info(_("trigger file does not exist. exiting quietly."))
            raise SystemExit(0)
        if os.readlink(self.magic_symlink) != self.datadir:
            logger.info(_("another upgrade tool is running. exiting quietly."))
            raise SystemExit(0)
        # Delete symlink ASAP to avoid reboot loops
        dnf.yum.misc.unlink_f(self.magic_symlink)
        command = self.state.upgrade_command
        if not command:
            command = self.opts.command
        self._check_state_version(command)
        if not self.state.upgrade_status == 'ready':
            msg = _("use 'dnf {command} reboot' to begin the upgrade").format(command=command)
            raise CliError(msg)

# == run_*: run the action/prep the transaction ===========================

def run_prepare(self):
        # make the magic symlink
        os.symlink(self.datadir, self.magic_symlink)
        # set upgrade_status so that the upgrade can run
        with self.state as state:
            state.upgrade_status = 'ready'

def run_reboot(self):
        self.run_prepare()

if not self.opts.tid[0] == "reboot":
            return

self.log_status(_("Rebooting to perform upgrade."),
                        REBOOT_REQUESTED_ID)
        reboot()

def run_download(self):
        # Mark everything in the world for upgrade/sync
        if self.opts.distro_sync:
            self.base.distro_sync()
        else:
            self.base.upgrade_all()

if self.opts.command not in ['offline-upgrade', 'offline-distrosync']:
            # Mark all installed groups and environments for upgrade
            self.base.read_comps()
            installed_groups = [g.id for g in self.base.comps.groups if self.base.history.group.get(g.id)]
            if installed_groups:
                self.base.env_group_upgrade(installed_groups)
            installed_environments = [g.id for g in self.base.comps.environments if self.base.history.env.get(g.id)]
            if installed_environments:
                self.base.env_group_upgrade(installed_environments)

with self.state as state:
            state.download_status = 'downloading'
            state.target_releasever = self.base.conf.releasever
            state.destdir = self.base.conf.destdir

def run_upgrade(self):
        # change the upgrade status (so we can detect crashed upgrades later)
        command = ''
        with self.state as state:
            state.upgrade_status = 'incomplete'
            command = state.upgrade_command
        if command == 'offline-upgrade':
            msg = _("Starting offline upgrade. This will take a while.")
        elif command == 'offline-distrosync':
            msg = _("Starting offline distrosync. This will take a while.")
        else:
            msg = _("Starting system upgrade. This will take a while.")

self.log_status(msg, UPGRADE_STARTED_ID)

# reset the splash mode and let the user know we're running
        Plymouth.set_mode()
        Plymouth.progress(0)
        Plymouth.message(msg)

# disable screen blanking
        disable_blanking()

self.replay = TransactionReplay(self.base, self.transaction_file)
        self.replay.run()

def run_clean(self):
        logger.info(_("Cleaning up downloaded data..."))
        # Don't delete persistor, it contains paths for downloaded packages
        # that are used by dnf during finalizing base to clean them up
        clear_dir(self.base.conf.cachedir,
                  [dnf.persistor.TempfilePersistor(self.base.conf.cachedir).db_path])
        with self.state as state:
            state.download_status = None
            state.state_version = None
            state.upgrade_status = None
            state.upgrade_command = None
            state.destdir = None

def run_log(self):
        if self.opts.number:
            show_log(self.opts.number)
        else:
            list_logs()

# == resolved_*: do staff after succesful resolvement =====================

def resolved_upgrade(self):
        """Adjust transaction reasons according to stored values"""
        self.replay.post_transaction()

# == transaction_*: do stuff after a successful transaction ===============

def transaction_download(self):
        transaction = self.base.history.get_current()

if not transaction.packages():
            logger.info(_("The system-upgrade transaction is empty, your system is already up-to-date."))
            return

data = serialize_transaction(transaction)
        try:
            with open(self.transaction_file, "w") as f:
                json.dump(data, f, indent=4, sort_keys=True)
                f.write("\n")

print(_("Transaction saved to {}.").format(self.transaction_file))

except OSError as e:
            raise dnf.cli.CliError(_('Error storing transaction: {}').format(str(e)))

# Okay! Write out the state so the upgrade can use it.
        system_ver = dnf.rpm.detect_releasever(self.base.conf.installroot)
        with self.state as state:
            state.download_status = 'complete'
            state.state_version = STATE_VERSION
            state.distro_sync = self.opts.distro_sync
            state.gpgcheck = self.base.conf.gpgcheck
            state.gpgcheck_repos = [
                repo.id for repo in self.base.repos.values() if repo.gpgcheck]
            state.repo_gpgcheck_repos = [
                repo.id for repo in self.base.repos.values() if repo.repo_gpgcheck]
            state.system_releasever = system_ver
            state.target_releasever = self.base.conf.releasever
            state.module_platform_id = self.base.conf.module_platform_id
            state.enable_disable_repos = self.opts.repos_ed
            state.destdir = self.base.conf.destdir
            state.upgrade_command = self.opts.command

msg = DOWNLOAD_FINISHED_MSG.format(command=self.opts.command)
        logger.info(msg)
        self.log_status(_("Download finished."), DOWNLOAD_FINISHED_ID)

def transaction_upgrade(self):
        Plymouth.message(_("Upgrade complete! Cleaning up and rebooting..."))
        self.log_status(_("Upgrade complete! Cleaning up and rebooting..."),
                        UPGRADE_FINISHED_ID)
        self.run_clean()
        if self.opts.tid[0] == "upgrade":
            reboot()

class OfflineUpgradeCommand(SystemUpgradeCommand):
    aliases = ('offline-upgrade',)
    summary = _("Prepare offline upgrade of the system")

class OfflineDistrosyncCommand(SystemUpgradeCommand):
    aliases = ('offline-distrosync',)
    summary = _("Prepare offline distrosync of the system")

MR SKB