CLOS-6911: remove SysV runlevel links that shadow a real unit after the upgrade

packaging/leapp-repository.spec

@@ -42,7 +42,7 @@ py2_byte_compile "%1" "%2"}
 
 Name:           leapp-repository
 Version:        0.20.0
-Release:        11%{?dist}.cloudlinux
+Release:        12%{?dist}.cloudlinux
 Summary:        Repositories for leapp
 
 License:        ASL 2.0
@@ -298,6 +298,9 @@ done;
 
 # DO NOT TOUCH SECTION BELOW IN UPSTREAM
 %changelog
+* Wed Sep 02 2026 Roman Prilipskii <rprilipskii@cloudlinux.com> - 0.20.0-12.cloudlinux
+- CLOS-6911: Remove SysV runlevel links left over from the old system where the new one provides a real systemd service, which otherwise started the service outside its own unit - MariaDB was affected on servers using CloudLinux MySQL
+
 * Tue Aug 11 2026 Roman Prilipskii <rprilipskii@cloudlinux.com> - 0.20.0-11.cloudlinux
 - CLOS-4518: Fix systemd timers that replace a cron job on the new system being left disabled after the upgrade
 - CLOS-6809: Fix the upgrade being blocked on servers running MySQL Governor with MariaDB 11.4 or 11.8, where the installed database version was reported in a form that the cl-mysql repository check did not recognise

repos/system_upgrade/cloudlinux/actors/removestalesysvlinks/actor.py

@@ -0,0 +1,58 @@
+from leapp.actors import Actor
+from leapp.libraries.actor import removestalesysvlinks
+from leapp.libraries.common.cllaunch import run_on_cloudlinux
+from leapp.reporting import Report
+from leapp.tags import FinalizationPhaseTag, IPUWorkflowTag
+
+
+class RemoveStaleSysvLinks(Actor):
+    """
+    Remove SysV runlevel links that shadow a native systemd unit after the upgrade.
+
+    A package can ship both an init script and a systemd unit, and the runlevel
+    links chkconfig created on the source system belong to no package at all -
+    so nothing removes them during the upgrade. On the target,
+    systemd-sysv-generator turns such a link back into an LSB compatibility unit
+    that races the real one.
+
+    cl-MariaDB103-server is the case this was written for: it ships
+    /etc/rc.d/init.d/mysql on EL9 as well, the EL8 links survive, and MariaDB
+    ends up started by mysqld_safe outside mariadb.service - so
+    'systemctl start mariadb' fails against a server that is already running,
+    and the init script itself is broken on EL9, where log_success_msg no longer
+    exists.
+
+    leapp's own systemd state transition does not see the rc links - it works on
+    units - but it is not blind to the service, and that decides when this actor
+    may run. The EL9 cl-MariaDB103-server %posttrans recreates the
+    mysql.service/mysqld.service alias links and enables mariadb.service during
+    the RPM transaction; the transition then sees the unit enabled against a
+    disable preset and emits to_disable=[mariadb.service]. Applying that disable
+    removes the WantedBy link AND both alias symlinks, which is what lets
+    systemd-sysv-generator build a mysql.service from the init script at all.
+
+    Links whose service has no real unit on the target are left alone, since
+    there the init script is the only way that service runs.
+
+    Runs in FinalizationPhase.After, against the mounted target root before the
+    new system has booted. Two separate constraints pin it there:
+
+    - Not FirstBoot: by then the generator has already turned the links into
+      units and started the services, so removing them takes effect only one
+      boot later, and the conversion's finish stage fails before that boot.
+    - Not Finalization *Main*: set_systemd_services_state applies the disable
+      above in that stage, and leapp orders actors within a stage only by
+      produce/consume edges. This actor consumes nothing and produces only a
+      Report, so nothing pins the two relative to each other - and when this one
+      runs first, its enable is undone seconds later, leaving the unit disabled
+      with its S links already gone and nothing starting the database at boot.
+    """
+
+    name = 'remove_stale_sysv_links'
+    consumes = ()
+    produces = (Report,)
+    tags = (FinalizationPhaseTag.After, IPUWorkflowTag)
+
+    @run_on_cloudlinux
+    def process(self):
+        removestalesysvlinks.process()

repos/system_upgrade/cloudlinux/actors/removestalesysvlinks/libraries/removestalesysvlinks.py

@@ -0,0 +1,215 @@
+import os
+import re
+
+from leapp import reporting
+from leapp.libraries.common import systemd
+from leapp.libraries.stdlib import api, CalledProcessError
+
+FMT_LIST_SEPARATOR = '\n    - '
+
+RC_DIR_GLOB = '/etc/rc.d/rc{}.d'
+RUNLEVELS = range(0, 7)
+
+# Where a real unit can be shipped or administratively placed. Deliberately NOT
+# /run/systemd: that is where systemd-sysv-generator writes the units it makes
+# FROM the very links this actor removes. Asking a booted system
+# 'is there a <name>.service?' therefore always answers yes for any SysV service
+# with an init script, which makes the check circular and the answer worthless.
+UNIT_DIRS = ['/usr/lib/systemd/system', '/etc/systemd/system']
+
+# S64mysql / K36mysql -> ('S', 'mysql')
+_LINK_RE = re.compile(r'^(?P<action>[SK])(?P<order>[0-9]{2})(?P<name>.+)$')
+
+
+def _rc_dirs():
+    return [RC_DIR_GLOB.format(runlevel) for runlevel in RUNLEVELS]
+
+
+def parse_link_name(filename):
+    """
+    Split a SysV runlevel link name into its start/stop action and service name.
+
+    :return: (action, service name), or None when the name is not a runlevel link
+    :rtype: tuple[str, str] | None
+    """
+    match = _LINK_RE.match(filename)
+    if not match:
+        return None
+    return match.group('action'), match.group('name')
+
+
+def collect_sysv_links(rc_dirs=None):
+    """
+    Find the SysV runlevel links present on the system.
+
+    :return: Dictionary mapping a service name to the list of link paths, and a
+             set of the service names that have at least one start link
+    :rtype: tuple[dict[str, list[str]], set[str]]
+    """
+    links = {}
+    started_at_boot = set()
+    for rc_dir in rc_dirs if rc_dirs is not None else _rc_dirs():
+        try:
+            entries = os.listdir(rc_dir)
+        except OSError:
+            continue
+        for entry in entries:
+            path = os.path.join(rc_dir, entry)
+            if not os.path.islink(path):
+                continue
+            parsed = parse_link_name(entry)
+            if parsed is None:
+                continue
+            action, name = parsed
+            links.setdefault(name, []).append(path)
+            if action == 'S':
+                started_at_boot.add(name)
+    return links, started_at_boot
+
+
+def scan_unit_providers(unit_dirs=None):
+    """
+    Map every service name a real unit file provides to the unit providing it.
+
+    A unit provides its own name, and also every name it lists as an Alias in
+    its [Install] section - which is how the name a SysV script uses reaches a
+    differently named unit. cl-MariaDB103-server is exactly this shape: it ships
+    /etc/init.d/mysql and mariadb.service, no mysql.service, and mariadb.service
+    carries 'Alias=mysql.service'.
+
+    :return: Dictionary mapping a provided service name to the unit file name
+    :rtype: dict[str, str]
+    """
+    providers = {}
+    for unit_dir in unit_dirs if unit_dirs is not None else UNIT_DIRS:
+        try:
+            entries = sorted(os.listdir(unit_dir))
+        except OSError:
+            continue
+        for entry in entries:
+            if not entry.endswith('.service'):
+                continue
+            path = os.path.join(unit_dir, entry)
+            if os.path.islink(path):
+                # An alias symlink an earlier 'systemctl enable' created. The
+                # unit that actually provides the name is its target.
+                providers.setdefault(entry[:-len('.service')],
+                                     os.path.basename(os.readlink(path)))
+                continue
+            if not os.path.isfile(path):
+                continue
+            providers.setdefault(entry[:-len('.service')], entry)
+            for alias in _parse_aliases(path):
+                providers.setdefault(alias, entry)
+    return providers
+
+
+def _parse_aliases(path):
+    """Return the service names a unit file declares as [Install] Alias."""
+    aliases = []
+    section = None
+    try:
+        with open(path) as unit_file:
+            for line in unit_file:
+                line = line.strip()
+                if line.startswith('[') and line.endswith(']'):
+                    section = line[1:-1].lower()
+                    continue
+                if section != 'install' or not line.lower().startswith('alias'):
+                    continue
+                _, _, value = line.partition('=')
+                for name in value.split():
+                    if name.endswith('.service'):
+                        aliases.append(name[:-len('.service')])
+    except (OSError, UnicodeDecodeError):
+        return []
+    return aliases
+
+
+def select_shadowed_services(links, providers):
+    """
+    Keep only the services whose SysV links are shadowed by a real unit.
+
+    A SysV link is only stale when the target system has a real unit to take
+    over. Where it does not, the init script is the only way that service runs
+    and the links must be left alone.
+
+    :return: Dictionary mapping service name to (link paths, unit to enable)
+    :rtype: dict[str, tuple[list[str], str]]
+    """
+    return {
+        name: (paths, providers[name])
+        for name, paths in links.items()
+        if name in providers
+    }
+
+
+def process():
+    links, started_at_boot = collect_sysv_links()
+    if not links:
+        return
+
+    shadowed = select_shadowed_services(links, scan_unit_providers())
+    if not shadowed:
+        return
+
+    removed = []
+    enabled = []
+    for name, (paths, unit) in sorted(shadowed.items()):
+        # The SysV link was what started this service at boot, so hand that
+        # intent to the real unit before taking the link away. Without this the
+        # service would simply stop being started.
+        #
+        # The unit, never '<name>.service': on the name a SysV script uses,
+        # 'systemctl enable' finds no native unit, reports 'redirecting to
+        # systemd-sysv-install' and calls chkconfig, which RE-CREATES the very
+        # links being removed. Enabling is idempotent, so this does not need to
+        # know whether the unit was already enabled - which cannot be read
+        # reliably against a target that is not running yet.
+        if name in started_at_boot:
+            try:
+                systemd.enable_unit(unit)
+                enabled.append(unit)
+            except CalledProcessError as err:
+                api.current_logger().warning(
+                    'Failed to enable {}, leaving the SysV links for {} in place: {}'.format(unit, name, err)
+                )
+                continue
+
+        for path in sorted(paths):
+            try:
+                os.unlink(path)
+            except OSError as err:
+                api.current_logger().warning(
+                    'Failed to remove stale SysV link {}: {}'.format(path, err)
+                )
+                continue
+            removed.append(path)
+
+    if not removed:
+        return
+
+    api.current_logger().info('Removed stale SysV runlevel links: {}'.format(', '.join(removed)))
+
+    summary = (
+        'The following SysV runlevel links were left over from the previous major version'
+        ' while the upgraded system provides a native systemd unit for the same service.'
+        ' systemd-sysv-generator turns such a link back into a unit, which then races the'
+        ' real one - the service ends up started outside its own unit, and'
+        ' "systemctl start <name>" fails against a server that is already running.'
+        ' They have been removed:{}{}'.format(FMT_LIST_SEPARATOR, FMT_LIST_SEPARATOR.join(removed))
+    )
+    if enabled:
+        summary += (
+            '\n\nThe native units below were enabled first, so the services keep starting'
+            ' at boot as their SysV links used to arrange:{}{}'.format(
+                FMT_LIST_SEPARATOR, FMT_LIST_SEPARATOR.join(sorted(enabled))
+            )
+        )
+
+    reporting.create_report([
+        reporting.Title('Stale SysV runlevel links were removed'),
+        reporting.Summary(summary),
+        reporting.Severity(reporting.Severity.MEDIUM),
+        reporting.Groups([reporting.Groups.SERVICES]),
+    ])

repos/system_upgrade/cloudlinux/actors/removestalesysvlinks/tests/component_test_removestalesysvlinks.py

@@ -0,0 +1,46 @@
+"""Which workflow stage the actor runs in.
+
+Separate from the unit tests because pinning the stage means reading the tags off
+the LOADED actor: phase tags are defined by the repository, not the framework, so
+the snactor fixtures are what expose the real classes. Reading them rather than
+parsing the source is the point - a tag imported from the wrong module would
+satisfy a source-level check and not this one. Keeping it out of
+test_removestalesysvlinks.py leaves that file runnable against the framework
+alone, which is how every other test_*.py in this repository behaves.
+"""
+from leapp.snactor.fixture import current_actor_libraries, loaded_leapp_repository  # noqa: F401; pylint: disable=unused-import
+from leapp.tags import FinalizationPhaseTag, FirstBootPhaseTag
+
+
+class TestActorPhase(object):
+    """The links have to be gone before the new system boots, and the enable has
+    to outlive leapp's own systemd state transition.
+
+    Two distinct failures live here. On FirstBoot the generator has already
+    turned the links into units and started the services, so the removal takes
+    effect only one boot later - and the conversion's finish stage fails first.
+    In Finalization *Main* the ordering against set_systemd_services_state is
+    undefined (leapp sorts within a stage by produce/consume edges only, and this
+    actor consumes nothing and produces only a Report); when this actor runs
+    first its `systemctl enable` is undone by that actor's
+    `systemctl disable mariadb.service` a fraction of a second later, leaving the
+    unit disabled with its S links already removed. Observed on a real
+    conversion:
+
+        21:38:55.163 remove_stale_sysv_links:      systemctl enable  mariadb.service
+        21:38:55.891 set_systemd_services_state:   systemctl disable mariadb.service
+
+    Nothing then starts the database at boot. The ticket symptom disappears -
+    Plesk's finish stage re-enables and starts it - which is exactly why the end
+    state of a converted host is not evidence that this actor worked.
+
+    The tags are read from the loaded actor rather than parsed out of the source:
+    the snactor fixtures expose the real tag classes, so a tag imported from the
+    wrong module cannot satisfy this.
+    """
+
+    def test_runs_after_finalization_main(self, current_actor_libraries):
+        assert FinalizationPhaseTag.After in current_actor_libraries.tags
+
+    def test_does_not_run_on_first_boot(self, current_actor_libraries):
+        assert FirstBootPhaseTag not in current_actor_libraries.tags