]> git.ipfire.org Git - thirdparty/systemd.git/blobdiff - test/test-network/systemd-networkd-tests.py
test-network: make path to networkd or friends configurable
[thirdparty/systemd.git] / test / test-network / systemd-networkd-tests.py
index 5c8b5db9eac5c106dfb7cc188e0094c83f39ac15..84de51686841453df51593b46b8a47d397d1bfd0 100755 (executable)
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: LGPL-2.1+
 # systemd-networkd tests
 
+import argparse
 import os
 import re
 import shutil
@@ -22,7 +23,12 @@ network_sysctl_ipv4_path='/proc/sys/net/ipv4/conf'
 dnsmasq_pid_file='/run/networkd-ci/test-test-dnsmasq.pid'
 dnsmasq_log_file='/run/networkd-ci/test-dnsmasq-log-file'
 
+networkd_bin='/usr/lib/systemd/systemd-networkd'
 wait_online_bin='/usr/lib/systemd/systemd-networkd-wait-online'
+networkctl_bin='/usr/bin/networkctl'
+use_valgrind=False
+enable_debug=False
+env = {}
 
 def is_module_available(module_name):
     lsmod_output = subprocess.check_output('lsmod', universal_newlines=True)
@@ -70,35 +76,6 @@ def expectedFailureIfRoutingPolicyIPProtoIsNotAvailable():
 
     return f
 
-def expectedFailureIf_ip6gre_do_not_support_ipv6ll():
-    def f(func):
-        rc = subprocess.call(['ip', 'link', 'add', 'name', 'test1', 'type', 'dummy'])
-        if rc != 0:
-            return unittest.expectedFailure(func)
-
-        time.sleep(1)
-        rc = subprocess.call(['ip', 'tunnel', 'add', 'tun99', 'local', '2a00:ffde:4567:edde::4986', 'remote', '2001:473:fece:cafe::5178', 'mode', 'ip6gre', 'dev', 'dummy99'])
-        if rc != 0:
-            return unittest.expectedFailure(func)
-
-        time.sleep(1)
-        # Not sure why, but '0' or '2' do not work.
-        rc = subprocess.call(['sysctl', '-w', 'net.ipv6.conf.tun99.addr_gen_mode=3'])
-        if rc != 0:
-            return unittest.expectedFailure(func)
-
-        time.sleep(1)
-        rc = subprocess.run(['ip', '-6', 'address', 'show', 'dev', 'tun99', 'scope', 'link'], stdout=subprocess.PIPE)
-        if rc.returncode != 0:
-            return unittest.expectedFailure(func)
-
-        if 'inet6' not in rc.stdout.rstrip().decode('utf-8'):
-            return unittest.expectedFailure(func)
-
-        return func
-
-    return f
-
 def setUpModule():
     os.makedirs(network_unit_file_path, exist_ok=True)
     os.makedirs(networkd_ci_path, exist_ok=True)
@@ -108,8 +85,35 @@ def setUpModule():
 
     subprocess.check_call('systemctl stop systemd-networkd.socket', shell=True)
 
+    drop_in = [
+        '[Service]',
+        'Restart=no',
+        'ExecStart=',
+    ]
+    if use_valgrind:
+        drop_in += [
+            'ExecStart=!!valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all ' + networkd_bin,
+            'PrivateTmp=yes',
+            'MemoryDenyWriteExecute=no',
+        ]
+    else:
+        drop_in += ['ExecStart=!!' + networkd_bin]
+    if enable_debug:
+        drop_in += ['Environment=SYSTEMD_LOG_LEVEL=debug']
+
+    drop_in_str = '\n'.join(drop_in)
+    print(drop_in_str)
+
+    os.makedirs('/run/systemd/system/systemd-networkd.service.d', exist_ok=True)
+    with open('/run/systemd/system/systemd-networkd.service.d/00-override.conf', mode='w') as f:
+        f.write(drop_in_str)
+
+    subprocess.check_call('systemctl daemon-reload', shell=True)
+
 def tearDownModule():
     shutil.rmtree(networkd_ci_path)
+    shutil.rmtree('/run/systemd/system/systemd-networkd.service.d')
+    subprocess.check_call('systemctl daemon-reload', shell=True)
 
     subprocess.check_call('systemctl stop systemd-networkd.service', shell=True)
     subprocess.check_call('systemctl start systemd-networkd.socket', shell=True)
@@ -129,17 +133,32 @@ class Utilities():
         with open(os.path.join(path, attribute)) as f:
             return f.readline().strip()
 
-    def link_exits(self, link):
+    def link_exists(self, link):
         return os.path.exists(os.path.join('/sys/class/net', link))
 
-    def link_remove(self, links):
+    def check_link_exists(self, link):
+        self.assertTrue(self.link_exists(link))
+
+    def remove_links(self, links):
         for link in links:
-            if os.path.exists(os.path.join('/sys/class/net', link)):
+            if self.link_exists(link):
                 subprocess.call(['ip', 'link', 'del', 'dev', link])
         time.sleep(1)
 
+    def remove_fou_ports(self, ports):
+        for port in ports:
+            subprocess.call(['ip', 'fou', 'del', 'port', port])
+
+    def remove_routing_policy_rule_tables(self, tables):
+        for table in tables:
+            subprocess.call(['ip', 'rule', 'del', 'table', table])
+
+    def remove_routes(self, routes):
+        for route_type, addr in routes:
+            subprocess.call(['ip', 'route', 'del', route_type, addr])
+
     def l2tp_tunnel_remove(self, tunnel_ids):
-        output = subprocess.check_output(['ip', 'l2tp', 'show', 'tunnel']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'l2tp', 'show', 'tunnel'], universal_newlines=True).rstrip()
         for tid in tunnel_ids:
             words='Tunnel ' + tid + ', encap'
             if words in output:
@@ -168,11 +187,15 @@ class Utilities():
                 if (os.path.exists(os.path.join(network_unit_file_path, unit + '.d'))):
                     shutil.rmtree(os.path.join(network_unit_file_path, unit + '.d'))
 
-    def start_dnsmasq(self, additional_options='', lease_time='1h'):
-        dnsmasq_command = f'dnsmasq -8 /var/run/networkd-ci/test-dnsmasq-log-file --log-queries=extra --log-dhcp --pid-file=/var/run/networkd-ci/test-test-dnsmasq.pid --conf-file=/dev/null --interface=veth-peer --enable-ra --dhcp-range=2600::10,2600::20,{lease_time} --dhcp-range=192.168.5.10,192.168.5.200,{lease_time} -R --dhcp-leasefile=/var/run/networkd-ci/lease --dhcp-option=26,1492 --dhcp-option=option:router,192.168.5.1 --dhcp-option=33,192.168.5.4,192.168.5.5 --port=0 ' + additional_options
-        subprocess.check_call(dnsmasq_command, shell=True)
+    def warn_about_firewalld(self):
+        rc = subprocess.call(['systemctl', '-q', 'is-active', 'firewalld.service'])
+        if rc == 0:
+            print('\nWARNING: firewalld.service is active. The test may fail.')
 
-        time.sleep(10)
+    def start_dnsmasq(self, additional_options='', ipv4_range='192.168.5.10,192.168.5.200', ipv6_range='2600::10,2600::20', lease_time='1h'):
+        self.warn_about_firewalld()
+        dnsmasq_command = f'dnsmasq -8 /var/run/networkd-ci/test-dnsmasq-log-file --log-queries=extra --log-dhcp --pid-file=/var/run/networkd-ci/test-test-dnsmasq.pid --conf-file=/dev/null --interface=veth-peer --enable-ra --dhcp-range={ipv6_range},{lease_time} --dhcp-range={ipv4_range},{lease_time} -R --dhcp-leasefile=/var/run/networkd-ci/lease --dhcp-option=26,1492 --dhcp-option=option:router,192.168.5.1 --dhcp-option=33,192.168.5.4,192.168.5.5 --port=0 ' + additional_options
+        subprocess.check_call(dnsmasq_command, shell=True)
 
     def stop_dnsmasq(self, pid_file):
         if os.path.exists(pid_file):
@@ -188,7 +211,7 @@ class Utilities():
                 contents = in_file.read()
                 if show_all:
                     print(contents)
-                for line in contents.split('\n'):
+                for line in contents.splitlines():
                     if words in line:
                         in_file.close()
                         print("%s, %s" % (words, line))
@@ -215,10 +238,39 @@ class Utilities():
             time.sleep(sleep_sec)
 
     def wait_online(self, links_with_operstate, timeout='20s', bool_any=False):
-        args = [wait_online_bin, f'--timeout={timeout}'] + [f'--interface={link}' for link in links_with_operstate]
+        args = wait_online_cmd + [f'--timeout={timeout}'] + [f'--interface={link}' for link in links_with_operstate]
         if bool_any:
             args += ['--any']
-        subprocess.check_call(args)
+        try:
+            subprocess.check_call(args, env=env)
+        except subprocess.CalledProcessError:
+            for link in links_with_operstate:
+                output = subprocess.check_output(networkctl_cmd + ['status', link.split(':')[0]], universal_newlines=True, env=env).rstrip()
+                print(output)
+            raise
+
+    def get_operstate(self, link, show_status=True, setup_state='configured'):
+        output = subprocess.check_output(networkctl_cmd + ['status', link], universal_newlines=True, env=env).rstrip()
+        if show_status:
+            print(output)
+        for line in output.splitlines():
+            if 'State:' in line and (not setup_state or setup_state in line):
+                return line.split()[1]
+        return None
+
+    def check_operstate(self, link, expected, show_status=True, setup_state='configured'):
+        self.assertRegex(self.get_operstate(link, show_status, setup_state), expected)
+
+    def wait_address(self, link, address_regex, scope='global', ipv='', timeout_sec=100):
+        for i in range(timeout_sec):
+            if i > 0:
+                time.sleep(1)
+            output = subprocess.check_output(['ip', ipv, 'address', 'show', 'dev', link, 'scope', scope], universal_newlines=True).rstrip()
+            if re.search(address_regex, output):
+                break
+        else:
+            self.assertRegex(output, address_regex)
+
 
 class NetworkdNetDevTests(unittest.TestCase, Utilities):
 
@@ -255,6 +307,7 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         'isataptun99',
         'macvlan99',
         'macvtap99',
+        'nlmon99',
         'sittun96',
         'sittun97',
         'sittun98',
@@ -272,6 +325,7 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         'vtitun97',
         'vtitun98',
         'vtitun99',
+        'vxcan99',
         'vxlan99',
         'wg98',
         'wg99']
@@ -324,6 +378,7 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         '25-macsec.key',
         '25-macsec.netdev',
         '25-macsec.network',
+        '25-nlmon.netdev',
         '25-sit-tunnel-local-any.netdev',
         '25-sit-tunnel-remote-any.netdev',
         '25-sit-tunnel.netdev',
@@ -341,6 +396,7 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         '25-vti-tunnel-local-any.netdev',
         '25-vti-tunnel-remote-any.netdev',
         '25-vti-tunnel.netdev',
+        '25-vxcan.netdev',
         '25-vxlan.netdev',
         '25-wireguard-23-peers.netdev',
         '25-wireguard-23-peers.network',
@@ -363,50 +419,57 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         'macsec.network',
         'macvlan.network',
         'macvtap.network',
+        'netdev-link-local-addressing-yes.network',
         'sit.network',
         'vti6.network',
         'vti.network',
         'vxlan-test1.network',
         'vxlan.network']
 
+    fou_ports = [
+        '55555',
+        '55556']
+
     def setUp(self):
-        self.link_remove(self.links)
+        self.remove_fou_ports(self.fou_ports)
+        self.remove_links(self.links)
 
     def tearDown(self):
-        self.link_remove(self.links)
+        self.remove_fou_ports(self.fou_ports)
+        self.remove_links(self.links)
         self.remove_unit_from_networkd_path(self.units)
 
-    def test_dropin(self):
+    def test_dropin_and_networkctl_glob(self):
         self.copy_unit_to_networkd_unit_path('10-dropin-test.netdev', '15-name-conflict-test.netdev')
-        self.start_networkd()
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('dropin-test'))
+        self.wait_online(['dropin-test:off'])
 
         # This also tests NetDev.Name= conflict and basic networkctl functionalities
 
-        output = subprocess.check_output(['ip', 'link', 'show', 'dropin-test']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'link', 'show', 'dropin-test'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '00:50:56:c0:00:28')
 
-        output = subprocess.check_output(['networkctl', 'list']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['list'], universal_newlines=True, env=env).rstrip()
         self.assertRegex(output, '1 lo ')
         self.assertRegex(output, 'dropin-test')
 
-        output = subprocess.check_output(['networkctl', 'list', 'dropin-test']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['list', 'dropin-test'], universal_newlines=True, env=env).rstrip()
         self.assertNotRegex(output, '1 lo ')
         self.assertRegex(output, 'dropin-test')
 
-        output = subprocess.check_output(['networkctl', 'list', 'dropin-*']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['list', 'dropin-*'], universal_newlines=True, env=env).rstrip()
         self.assertNotRegex(output, '1 lo ')
         self.assertRegex(output, 'dropin-test')
 
-        output = subprocess.check_output(['networkctl', 'status', 'dropin-*']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'dropin-*'], universal_newlines=True, env=env).rstrip()
         self.assertNotRegex(output, '1: lo ')
         self.assertRegex(output, 'dropin-test')
 
-        ret = subprocess.run(['ethtool', '--driver', 'dropin-test'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        print(ret.stdout.rstrip().decode('utf-8'))
-        if ret.returncode == 0 and re.search('driver: dummy', ret.stdout.rstrip().decode('utf-8')) != None:
+        ret = subprocess.run(['ethtool', '--driver', 'dropin-test'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+        print(ret.stdout.rstrip())
+        if ret.returncode == 0 and re.search('driver: dummy', ret.stdout.rstrip()) != None:
             self.assertRegex(output, 'Driver: dummy')
         else:
             print('ethtool does not support driver field at least for dummy interfaces, skipping test for Driver field of networkctl.')
@@ -416,37 +479,31 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
 
         self.wait_online(['bridge99', 'test1:degraded'], bool_any=True)
-        self.assertTrue(self.link_exits('bridge99'))
-        self.assertTrue(self.link_exits('test1'))
 
-        output = subprocess.check_output(['networkctl', 'status', 'bridge99']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: (?:off|no-carrier) \(configuring\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: degraded \(configured\)')
+        self.check_operstate('bridge99', '(?:off|no-carrier)', setup_state='configuring')
+        self.check_operstate('test1', 'degraded')
 
     def test_bridge(self):
         self.copy_unit_to_networkd_unit_path('25-bridge.netdev')
-        self.start_networkd()
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('bridge99'))
+        self.wait_online(['bridge99:off'])
 
-        self.assertEqual('900', self.read_link_attr('bridge99', 'bridge', 'hello_time'))
-        self.assertEqual('900', self.read_link_attr('bridge99', 'bridge', 'max_age'))
-        self.assertEqual('900', self.read_link_attr('bridge99', 'bridge','forward_delay'))
-        self.assertEqual('900', self.read_link_attr('bridge99', 'bridge','ageing_time'))
-        self.assertEqual('9',   self.read_link_attr('bridge99', 'bridge','priority'))
-        self.assertEqual('1',   self.read_link_attr('bridge99', 'bridge','multicast_querier'))
-        self.assertEqual('1',   self.read_link_attr('bridge99', 'bridge','multicast_snooping'))
-        self.assertEqual('1',   self.read_link_attr('bridge99', 'bridge','stp_state'))
+        tick = os.sysconf('SC_CLK_TCK')
+        self.assertEqual(9, round(float(self.read_link_attr('bridge99', 'bridge', 'hello_time')) / tick))
+        self.assertEqual(9, round(float(self.read_link_attr('bridge99', 'bridge', 'max_age')) / tick))
+        self.assertEqual(9, round(float(self.read_link_attr('bridge99', 'bridge','forward_delay')) / tick))
+        self.assertEqual(9, round(float(self.read_link_attr('bridge99', 'bridge','ageing_time')) / tick))
+        self.assertEqual(9,         int(self.read_link_attr('bridge99', 'bridge','priority')))
+        self.assertEqual(1,         int(self.read_link_attr('bridge99', 'bridge','multicast_querier')))
+        self.assertEqual(1,         int(self.read_link_attr('bridge99', 'bridge','multicast_snooping')))
+        self.assertEqual(1,         int(self.read_link_attr('bridge99', 'bridge','stp_state')))
 
     def test_bond(self):
-        self.copy_unit_to_networkd_unit_path('25-bond.netdev')
-        self.start_networkd()
+        self.copy_unit_to_networkd_unit_path('25-bond.netdev', '25-bond-balanced-tlb.netdev')
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('bond99'))
+        self.wait_online(['bond99:off', 'bond98:off'])
 
         self.assertEqual('802.3ad 4',         self.read_link_attr('bond99', 'bonding', 'mode'))
         self.assertEqual('layer3+4 1',        self.read_link_attr('bond99', 'bonding', 'xmit_hash_policy'))
@@ -460,28 +517,21 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.assertEqual('811',               self.read_link_attr('bond99', 'bonding', 'ad_user_port_key'))
         self.assertEqual('00:11:22:33:44:55', self.read_link_attr('bond99', 'bonding', 'ad_actor_system'))
 
-    def test_bond_balanced_tlb(self):
-        self.copy_unit_to_networkd_unit_path('25-bond-balanced-tlb.netdev')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('bond99'))
-
-        self.assertEqual('balance-tlb 5', self.read_link_attr('bond99', 'bonding', 'mode'))
-        self.assertEqual('1',             self.read_link_attr('bond99', 'bonding', 'tlb_dynamic_lb'))
+        self.assertEqual('balance-tlb 5',     self.read_link_attr('bond98', 'bonding', 'mode'))
+        self.assertEqual('1',                 self.read_link_attr('bond98', 'bonding', 'tlb_dynamic_lb'))
 
     def test_vlan(self):
         self.copy_unit_to_networkd_unit_path('21-vlan.netdev', '11-dummy.netdev',
                                              '21-vlan.network', '21-vlan-test1.network')
-        self.start_networkd()
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('test1'))
-        self.assertTrue(self.link_exits('vlan99'))
+        self.wait_online(['test1:degraded', 'vlan99:routable'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, ' mtu 2000 ')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vlan99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vlan99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, ' mtu 2000 ')
         self.assertRegex(output, 'REORDER_HDR')
@@ -490,87 +540,145 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.assertRegex(output, 'MVRP')
         self.assertRegex(output, ' id 99 ')
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet 192.168.24.5/24 brd 192.168.24.255 scope global test1')
         self.assertRegex(output, 'inet 192.168.25.5/24 brd 192.168.25.255 scope global test1')
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'vlan99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'vlan99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet 192.168.23.5/24 brd 192.168.23.255 scope global vlan99')
 
     def test_macvtap(self):
-        self.copy_unit_to_networkd_unit_path('21-macvtap.netdev', '11-dummy.netdev', 'macvtap.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('macvtap99'))
+        for mode in ['private', 'vepa', 'bridge', 'passthru']:
+            with self.subTest(mode=mode):
+                if mode != 'private':
+                    self.tearDown()
+                self.copy_unit_to_networkd_unit_path('21-macvtap.netdev', 'netdev-link-local-addressing-yes.network',
+                                                     '11-dummy.netdev', 'macvtap.network')
+                with open(os.path.join(network_unit_file_path, '21-macvtap.netdev'), mode='a') as f:
+                    f.write('[MACVTAP]\nMode=' + mode)
+                self.start_networkd(0)
+
+                self.wait_online(['macvtap99:degraded', 'test1:degraded'])
+
+                output = subprocess.check_output(['ip', '-d', 'link', 'show', 'macvtap99'], universal_newlines=True).rstrip()
+                print(output)
+                self.assertRegex(output, 'macvtap mode ' + mode + ' ')
 
     def test_macvlan(self):
-        self.copy_unit_to_networkd_unit_path('21-macvlan.netdev', '11-dummy.netdev', 'macvlan.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('test1'))
-        self.assertTrue(self.link_exits('macvlan99'))
-
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'test1']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, ' mtu 2000 ')
-
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'macvlan99']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, ' mtu 2000 ')
+        for mode in ['private', 'vepa', 'bridge', 'passthru']:
+            with self.subTest(mode=mode):
+                if mode != 'private':
+                    self.tearDown()
+                self.copy_unit_to_networkd_unit_path('21-macvlan.netdev', 'netdev-link-local-addressing-yes.network',
+                                                     '11-dummy.netdev', 'macvlan.network')
+                with open(os.path.join(network_unit_file_path, '21-macvlan.netdev'), mode='a') as f:
+                    f.write('[MACVLAN]\nMode=' + mode)
+                self.start_networkd(0)
+
+                self.wait_online(['macvlan99:degraded', 'test1:degraded'])
+
+                output = subprocess.check_output(['ip', '-d', 'link', 'show', 'test1'], universal_newlines=True).rstrip()
+                print(output)
+                self.assertRegex(output, ' mtu 2000 ')
+
+                output = subprocess.check_output(['ip', '-d', 'link', 'show', 'macvlan99'], universal_newlines=True).rstrip()
+                print(output)
+                self.assertRegex(output, ' mtu 2000 ')
+                self.assertRegex(output, 'macvlan mode ' + mode + ' ')
 
     @expectedFailureIfModuleIsNotAvailable('ipvlan')
     def test_ipvlan(self):
-        self.copy_unit_to_networkd_unit_path('25-ipvlan.netdev', '11-dummy.netdev', 'ipvlan.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('ipvlan99'))
+        for mode, flag in [['L2', 'private'], ['L3', 'vepa'], ['L3S', 'bridge']]:
+            with self.subTest(mode=mode, flag=flag):
+                if mode != 'L2':
+                    self.tearDown()
+                self.copy_unit_to_networkd_unit_path('25-ipvlan.netdev', 'netdev-link-local-addressing-yes.network',
+                                                     '11-dummy.netdev', 'ipvlan.network')
+                with open(os.path.join(network_unit_file_path, '25-ipvlan.netdev'), mode='a') as f:
+                    f.write('[IPVLAN]\nMode=' + mode + '\nFlags=' + flag)
+
+                self.start_networkd(0)
+                self.wait_online(['ipvlan99:degraded', 'test1:degraded'])
+
+                output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipvlan99'], universal_newlines=True).rstrip()
+                print(output)
+                self.assertRegex(output, 'ipvlan  *mode ' + mode.lower() + ' ' + flag)
 
     @expectedFailureIfModuleIsNotAvailable('ipvtap')
     def test_ipvtap(self):
-        self.copy_unit_to_networkd_unit_path('25-ipvtap.netdev', '11-dummy.netdev', 'ipvtap.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('ipvtap99'))
+        for mode, flag in [['L2', 'private'], ['L3', 'vepa'], ['L3S', 'bridge']]:
+            with self.subTest(mode=mode, flag=flag):
+                if mode != 'L2':
+                    self.tearDown()
+                self.copy_unit_to_networkd_unit_path('25-ipvtap.netdev', 'netdev-link-local-addressing-yes.network',
+                                                     '11-dummy.netdev', 'ipvtap.network')
+                with open(os.path.join(network_unit_file_path, '25-ipvtap.netdev'), mode='a') as f:
+                    f.write('[IPVTAP]\nMode=' + mode + '\nFlags=' + flag)
+
+                self.start_networkd(0)
+                self.wait_online(['ipvtap99:degraded', 'test1:degraded'])
+
+                output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipvtap99'], universal_newlines=True).rstrip()
+                print(output)
+                self.assertRegex(output, 'ipvtap  *mode ' + mode.lower() + ' ' + flag)
 
     def test_veth(self):
-        self.copy_unit_to_networkd_unit_path('25-veth.netdev')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
+        self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'netdev-link-local-addressing-yes.network')
+        self.start_networkd(0)
 
-    def test_dummy(self):
-        self.copy_unit_to_networkd_unit_path('11-dummy.netdev')
-        self.start_networkd()
+        self.wait_online(['veth99:degraded', 'veth-peer:degraded'])
 
-        self.assertTrue(self.link_exits('test1'))
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'veth99'], universal_newlines=True).rstrip()
+        print(output)
+        self.assertRegex(output, 'link/ether 12:34:56:78:9a:bc')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'veth-peer'], universal_newlines=True).rstrip()
+        print(output)
+        self.assertRegex(output, 'link/ether 12:34:56:78:9a:bd')
 
     def test_tun(self):
         self.copy_unit_to_networkd_unit_path('25-tun.netdev')
-        self.start_networkd()
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('tun99'))
+        self.wait_online(['tun99:off'])
+
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'tun99'], universal_newlines=True).rstrip()
+        print(output)
+        # Old ip command does not support IFF_ flags
+        self.assertRegex(output, 'tun (?:type tun pi on vnet_hdr on multi_queue|addrgenmode) ')
 
     def test_tap(self):
         self.copy_unit_to_networkd_unit_path('25-tap.netdev')
-        self.start_networkd()
+        self.start_networkd(0)
+
+        self.wait_online(['tap99:off'])
 
-        self.assertTrue(self.link_exits('tap99'))
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'tap99'], universal_newlines=True).rstrip()
+        print(output)
+        # Old ip command does not support IFF_ flags
+        self.assertRegex(output, 'tun (?:type tap pi on vnet_hdr on multi_queue|addrgenmode) ')
 
     @expectedFailureIfModuleIsNotAvailable('vrf')
     def test_vrf(self):
-        self.copy_unit_to_networkd_unit_path('25-vrf.netdev')
-        self.start_networkd()
+        self.copy_unit_to_networkd_unit_path('25-vrf.netdev', 'netdev-link-local-addressing-yes.network')
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('vrf99'))
+        self.wait_online(['vrf99:carrier'])
 
     @expectedFailureIfModuleIsNotAvailable('vcan')
     def test_vcan(self):
-        self.copy_unit_to_networkd_unit_path('25-vcan.netdev')
-        self.start_networkd()
+        self.copy_unit_to_networkd_unit_path('25-vcan.netdev', 'netdev-link-local-addressing-yes.network')
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('vcan99'))
+        self.wait_online(['vcan99:carrier'])
+
+    @expectedFailureIfModuleIsNotAvailable('vxcan')
+    def test_vxcan(self):
+        self.copy_unit_to_networkd_unit_path('25-vxcan.netdev', 'netdev-link-local-addressing-yes.network')
+        self.start_networkd(0)
+
+        self.wait_online(['vxcan99:carrier', 'vxcan-peer:carrier'])
 
     @expectedFailureIfModuleIsNotAvailable('wireguard')
     def test_wireguard(self):
@@ -580,39 +688,36 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['wg99:carrier', 'wg98:routable'])
 
-        self.assertTrue(self.link_exits('wg99'))
-        self.assertTrue(self.link_exits('wg98'))
-
         if shutil.which('wg'):
             subprocess.call('wg')
 
-            output = subprocess.check_output(['wg', 'show', 'wg99', 'listen-port']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['wg', 'show', 'wg99', 'listen-port'], universal_newlines=True).rstrip()
             self.assertRegex(output, '51820')
-            output = subprocess.check_output(['wg', 'show', 'wg99', 'fwmark']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['wg', 'show', 'wg99', 'fwmark'], universal_newlines=True).rstrip()
             self.assertRegex(output, '0x4d2')
-            output = subprocess.check_output(['wg', 'show', 'wg99', 'allowed-ips']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['wg', 'show', 'wg99', 'allowed-ips'], universal_newlines=True).rstrip()
             self.assertRegex(output, 'RDf\+LSpeEre7YEIKaxg\+wbpsNV7du\+ktR99uBEtIiCA=\t192.168.26.0/24 fd31:bf08:57cb::/48')
             self.assertRegex(output, 'lsDtM3AbjxNlauRKzHEPfgS1Zp7cp/VX5Use/P4PQSc=\tfdbc:bae2:7871:e1fe:793:8636::/96 fdbc:bae2:7871:500:e1fe:793:8636:dad1/128')
-            output = subprocess.check_output(['wg', 'show', 'wg99', 'persistent-keepalive']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['wg', 'show', 'wg99', 'persistent-keepalive'], universal_newlines=True).rstrip()
             self.assertRegex(output, 'RDf\+LSpeEre7YEIKaxg\+wbpsNV7du\+ktR99uBEtIiCA=\t20')
-            output = subprocess.check_output(['wg', 'show', 'wg99', 'endpoints']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['wg', 'show', 'wg99', 'endpoints'], universal_newlines=True).rstrip()
             self.assertRegex(output, 'RDf\+LSpeEre7YEIKaxg\+wbpsNV7du\+ktR99uBEtIiCA=\t192.168.27.3:51820')
-            output = subprocess.check_output(['wg', 'show', 'wg99', 'private-key']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['wg', 'show', 'wg99', 'private-key'], universal_newlines=True).rstrip()
             self.assertRegex(output, 'EEGlnEPYJV//kbvvIqxKkQwOiS\+UENyPncC4bF46ong=')
-            output = subprocess.check_output(['wg', 'show', 'wg99', 'preshared-keys']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['wg', 'show', 'wg99', 'preshared-keys'], universal_newlines=True).rstrip()
             self.assertRegex(output, 'RDf\+LSpeEre7YEIKaxg\+wbpsNV7du\+ktR99uBEtIiCA=  IIWIV17wutHv7t4cR6pOT91z6NSz/T8Arh0yaywhw3M=')
             self.assertRegex(output, 'lsDtM3AbjxNlauRKzHEPfgS1Zp7cp/VX5Use/P4PQSc=     cPLOy1YUrEI0EMMIycPJmOo0aTu3RZnw8bL5meVD6m0=')
 
-            output = subprocess.check_output(['wg', 'show', 'wg98', 'private-key']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['wg', 'show', 'wg98', 'private-key'], universal_newlines=True).rstrip()
             self.assertRegex(output, 'CJQUtcS9emY2fLYqDlpSZiE/QJyHkPWr\+WHtZLZ90FU=')
 
     def test_geneve(self):
-        self.copy_unit_to_networkd_unit_path('25-geneve.netdev')
-        self.start_networkd()
+        self.copy_unit_to_networkd_unit_path('25-geneve.netdev', 'netdev-link-local-addressing-yes.network')
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('geneve99'))
+        self.wait_online(['geneve99:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'geneve99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'geneve99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '192.168.22.1')
         self.assertRegex(output, '6082')
@@ -627,13 +732,13 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['ipiptun99:routable', 'ipiptun98:routable', 'ipiptun97:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipiptun99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipiptun99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ipip (?:ipip |)remote 192.169.224.239 local 192.168.223.238 dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipiptun98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipiptun98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ipip (?:ipip |)remote 192.169.224.239 local any dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipiptun97']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipiptun97'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ipip (?:ipip |)remote any local 192.168.223.238 dev dummy98')
 
@@ -645,21 +750,21 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['gretun99:routable', 'gretun98:routable', 'gretun97:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretun99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretun99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'gre remote 10.65.223.239 local 10.65.223.238 dev dummy98')
         self.assertRegex(output, 'ikey 1.2.3.103')
         self.assertRegex(output, 'okey 1.2.4.103')
         self.assertRegex(output, 'iseq')
         self.assertRegex(output, 'oseq')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretun98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretun98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'gre remote 10.65.223.239 local any dev dummy98')
         self.assertRegex(output, 'ikey 0.0.0.104')
         self.assertRegex(output, 'okey 0.0.0.104')
         self.assertNotRegex(output, 'iseq')
         self.assertNotRegex(output, 'oseq')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretun97']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretun97'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'gre remote any local 10.65.223.238 dev dummy98')
         self.assertRegex(output, 'ikey 0.0.0.105')
@@ -667,7 +772,6 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.assertNotRegex(output, 'iseq')
         self.assertNotRegex(output, 'oseq')
 
-    @expectedFailureIf_ip6gre_do_not_support_ipv6ll()
     def test_ip6gre_tunnel(self):
         self.copy_unit_to_networkd_unit_path('12-dummy.netdev', 'ip6gretun.network',
                                              '25-ip6gre-tunnel.netdev', '25-tunnel.network',
@@ -675,24 +779,23 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
                                              '25-ip6gre-tunnel-remote-any.netdev', '25-tunnel-remote-any.network')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
-        self.assertTrue(self.link_exits('ip6gretun99'))
-        self.assertTrue(self.link_exits('ip6gretun98'))
-        self.assertTrue(self.link_exits('ip6gretun97'))
+        # Old kernels seem not to support IPv6LL address on ip6gre tunnel, So please do not use wait_online() here.
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretun99']).rstrip().decode('utf-8')
+        self.check_link_exists('dummy98')
+        self.check_link_exists('ip6gretun99')
+        self.check_link_exists('ip6gretun98')
+        self.check_link_exists('ip6gretun97')
+
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretun99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ip6gre remote 2001:473:fece:cafe::5179 local 2a00:ffde:4567:edde::4987 dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretun98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretun98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ip6gre remote 2001:473:fece:cafe::5179 local any dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretun97']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretun97'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ip6gre remote any local 2a00:ffde:4567:edde::4987 dev dummy98')
 
-        # Old kernels may not support IPv6LL address on ip6gre tunnel, and the following test may fails.
-        self.wait_online(['ip6gretun99:routable', 'ip6gretun98:routable', 'ip6gretun97:routable', 'dummy98:degraded'])
-
     def test_gretap_tunnel(self):
         self.copy_unit_to_networkd_unit_path('12-dummy.netdev', 'gretap.network',
                                              '25-gretap-tunnel.netdev', '25-tunnel.network',
@@ -700,14 +803,14 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['gretap99:routable', 'gretap98:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretap99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretap99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'gretap remote 10.65.223.239 local 10.65.223.238 dev dummy98')
         self.assertRegex(output, 'ikey 0.0.0.106')
         self.assertRegex(output, 'okey 0.0.0.106')
         self.assertRegex(output, 'iseq')
         self.assertRegex(output, 'oseq')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretap98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretap98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'gretap remote 10.65.223.239 local any dev dummy98')
         self.assertRegex(output, 'ikey 0.0.0.107')
@@ -722,10 +825,10 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['ip6gretap99:routable', 'ip6gretap98:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretap99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretap99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ip6gretap remote 2001:473:fece:cafe::5179 local 2a00:ffde:4567:edde::4987 dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretap98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6gretap98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ip6gretap remote 2001:473:fece:cafe::5179 local any dev dummy98')
 
@@ -737,13 +840,13 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['vtitun99:routable', 'vtitun98:routable', 'vtitun97:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vtitun99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vtitun99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'vti remote 10.65.223.239 local 10.65.223.238 dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vtitun98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vtitun98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'vti remote 10.65.223.239 local any dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vtitun97']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vtitun97'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'vti remote any local 10.65.223.238 dev dummy98')
 
@@ -755,13 +858,13 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['vti6tun99:routable', 'vti6tun98:routable', 'vti6tun97:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vti6tun99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vti6tun99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'vti6 remote 2001:473:fece:cafe::5179 local 2a00:ffde:4567:edde::4987 dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vti6tun98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vti6tun98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'vti6 remote 2001:473:fece:cafe::5179 local (?:any|::) dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vti6tun97']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vti6tun97'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'vti6 remote (?:any|::) local 2a00:ffde:4567:edde::4987 dev dummy98')
 
@@ -773,13 +876,13 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['ip6tnl99:routable', 'ip6tnl98:routable', 'ip6tnl97:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6tnl99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6tnl99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ip6tnl ip6ip6 remote 2001:473:fece:cafe::5179 local 2a00:ffde:4567:edde::4987 dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6tnl98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6tnl98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ip6tnl ip6ip6 remote 2001:473:fece:cafe::5179 local (?:any|::) dev dummy98')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6tnl97']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ip6tnl97'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'ip6tnl ip6ip6 remote (?:any|::) local 2a00:ffde:4567:edde::4987 dev dummy98')
 
@@ -791,13 +894,13 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['sittun99:routable', 'sittun98:routable', 'sittun97:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "sit (?:ip6ip |)remote 10.65.223.239 local 10.65.223.238 dev dummy98")
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "sit (?:ip6ip |)remote 10.65.223.239 local any dev dummy98")
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun97']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun97'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "sit (?:ip6ip |)remote any local 10.65.223.238 dev dummy98")
 
@@ -807,10 +910,7 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['isataptun99:routable', 'dummy98:degraded'])
 
-        self.assertTrue(self.link_exits('dummy98'))
-        self.assertTrue(self.link_exits('isataptun99'))
-
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'isataptun99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'isataptun99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "isatap ")
 
@@ -820,7 +920,7 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['sittun99:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '6rd-prefix 2602::/24')
 
@@ -832,14 +932,14 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['erspan99:routable', 'erspan98:routable', 'dummy98:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'erspan99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'erspan99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'erspan remote 172.16.1.100 local 172.16.1.200')
         self.assertRegex(output, 'ikey 0.0.0.101')
         self.assertRegex(output, 'okey 0.0.0.101')
         self.assertRegex(output, 'iseq')
         self.assertRegex(output, 'oseq')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'erspan98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'erspan98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'erspan remote 172.16.1.100 local any')
         self.assertRegex(output, '102')
@@ -849,10 +949,10 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.assertRegex(output, 'oseq')
 
     def test_tunnel_independent(self):
-        self.copy_unit_to_networkd_unit_path('25-ipip-tunnel-independent.netdev')
-        self.start_networkd()
+        self.copy_unit_to_networkd_unit_path('25-ipip-tunnel-independent.netdev', 'netdev-link-local-addressing-yes.network')
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('ipiptun99'))
+        self.wait_online(['ipiptun99:carrier'])
 
     @expectedFailureIfModuleIsNotAvailable('fou')
     def test_fou(self):
@@ -863,34 +963,28 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('25-fou-ipproto-ipip.netdev', '25-fou-ipproto-gre.netdev',
                                              '25-fou-ipip.netdev', '25-fou-sit.netdev',
                                              '25-fou-gre.netdev', '25-fou-gretap.netdev')
-        self.start_networkd()
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('ipiptun96'))
-        self.assertTrue(self.link_exits('sittun96'))
-        self.assertTrue(self.link_exits('gretun96'))
-        self.assertTrue(self.link_exits('gretap96'))
+        self.wait_online(['ipiptun96:off', 'sittun96:off', 'gretun96:off', 'gretap96:off'])
 
-        output = subprocess.check_output(['ip', 'fou', 'show']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'fou', 'show'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'port 55555 ipproto 4')
         self.assertRegex(output, 'port 55556 ipproto 47')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipiptun96']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'ipiptun96'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'encap fou encap-sport auto encap-dport 55555')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun96']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'sittun96'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'encap fou encap-sport auto encap-dport 55555')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretun96']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretun96'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'encap fou encap-sport 1001 encap-dport 55556')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretap96']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'gretap96'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'encap fou encap-sport auto encap-dport 55556')
 
-        subprocess.call(['ip', 'fou', 'del', 'port', '55555'])
-        subprocess.call(['ip', 'fou', 'del', 'port', '55556'])
-
     def test_vxlan(self):
         self.copy_unit_to_networkd_unit_path('25-vxlan.netdev', 'vxlan.network',
                                              '11-dummy.netdev', 'vxlan-test1.network')
@@ -898,7 +992,7 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
 
         self.wait_online(['test1:degraded', 'vxlan99:degraded'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vxlan99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'vxlan99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '999')
         self.assertRegex(output, '5555')
@@ -911,7 +1005,7 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.assertRegex(output, 'remcsumrx')
         self.assertRegex(output, 'gbp')
 
-        output = subprocess.check_output(['bridge', 'fdb', 'show', 'dev', 'vxlan99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['bridge', 'fdb', 'show', 'dev', 'vxlan99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '00:11:22:33:44:55 dst 10.0.0.5 self permanent')
         self.assertRegex(output, '00:11:22:33:44:66 dst 10.0.0.6 self permanent')
@@ -924,13 +1018,13 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
 
         self.wait_online(['dummy98:degraded', 'macsec99:routable'])
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'macsec99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'macsec99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'macsec99@dummy98')
         self.assertRegex(output, 'macsec sci [0-9a-f]*000b')
         self.assertRegex(output, 'encrypt on')
 
-        output = subprocess.check_output(['ip', 'macsec', 'show', 'macsec99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'macsec', 'show', 'macsec99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'encrypt on')
         self.assertRegex(output, 'TXSC: [0-9a-f]*000b on SA 1')
@@ -945,6 +1039,11 @@ class NetworkdNetDevTests(unittest.TestCase, Utilities):
         self.assertRegex(output, 'RXSC: 8c16456c83a90002, state on')
         self.assertRegex(output, '0: PN [0-9]*, state off, key 02030400000000000000000000000000')
 
+    def test_nlmon(self):
+        self.copy_unit_to_networkd_unit_path('25-nlmon.netdev', 'netdev-link-local-addressing-yes.network')
+        self.start_networkd()
+
+        self.wait_online(['nlmon99:carrier'])
 
 class NetworkdL2TPTests(unittest.TestCase, Utilities):
 
@@ -965,23 +1064,21 @@ class NetworkdL2TPTests(unittest.TestCase, Utilities):
 
     def setUp(self):
         self.l2tp_tunnel_remove(self.l2tp_tunnel_ids)
-        self.link_remove(self.links)
+        self.remove_links(self.links)
 
     def tearDown(self):
         self.l2tp_tunnel_remove(self.l2tp_tunnel_ids)
-        self.link_remove(self.links)
+        self.remove_links(self.links)
         self.remove_unit_from_networkd_path(self.units)
 
     @expectedFailureIfModuleIsNotAvailable('l2tp_eth')
     def test_l2tp_udp(self):
         self.copy_unit_to_networkd_unit_path('11-dummy.netdev', '25-l2tp-dummy.network', '25-l2tp-udp.netdev')
-        self.start_networkd()
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('test1'))
-        self.assertTrue(self.link_exits('l2tp-ses1'))
-        self.assertTrue(self.link_exits('l2tp-ses2'))
+        self.wait_online(['test1:routable', 'l2tp-ses1:off', 'l2tp-ses2:off'])
 
-        output = subprocess.check_output(['ip', 'l2tp', 'show', 'tunnel', 'tunnel_id', '10']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'l2tp', 'show', 'tunnel', 'tunnel_id', '10'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "Tunnel 10, encap UDP")
         self.assertRegex(output, "From 192.168.30.100 to 192.168.30.101")
@@ -989,13 +1086,13 @@ class NetworkdL2TPTests(unittest.TestCase, Utilities):
         self.assertRegex(output, "UDP source / dest ports: 3000/4000")
         self.assertRegex(output, "UDP checksum: enabled")
 
-        output = subprocess.check_output(['ip', 'l2tp', 'show', 'session', 'tid', '10', 'session_id', '15']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'l2tp', 'show', 'session', 'tid', '10', 'session_id', '15'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "Session 15 in tunnel 10")
         self.assertRegex(output, "Peer session 16, tunnel 11")
         self.assertRegex(output, "interface name: l2tp-ses1")
 
-        output = subprocess.check_output(['ip', 'l2tp', 'show', 'session', 'tid', '10', 'session_id', '17']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'l2tp', 'show', 'session', 'tid', '10', 'session_id', '17'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "Session 17 in tunnel 10")
         self.assertRegex(output, "Peer session 18, tunnel 11")
@@ -1004,31 +1101,29 @@ class NetworkdL2TPTests(unittest.TestCase, Utilities):
     @expectedFailureIfModuleIsNotAvailable('l2tp_ip')
     def test_l2tp_ip(self):
         self.copy_unit_to_networkd_unit_path('11-dummy.netdev', '25-l2tp-dummy.network', '25-l2tp-ip.netdev')
-        self.start_networkd()
+        self.start_networkd(0)
 
-        self.assertTrue(self.link_exits('test1'))
-        self.assertTrue(self.link_exits('l2tp-ses3'))
-        self.assertTrue(self.link_exits('l2tp-ses4'))
+        self.wait_online(['test1:routable', 'l2tp-ses3:off', 'l2tp-ses4:off'])
 
-        output = subprocess.check_output(['ip', 'l2tp', 'show', 'tunnel', 'tunnel_id', '10']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'l2tp', 'show', 'tunnel', 'tunnel_id', '10'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "Tunnel 10, encap IP")
         self.assertRegex(output, "From 192.168.30.100 to 192.168.30.101")
         self.assertRegex(output, "Peer tunnel 12")
 
-        output = subprocess.check_output(['ip', 'l2tp', 'show', 'session', 'tid', '10', 'session_id', '25']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'l2tp', 'show', 'session', 'tid', '10', 'session_id', '25'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "Session 25 in tunnel 10")
         self.assertRegex(output, "Peer session 26, tunnel 12")
         self.assertRegex(output, "interface name: l2tp-ses3")
 
-        output = subprocess.check_output(['ip', 'l2tp', 'show', 'session', 'tid', '10', 'session_id', '27']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'l2tp', 'show', 'session', 'tid', '10', 'session_id', '27'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, "Session 27 in tunnel 10")
         self.assertRegex(output, "Peer session 28, tunnel 12")
         self.assertRegex(output, "interface name: l2tp-ses4")
 
-class NetworkdNetWorkTests(unittest.TestCase, Utilities):
+class NetworkdNetworkTests(unittest.TestCase, Utilities):
     links = [
         'bond199',
         'dummy98',
@@ -1039,6 +1134,7 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         '11-dummy.netdev',
         '12-dummy.netdev',
         '23-active-slave.network',
+        '24-search-domain.network',
         '25-address-link-section.network',
         '25-address-preferred-lifetime-zero-ipv6.network',
         '25-address-static.network',
@@ -1059,11 +1155,18 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         'routing-policy-rule-dummy98.network',
         'routing-policy-rule-test1.network']
 
+    routing_policy_rule_tables = ['7', '8']
+    routes = [['blackhole', '202.54.1.2'], ['unreachable', '202.54.1.3'], ['prohibit', '202.54.1.4']]
+
     def setUp(self):
-        self.link_remove(self.links)
+        self.remove_routing_policy_rule_tables(self.routing_policy_rule_tables)
+        self.remove_routes(self.routes)
+        self.remove_links(self.links)
 
     def tearDown(self):
-        self.link_remove(self.links)
+        self.remove_routing_policy_rule_tables(self.routing_policy_rule_tables)
+        self.remove_routes(self.routes)
+        self.remove_links(self.links)
         self.remove_unit_from_networkd_path(self.units)
 
     def test_address_static(self):
@@ -1072,7 +1175,7 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
 
         self.wait_online(['dummy98:routable'])
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet 10.1.2.3/16 brd 10.1.255.255 scope global dummy98')
         self.assertRegex(output, 'inet 10.1.2.4/16 brd 10.1.255.255 scope global secondary dummy98')
@@ -1082,19 +1185,19 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.assertNotRegex(output, '10.10.0.1/16')
         self.assertNotRegex(output, '10.10.0.2/16')
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98', 'label', '32']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98', 'label', '32'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet 10.3.2.3/16 brd 10.3.255.255 scope global 32')
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98', 'label', '33']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98', 'label', '33'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet 10.4.2.3 peer 10.4.2.4/16 scope global 33')
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98', 'label', '34']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98', 'label', '34'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet 192.168.[0-9]*.1/24 brd 192.168.[0-9]*.255 scope global 34')
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98', 'label', '35']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'dummy98', 'label', '35'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet 172.[0-9]*.0.1/16 brd 172.[0-9]*.255.255 scope global 35')
 
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet6 2001:db8:0:f101::15/64 scope global')
         self.assertRegex(output, 'inet6 2001:db8:0:f101::16/64 scope global')
@@ -1107,13 +1210,11 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('25-address-preferred-lifetime-zero-ipv6.network', '12-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
+        self.check_link_exists('dummy98')
 
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: routable \(configuring\)')
+        self.check_operstate('dummy98', 'routable', setup_state='configuring')
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet 10.2.3.4/16 brd 10.2.255.255 scope link deprecated dummy98')
         self.assertRegex(output, 'inet6 2001:db8:0:f101::1/64 scope global')
@@ -1122,9 +1223,9 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('configure-without-carrier.network', '11-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('test1'))
+        self.check_link_exists('test1')
 
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'test1'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, '192.168.0.15')
         self.assertRegex(output, '192.168.0.1')
@@ -1133,13 +1234,11 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
     def test_routing_policy_rule(self):
         self.copy_unit_to_networkd_unit_path('routing-policy-rule-test1.network', '11-dummy.netdev')
 
-        subprocess.call(['ip', 'rule', 'del', 'table', '7'])
-
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('test1'))
+        self.check_link_exists('test1')
 
-        output = subprocess.check_output(['ip', 'rule']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'rule'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '111')
         self.assertRegex(output, 'from 192.168.100.18')
@@ -1148,44 +1247,34 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.assertRegex(output, 'oif test1')
         self.assertRegex(output, 'lookup 7')
 
-        subprocess.call(['ip', 'rule', 'del', 'table', '7'])
-
     def test_routing_policy_rule_issue_11280(self):
         self.copy_unit_to_networkd_unit_path('routing-policy-rule-test1.network', '11-dummy.netdev',
                                              'routing-policy-rule-dummy98.network', '12-dummy.netdev')
 
-        subprocess.call(['ip', 'rule', 'del', 'table', '7'])
-        subprocess.call(['ip', 'rule', 'del', 'table', '8'])
-
         for trial in range(3):
             # Remove state files only first time
             self.start_networkd(remove_state_files=(trial == 0))
 
-            self.assertTrue(self.link_exits('test1'))
-            self.assertTrue(self.link_exits('dummy98'))
+            self.check_link_exists('test1')
+            self.check_link_exists('dummy98')
 
-            output = subprocess.check_output(['ip', 'rule', 'list', 'table', '7']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['ip', 'rule', 'list', 'table', '7'], universal_newlines=True).rstrip()
             print(output)
             self.assertRegex(output, '111:     from 192.168.100.18 tos (?:0x08|throughput) iif test1 oif test1 lookup 7')
 
-            output = subprocess.check_output(['ip', 'rule', 'list', 'table', '8']).rstrip().decode('utf-8')
+            output = subprocess.check_output(['ip', 'rule', 'list', 'table', '8'], universal_newlines=True).rstrip()
             print(output)
             self.assertRegex(output, '112:     from 192.168.101.18 tos (?:0x08|throughput) iif dummy98 oif dummy98 lookup 8')
 
-        subprocess.call(['ip', 'rule', 'del', 'table', '7'])
-        subprocess.call(['ip', 'rule', 'del', 'table', '8'])
-
     @expectedFailureIfRoutingPolicyPortRangeIsNotAvailable()
     def test_routing_policy_rule_port_range(self):
         self.copy_unit_to_networkd_unit_path('25-fibrule-port-range.network', '11-dummy.netdev')
 
-        subprocess.call(['ip', 'rule', 'del', 'table', '7'])
-
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('test1'))
+        self.check_link_exists('test1')
 
-        output = subprocess.check_output(['ip', 'rule']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'rule'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '111')
         self.assertRegex(output, 'from 192.168.100.18')
@@ -1194,42 +1283,36 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.assertRegex(output, 'tcp')
         self.assertRegex(output, 'lookup 7')
 
-        subprocess.call(['ip', 'rule', 'del', 'table', '7'])
-
     @expectedFailureIfRoutingPolicyIPProtoIsNotAvailable()
     def test_routing_policy_rule_invert(self):
         self.copy_unit_to_networkd_unit_path('25-fibrule-invert.network', '11-dummy.netdev')
 
-        subprocess.call(['ip', 'rule', 'del', 'table', '7'])
-
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('test1'))
+        self.check_link_exists('test1')
 
-        output = subprocess.check_output(['ip', 'rule']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'rule'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '111')
         self.assertRegex(output, 'not.*?from.*?192.168.100.18')
         self.assertRegex(output, 'tcp')
         self.assertRegex(output, 'lookup 7')
 
-        subprocess.call(['ip', 'rule', 'del', 'table', '7'])
-
     def test_route_static(self):
         self.copy_unit_to_networkd_unit_path('25-route-static.network', '12-dummy.netdev')
         self.start_networkd(0)
 
         self.wait_online(['dummy98:routable'])
 
-        output = subprocess.check_output(['ip', '-6', 'route', 'show', 'dev', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'route', 'show', 'dev', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '2001:1234:5:8fff:ff:ff:ff:ff proto static')
         self.assertRegex(output, '2001:1234:5:8f63::1 proto kernel')
 
-        output = subprocess.check_output(['ip', '-6', 'route', 'show', 'dev', 'dummy98', 'default']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'route', 'show', 'dev', 'dummy98', 'default'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'default via 2001:1234:5:8fff:ff:ff:ff:ff proto static metric 1024 pref medium')
 
-        output = subprocess.check_output(['ip', '-4', 'route', 'show', 'dev', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'route', 'show', 'dev', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '149.10.124.48/28 proto kernel scope link src 149.10.124.58')
         self.assertRegex(output, '149.10.124.64 proto static scope link')
@@ -1237,27 +1320,23 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.assertRegex(output, '192.168.1.1 proto static initcwnd 20')
         self.assertRegex(output, '192.168.1.2 proto static initrwnd 30')
 
-        output = subprocess.check_output(['ip', '-4', 'route', 'show', 'dev', 'dummy98', 'default']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'route', 'show', 'dev', 'dummy98', 'default'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'default via 149.10.125.65 proto static onlink')
         self.assertRegex(output, 'default via 149.10.124.64 proto static')
         self.assertRegex(output, 'default proto static')
 
-        output = subprocess.check_output(['ip', 'route', 'show', 'type', 'blackhole']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'type', 'blackhole'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'blackhole 202.54.1.2 proto static')
 
-        output = subprocess.check_output(['ip', 'route', 'show', 'type', 'unreachable']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'type', 'unreachable'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'unreachable 202.54.1.3 proto static')
 
-        output = subprocess.check_output(['ip', 'route', 'show', 'type', 'prohibit']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'type', 'prohibit'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'prohibit 202.54.1.4 proto static')
 
-        subprocess.call(['ip', 'route', 'del', 'blackhole', '202.54.1.2'])
-        subprocess.call(['ip', 'route', 'del', 'unreachable', '202.54.1.3'])
-        subprocess.call(['ip', 'route', 'del', 'prohibit', '202.54.1.4'])
-
     def test_ip_route_ipv6_src_route(self):
         # a dummy device does not make the addresses go through tentative state, so we
         # reuse a bond from an earlier test, which does make the addresses go through
@@ -1265,10 +1344,10 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('23-active-slave.network', '25-route-ipv6-src.network', '25-bond-active-backup-slave.netdev', '12-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
-        self.assertTrue(self.link_exits('bond199'))
+        self.check_link_exists('dummy98')
+        self.check_link_exists('bond199')
 
-        output = subprocess.check_output(['ip', '-6', 'route', 'list', 'dev', 'bond199']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'route', 'list', 'dev', 'bond199'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'abcd::/16')
         self.assertRegex(output, 'src')
@@ -1278,9 +1357,9 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('25-address-link-section.network', '12-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
+        self.check_link_exists('dummy98')
 
-        output = subprocess.check_output(['ip', 'link', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'link', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '00:01:02:aa:bb:cc')
 
@@ -1288,9 +1367,9 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('25-link-section-unmanaged.network', '12-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
+        self.check_link_exists('dummy98')
 
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'dummy98'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, 'unmanaged')
 
@@ -1298,9 +1377,9 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('25-ipv6-address-label-section.network', '12-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
+        self.check_link_exists('dummy98')
 
-        output = subprocess.check_output(['ip', 'addrlabel', 'list']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'addrlabel', 'list'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '2004:da8:1::/64')
 
@@ -1308,9 +1387,9 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('25-neighbor-section.network', '12-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
+        self.check_link_exists('dummy98')
 
-        output = subprocess.check_output(['ip', 'neigh', 'list']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'neigh', 'list'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '192.168.10.1.*00:00:5e:00:02:65.*PERMANENT')
         self.assertRegex(output, '2004:da8:1::1.*00:00:5e:00:02:66.*PERMANENT')
@@ -1321,25 +1400,20 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['test1:degraded', 'dummy98:carrier'])
 
-        self.assertTrue(self.link_exits('test1'))
-        self.assertTrue(self.link_exits('dummy98'))
+        self.check_link_exists('test1')
+        self.check_link_exists('dummy98')
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet .* scope link')
         self.assertRegex(output, 'inet6 .* scope link')
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertNotRegex(output, 'inet6* .* scope link')
 
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: degraded \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: carrier \(configured\)')
+        self.check_operstate('test1', 'degraded')
+        self.check_operstate('dummy98', 'carrier')
 
         '''
         Documentation/networking/ip-sysctl.txt
@@ -1379,8 +1453,6 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['dummy98:degraded'])
 
-        self.assertTrue(self.link_exits('dummy98'))
-
         self.assertEqual(self.read_ipv6_sysctl_attr('dummy98', 'forwarding'), '1')
         self.assertEqual(self.read_ipv6_sysctl_attr('dummy98', 'use_tempaddr'), '2')
         self.assertEqual(self.read_ipv6_sysctl_attr('dummy98', 'dad_transmits'), '3')
@@ -1399,16 +1471,13 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['dummy98:routable'])
 
-        self.assertTrue(self.link_exits('dummy98'))
-
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet 10.2.3.4/16 brd 10.2.255.255 scope global dummy98')
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertEqual(output, '')
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('dummy98', 'routable')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'del', 'dummy98']), 0)
 
@@ -1419,73 +1488,76 @@ class NetworkdNetWorkTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['dummy98:routable'])
 
-        self.assertTrue(self.link_exits('dummy98'))
-
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet 10.2.3.4/16 brd 10.2.255.255 scope global dummy98')
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet6 .* scope link')
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('dummy98', 'routable')
 
     def test_bind_carrier(self):
         self.copy_unit_to_networkd_unit_path('25-bind-carrier.network', '11-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('test1'))
+        self.check_link_exists('test1')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'add', 'dummy98', 'type', 'dummy']), 0)
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'dummy98', 'up']), 0)
         time.sleep(2)
-        output = subprocess.check_output(['ip', 'address', 'show', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'UP,LOWER_UP')
         self.assertRegex(output, 'inet 192.168.10.30/24 brd 192.168.10.255 scope global test1')
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('test1', 'routable')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'add', 'dummy99', 'type', 'dummy']), 0)
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'dummy99', 'up']), 0)
         time.sleep(2)
-        output = subprocess.check_output(['ip', 'address', 'show', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'UP,LOWER_UP')
         self.assertRegex(output, 'inet 192.168.10.30/24 brd 192.168.10.255 scope global test1')
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('test1', 'routable')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'del', 'dummy98']), 0)
         time.sleep(2)
-        output = subprocess.check_output(['ip', 'address', 'show', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'UP,LOWER_UP')
         self.assertRegex(output, 'inet 192.168.10.30/24 brd 192.168.10.255 scope global test1')
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('test1', 'routable')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'del', 'dummy99']), 0)
         time.sleep(2)
-        output = subprocess.check_output(['ip', 'address', 'show', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertNotRegex(output, 'UP,LOWER_UP')
         self.assertRegex(output, 'DOWN')
         self.assertNotRegex(output, '192.168.10')
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: off \(configured\)')
+        self.check_operstate('test1', 'off')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'add', 'dummy98', 'type', 'dummy']), 0)
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'dummy98', 'up']), 0)
         time.sleep(2)
-        output = subprocess.check_output(['ip', 'address', 'show', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'UP,LOWER_UP')
         self.assertRegex(output, 'inet 192.168.10.30/24 brd 192.168.10.255 scope global test1')
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('test1', 'routable')
+
+    def test_domain(self):
+        self.copy_unit_to_networkd_unit_path('12-dummy.netdev', '24-search-domain.network')
+        self.start_networkd(0)
+        self.wait_online(['dummy98:routable'])
 
-class NetworkdNetWorkBondTests(unittest.TestCase, Utilities):
+        output = subprocess.check_output(networkctl_cmd + ['status', 'dummy98'], universal_newlines=True, env=env).rstrip()
+        print(output)
+        self.assertRegex(output, 'Address: 192.168.42.100')
+        self.assertRegex(output, 'DNS: 192.168.42.1')
+        self.assertRegex(output, 'Search Domains: one')
+
+class NetworkdBondTests(unittest.TestCase, Utilities):
     links = [
         'bond199',
         'bond99',
@@ -1505,20 +1577,20 @@ class NetworkdNetWorkBondTests(unittest.TestCase, Utilities):
         'bond-slave.network']
 
     def setUp(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
 
     def tearDown(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
         self.remove_unit_from_networkd_path(self.units)
 
     def test_bond_active_slave(self):
         self.copy_unit_to_networkd_unit_path('23-active-slave.network', '23-bond199.network', '25-bond-active-backup-slave.netdev', '12-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
-        self.assertTrue(self.link_exits('bond199'))
+        self.check_link_exists('dummy98')
+        self.check_link_exists('bond199')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'bond199']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'bond199'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'active_slave dummy98')
 
@@ -1526,10 +1598,10 @@ class NetworkdNetWorkBondTests(unittest.TestCase, Utilities):
         self.copy_unit_to_networkd_unit_path('23-primary-slave.network', '23-test1-bond199.network', '25-bond-active-backup-slave.netdev', '11-dummy.netdev')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('test1'))
-        self.assertTrue(self.link_exits('bond199'))
+        self.check_link_exists('test1')
+        self.check_link_exists('bond199')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'bond199']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'bond199'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'primary test1')
 
@@ -1538,81 +1610,60 @@ class NetworkdNetWorkBondTests(unittest.TestCase, Utilities):
                                              'bond99.network','bond-slave.network')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('bond99'))
-        self.assertTrue(self.link_exits('dummy98'))
-        self.assertTrue(self.link_exits('test1'))
+        self.check_link_exists('bond99')
+        self.check_link_exists('dummy98')
+        self.check_link_exists('test1')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'SLAVE,UP,LOWER_UP')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'SLAVE,UP,LOWER_UP')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'bond99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'bond99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'MASTER,UP,LOWER_UP')
 
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: enslaved \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: enslaved \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'bond99']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('dummy98', 'enslaved')
+        self.check_operstate('test1', 'enslaved')
+        self.check_operstate('bond99', 'routable')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'dummy98', 'down']), 0)
         time.sleep(2)
 
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: off \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: enslaved \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'bond99']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: degraded-carrier \(configured\)')
+        self.check_operstate('dummy98', 'off')
+        self.check_operstate('test1', 'enslaved')
+        self.check_operstate('bond99', 'degraded-carrier')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'dummy98', 'up']), 0)
         time.sleep(2)
 
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: enslaved \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: enslaved \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'bond99']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('dummy98', 'enslaved')
+        self.check_operstate('test1', 'enslaved')
+        self.check_operstate('bond99', 'routable')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'dummy98', 'down']), 0)
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'test1', 'down']), 0)
-        time.sleep(5)
-
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: off \(configured\)')
+        time.sleep(2)
 
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: off \(configured\)')
+        self.check_operstate('dummy98', 'off')
+        self.check_operstate('test1', 'off')
 
-        output = subprocess.check_output(['networkctl', 'status', 'bond99']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: no-carrier \(configured\)')
+        for trial in range(30):
+            if trial > 0:
+                time.sleep(1)
+            output = subprocess.check_output(['ip', 'address', 'show', 'bond99'], universal_newlines=True).rstrip()
+            print(output)
+            if self.get_operstate('bond99') == 'no-carrier':
+                break
+        else:
+            # Huh? Kernel does not recognize that all slave interfaces are down?
+            # Let's confirm that networkd's operstate is consistent with ip's result.
+            self.assertNotRegex(output, 'NO-CARRIER')
 
-class NetworkdNetWorkBridgeTests(unittest.TestCase, Utilities):
+class NetworkdBridgeTests(unittest.TestCase, Utilities):
     links = [
         'bridge99',
         'dummy98',
@@ -1628,10 +1679,10 @@ class NetworkdNetWorkBridgeTests(unittest.TestCase, Utilities):
         'bridge99.network']
 
     def setUp(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
 
     def tearDown(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
         self.remove_unit_from_networkd_path(self.units)
 
     def test_bridge_property(self):
@@ -1640,25 +1691,25 @@ class NetworkdNetWorkBridgeTests(unittest.TestCase, Utilities):
                                              'bridge99.network')
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
-        self.assertTrue(self.link_exits('test1'))
-        self.assertTrue(self.link_exits('bridge99'))
+        self.check_link_exists('dummy98')
+        self.check_link_exists('test1')
+        self.check_link_exists('bridge99')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'test1']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'test1'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'master')
         self.assertRegex(output, 'bridge')
 
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'master')
         self.assertRegex(output, 'bridge')
 
-        output = subprocess.check_output(['ip', 'addr', 'show', 'bridge99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'addr', 'show', 'bridge99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '192.168.0.15/24')
 
-        output = subprocess.check_output(['bridge', '-d', 'link', 'show', 'dummy98']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['bridge', '-d', 'link', 'show', 'dummy98'], universal_newlines=True).rstrip()
         print(output)
         self.assertEqual(self.read_bridge_port_attr('bridge99', 'dummy98', 'hairpin_mode'), '1')
         self.assertEqual(self.read_bridge_port_attr('bridge99', 'dummy98', 'path_cost'), '400')
@@ -1673,38 +1724,30 @@ class NetworkdNetWorkBridgeTests(unittest.TestCase, Utilities):
         if (os.path.exists('/sys/devices/virtual/net/bridge00/lower_dummy98/brport/multicast_to_unicast')):
             self.assertEqual(self.read_bridge_port_attr('bridge99', 'dummy98', 'multicast_to_unicast'), '1')
 
-        output = subprocess.check_output(['networkctl', 'status', 'test1']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: enslaved \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: enslaved \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'bridge99']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('test1', 'enslaved')
+        self.check_operstate('dummy98', 'enslaved')
+        self.check_operstate('bridge99', 'routable')
 
         self.assertEqual(subprocess.call(['ip', 'address', 'add', '192.168.0.16/24', 'dev', 'bridge99']), 0)
         time.sleep(1)
 
-        output = subprocess.check_output(['ip', 'addr', 'show', 'bridge99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'addr', 'show', 'bridge99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '192.168.0.16/24')
 
-        output = subprocess.check_output(['networkctl', 'status', 'bridge99']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('bridge99', 'routable')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'del', 'test1']), 0)
         time.sleep(3)
 
-        output = subprocess.check_output(['networkctl', 'status', 'bridge99']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: degraded-carrier \(configured\)')
+        self.check_operstate('bridge99', 'degraded-carrier')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'del', 'dummy98']), 0)
         time.sleep(3)
 
-        output = subprocess.check_output(['networkctl', 'status', 'bridge99']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: no-carrier \(configured\)')
+        self.check_operstate('bridge99', 'no-carrier')
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'bridge99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'bridge99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'NO-CARRIER')
         self.assertNotRegex(output, '192.168.0.15/24')
@@ -1719,9 +1762,9 @@ class NetworkdNetWorkBridgeTests(unittest.TestCase, Utilities):
 
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('dummy98'))
-        self.assertTrue(self.link_exits('test1'))
-        self.assertTrue(self.link_exits('bridge99'))
+        self.check_link_exists('dummy98')
+        self.check_link_exists('test1')
+        self.check_link_exists('bridge99')
 
         self.assertEqual(subprocess.call(['ip', 'address', 'add', '192.168.0.16/24', 'dev', 'bridge99']), 0)
         time.sleep(1)
@@ -1730,7 +1773,7 @@ class NetworkdNetWorkBridgeTests(unittest.TestCase, Utilities):
         self.assertEqual(subprocess.call(['ip', 'link', 'del', 'dummy98']), 0)
         time.sleep(3)
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'bridge99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'bridge99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'NO-CARRIER')
         self.assertRegex(output, 'inet 192.168.0.15/24 brd 192.168.0.255 scope global bridge99')
@@ -1746,7 +1789,7 @@ class NetworkdNetWorkBridgeTests(unittest.TestCase, Utilities):
 
         self.start_networkd()
 
-        self.assertTrue(self.link_exits('bridge99'))
+        self.check_link_exists('bridge99')
 
         self.assertEqual(subprocess.call(['ip', 'link', 'add', 'dummy98', 'type', 'dummy']), 0)
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'dummy98', 'up']), 0)
@@ -1763,25 +1806,25 @@ class NetworkdNetWorkBridgeTests(unittest.TestCase, Utilities):
         self.assertEqual(subprocess.call(['ip', 'link', 'add', 'dummy98', 'type', 'dummy']), 0)
         self.assertEqual(subprocess.call(['ip', 'link', 'set', 'dummy98', 'up']), 0)
 
-        time.sleep(3)
+        for trial in range(30):
+            if trial > 0:
+                time.sleep(1)
+            if self.get_operstate('bridge99') == 'routable' and self.get_operstate('dummy98') == 'enslaved':
+                break
+        else:
+            self.assertTrue(False)
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'bridge99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'bridge99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'inet 192.168.0.15/24 brd 192.168.0.255 scope global bridge99')
 
-        output = subprocess.check_output(['networkctl', 'status', 'bridge99']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: routable \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        self.assertRegex(output, 'State: enslaved \(configured\)')
-
-        output = subprocess.check_output(['ip', 'rule', 'list', 'table', '100']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'rule', 'list', 'table', '100'], universal_newlines=True).rstrip()
         print(output)
         self.assertEqual(output, '0:   from all to 8.8.8.8 lookup 100')
 
         subprocess.call(['ip', 'rule', 'del', 'table', '100'])
 
-class NetworkdNetWorkLLDPTests(unittest.TestCase, Utilities):
+class NetworkdLLDPTests(unittest.TestCase, Utilities):
     links = ['veth99']
 
     units = [
@@ -1790,24 +1833,23 @@ class NetworkdNetWorkLLDPTests(unittest.TestCase, Utilities):
         '25-veth.netdev']
 
     def setUp(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
 
     def tearDown(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
         self.remove_unit_from_networkd_path(self.units)
 
     def test_lldp(self):
         self.copy_unit_to_networkd_unit_path('23-emit-lldp.network', '24-lldp.network', '25-veth.netdev')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
+        self.start_networkd(0)
+        self.wait_online(['veth99:degraded', 'veth-peer:degraded'])
 
-        output = subprocess.check_output(['networkctl', 'lldp']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['lldp'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, 'veth-peer')
         self.assertRegex(output, 'veth99')
 
-class NetworkdNetworkRATests(unittest.TestCase, Utilities):
+class NetworkdRATests(unittest.TestCase, Utilities):
     links = ['veth99']
 
     units = [
@@ -1816,30 +1858,26 @@ class NetworkdNetworkRATests(unittest.TestCase, Utilities):
         'ipv6-prefix-veth.network']
 
     def setUp(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
 
     def tearDown(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
         self.remove_unit_from_networkd_path(self.units)
 
     def test_ipv6_prefix_delegation(self):
+        self.warn_about_firewalld()
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'ipv6-prefix.network', 'ipv6-prefix-veth.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
+        self.start_networkd(0)
+        self.wait_online(['veth99:routable', 'veth-peer:degraded'])
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, '2002:da8:1:0')
 
-class NetworkdNetworkDHCPServerTests(unittest.TestCase, Utilities):
-    links = [
-        'dummy98',
-        'veth99']
+class NetworkdDHCPServerTests(unittest.TestCase, Utilities):
+    links = ['veth99']
 
     units = [
-        '12-dummy.netdev',
-        '24-search-domain.network',
         '25-veth.netdev',
         'dhcp-client.network',
         'dhcp-client-timezone-router.network',
@@ -1847,52 +1885,39 @@ class NetworkdNetworkDHCPServerTests(unittest.TestCase, Utilities):
         'dhcp-server-timezone-router.network']
 
     def setUp(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
 
     def tearDown(self):
-        self.link_remove(self.links)
+        self.remove_links(self.links)
         self.remove_unit_from_networkd_path(self.units)
 
     def test_dhcp_server(self):
+        self.warn_about_firewalld()
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-client.network', 'dhcp-server.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
+        self.start_networkd(0)
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, '192.168.5.*')
         self.assertRegex(output, 'Gateway: 192.168.5.1')
         self.assertRegex(output, 'DNS: 192.168.5.1')
         self.assertRegex(output, 'NTP: 192.168.5.1')
 
-    def test_domain(self):
-        self.copy_unit_to_networkd_unit_path('12-dummy.netdev', '24-search-domain.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('dummy98'))
-
-        output = subprocess.check_output(['networkctl', 'status', 'dummy98']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'Address: 192.168.42.100')
-        self.assertRegex(output, 'DNS: 192.168.42.1')
-        self.assertRegex(output, 'Search Domains: one')
-
     def test_emit_router_timezone(self):
+        self.warn_about_firewalld()
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-client-timezone-router.network', 'dhcp-server-timezone-router.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
+        self.start_networkd(0)
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, 'Gateway: 192.168.5.*')
         self.assertRegex(output, '192.168.5.*')
         self.assertRegex(output, 'Europe/Berlin')
 
-class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
+class NetworkdDHCPClientTests(unittest.TestCase, Utilities):
     links = [
-        'dummy98',
         'veth99',
         'vrf99']
 
@@ -1920,15 +1945,15 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
         'static.network']
 
     def setUp(self):
-        self.link_remove(self.links)
         self.stop_dnsmasq(dnsmasq_pid_file)
+        self.remove_links(self.links)
 
     def tearDown(self):
-        self.link_remove(self.links)
-        self.remove_unit_from_networkd_path(self.units)
         self.stop_dnsmasq(dnsmasq_pid_file)
         self.remove_lease_file()
         self.remove_log_file()
+        self.remove_links(self.links)
+        self.remove_unit_from_networkd_path(self.units)
 
     def test_dhcp_client_ipv6_only(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-ipv6-only.network')
@@ -1938,25 +1963,25 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
         self.start_dnsmasq()
         self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, '2600::')
         self.assertNotRegex(output, '192.168.5')
 
         # Confirm that ipv6 token is not set in the kernel
-        output = subprocess.check_output(['ip', 'token', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'token', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'token :: dev veth99')
 
     def test_dhcp_client_ipv4_only(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-ipv4-only-ipv6-disabled.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
 
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertNotRegex(output, '2600::')
         self.assertRegex(output, '192.168.5')
@@ -1964,27 +1989,30 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
     def test_dhcp_client_ipv4_ipv6(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-ipv6-only.network',
                                              'dhcp-client-ipv4-only.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        # link become 'routable' when at least one protocol provide an valid address.
+        self.wait_address('veth99', r'inet 192.168.5.[0-9]*/24 brd 192.168.5.255', ipv='-4')
+        self.wait_address('veth99', r'inet6 2600::[0-9a-f]*/128', ipv='-6')
+
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, '2600::')
         self.assertRegex(output, '192.168.5')
 
     def test_dhcp_client_settings(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-ipv4-dhcp-settings.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
 
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
         print('## ip address show dev veth99')
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '12:34:56:78:9a:bc')
         self.assertRegex(output, '192.168.5')
@@ -1992,12 +2020,12 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
 
         # issue #8726
         print('## ip route show table main dev veth99')
-        output = subprocess.check_output(['ip', 'route', 'show', 'table', 'main', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'table', 'main', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertNotRegex(output, 'proto dhcp')
 
         print('## ip route show table 211 dev veth99')
-        output = subprocess.check_output(['ip', 'route', 'show', 'table', '211', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'table', '211', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'default via 192.168.5.1 proto dhcp')
         self.assertRegex(output, '192.168.5.0/24 via 192.168.5.5 proto dhcp')
@@ -2011,37 +2039,34 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
 
     def test_dhcp6_client_settings_rapidcommit_true(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-ipv6-only.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '12:34:56:78:9a:bc')
         self.assertTrue(self.search_words_in_dnsmasq_log('14:rapid-commit', True))
 
     def test_dhcp6_client_settings_rapidcommit_false(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-ipv6-rapid-commit.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '12:34:56:78:9a:bc')
         self.assertFalse(self.search_words_in_dnsmasq_log('14:rapid-commit', True))
 
     def test_dhcp_client_settings_anonymize(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-anonymize.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
         self.assertFalse(self.search_words_in_dnsmasq_log('VendorClassIdentifier=SusantVendorTest', True))
         self.assertFalse(self.search_words_in_dnsmasq_log('test-hostname'))
@@ -2049,50 +2074,50 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
 
     def test_dhcp_client_listen_port(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-listen-port.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq('--dhcp-alternate-port=67,5555')
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
+
+        # link become 'routable' when at least one protocol provide an valid address.
+        self.wait_address('veth99', r'inet 192.168.5.[0-9]*/24 brd 192.168.5.255', ipv='-4')
+        self.wait_address('veth99', r'inet6 2600::[0-9a-f]*/128', ipv='-6')
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '192.168.5.* dynamic')
 
     def test_dhcp_route_table_id(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-v4-server-veth-peer.network', 'dhcp-client-route-table.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['ip', 'route', 'show', 'table', '12']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'table', '12'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'veth99 proto dhcp')
         self.assertRegex(output, '192.168.5.1')
 
     def test_dhcp_route_metric(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-v4-server-veth-peer.network', 'dhcp-client-route-metric.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['ip', 'route', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'metric 24')
 
     def test_dhcp_route_criticalconnection_true(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-v4-server-veth-peer.network', 'dhcp-client-critical-connection.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, '192.168.5.*')
 
@@ -2102,25 +2127,28 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
         # Sleep for 120 sec as the dnsmasq minimum lease time can only be set to 120
         time.sleep(125)
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, '192.168.5.*')
 
     def test_dhcp_client_reuse_address_as_static(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
+
+        # link become 'routable' when at least one protocol provide an valid address.
+        self.wait_address('veth99', r'inet 192.168.5.[0-9]*/24 brd 192.168.5.255', ipv='-4')
+        self.wait_address('veth99', r'inet6 2600::[0-9a-f]*/128', ipv='-6')
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99', 'scope', 'global']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99', 'scope', 'global'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '192.168.5')
         self.assertRegex(output, '2600::')
 
-        ipv4_address = re.search('192\.168\.5\.[0-9]*/24', output)
-        ipv6_address = re.search('2600::[0-9a-f:]*/128', output)
+        ipv4_address = re.search(r'192.168.5.[0-9]*/24', output)
+        ipv6_address = re.search(r'2600::[0-9a-f:]*/128', output)
         static_network = '\n'.join(['[Match]', 'Name=veth99', '[Network]', 'IPv6AcceptRA=no', 'Address=' + ipv4_address.group(), 'Address=' + ipv6_address.group()])
         print(static_network)
 
@@ -2129,16 +2157,17 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
         with open(os.path.join(network_unit_file_path, 'static.network'), mode='w') as f:
             f.write(static_network)
 
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
+        # When networkd started, the links are already configured, so let's wait for 5 seconds
+        # the links to be re-configured.
+        self.start_networkd(5)
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '192.168.5')
         self.assertRegex(output, 'valid_lft forever preferred_lft forever')
 
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'global']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'global'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, '2600::')
         self.assertRegex(output, 'valid_lft forever preferred_lft forever')
@@ -2147,24 +2176,26 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
     def test_dhcp_client_vrf(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network', 'dhcp-client-vrf.network',
                                              '25-vrf.netdev', '25-vrf.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-        self.assertTrue(self.link_exits('vrf99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable', 'vrf99:carrier'])
+
+        # link become 'routable' when at least one protocol provide an valid address.
+        self.wait_address('veth99', r'inet 192.168.5.[0-9]*/24 brd 192.168.5.255', ipv='-4')
+        self.wait_address('veth99', r'inet6 2600::[0-9a-f]*/128', ipv='-6')
 
         print('## ip -d link show dev vrf99')
-        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'dev', 'vrf99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-d', 'link', 'show', 'dev', 'vrf99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'vrf table 42')
 
         print('## ip address show vrf vrf99')
-        output_ip_vrf = subprocess.check_output(['ip', 'address', 'show', 'vrf', 'vrf99']).rstrip().decode('utf-8')
+        output_ip_vrf = subprocess.check_output(['ip', 'address', 'show', 'vrf', 'vrf99'], universal_newlines=True).rstrip()
         print(output_ip_vrf)
 
         print('## ip address show dev veth99')
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertEqual(output, output_ip_vrf)
         self.assertRegex(output, 'inet 169.254.[0-9]*.[0-9]*/16 brd 169.254.255.255 scope link veth99')
@@ -2173,7 +2204,7 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
         self.assertRegex(output, 'inet6 .* scope link')
 
         print('## ip route show vrf vrf99')
-        output = subprocess.check_output(['ip', 'route', 'show', 'vrf', 'vrf99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'vrf', 'vrf99'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'default via 192.168.5.1 dev veth99 proto dhcp src 192.168.5.')
         self.assertRegex(output, 'default dev veth99 proto static scope link')
@@ -2183,35 +2214,29 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
         self.assertRegex(output, '192.168.5.1 dev veth99 proto dhcp scope link src 192.168.5')
 
         print('## ip route show table main dev veth99')
-        output = subprocess.check_output(['ip', 'route', 'show', 'table', 'main', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'show', 'table', 'main', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
         self.assertEqual(output, '')
 
-        output = subprocess.check_output(['networkctl', 'status', 'vrf99']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: carrier \(configured\)')
-
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
-        print(output)
-        self.assertRegex(output, 'State: routable \(configured\)')
+        self.check_operstate('vrf99', 'carrier')
+        self.check_operstate('veth99', 'routable')
 
     def test_dhcp_client_gateway_onlink_implicit(self):
         self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network',
                                              'dhcp-client-gateway-onlink-implicit.network')
-        self.start_networkd()
-
-        self.assertTrue(self.link_exits('veth99'))
-
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
         self.start_dnsmasq()
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['networkctl', 'status', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(networkctl_cmd + ['status', 'veth99'], universal_newlines=True, env=env).rstrip()
         print(output)
         self.assertRegex(output, '192.168.5')
 
-        output = subprocess.check_output(['ip', 'route', 'list', 'dev', 'veth99', '10.0.0.0/8']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'list', 'dev', 'veth99', '10.0.0.0/8'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'onlink')
-        output = subprocess.check_output(['ip', 'route', 'list', 'dev', 'veth99', '192.168.100.0/24']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'route', 'list', 'dev', 'veth99', '192.168.100.0/24'], universal_newlines=True).rstrip()
         print(output)
         self.assertRegex(output, 'onlink')
 
@@ -2223,31 +2248,31 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
         self.start_dnsmasq(lease_time='2m')
         self.wait_online(['veth99:routable', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
 
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic'], universal_newlines=True).rstrip()
         self.assertNotRegex(output, 'inet6 2600::[0-9a-f]*/128 scope global dynamic')
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'link']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'link'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet6 .* scope link')
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet 192.168.5.[0-9]*/24 brd 192.168.5.255 scope global dynamic veth99')
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'link']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'link'], universal_newlines=True).rstrip()
         self.assertNotRegex(output, 'inet .* scope link')
 
         print('Wait for the dynamic address to be expired')
         time.sleep(130)
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
 
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic'], universal_newlines=True).rstrip()
         self.assertNotRegex(output, 'inet6 2600::[0-9a-f]*/128 scope global dynamic')
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'link']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'link'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet6 .* scope link')
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet 192.168.5.[0-9]*/24 brd 192.168.5.255 scope global dynamic veth99')
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'link']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'link'], universal_newlines=True).rstrip()
         self.assertNotRegex(output, 'inet .* scope link')
 
         self.search_words_in_dnsmasq_log('DHCPOFFER', show_all=True)
@@ -2258,18 +2283,103 @@ class NetworkdNetworkDHCPClientTests(unittest.TestCase, Utilities):
         self.start_networkd(0)
         self.wait_online(['veth99:degraded', 'veth-peer:routable'])
 
-        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', 'address', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
         print(output)
 
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic'], universal_newlines=True).rstrip()
         self.assertNotRegex(output, 'inet6 2600::[0-9a-f]*/128 scope global dynamic')
-        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'link']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-6', 'address', 'show', 'dev', 'veth99', 'scope', 'link'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet6 .* scope link')
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic'], universal_newlines=True).rstrip()
         self.assertNotRegex(output, 'inet 192.168.5.[0-9]*/24 brd 192.168.5.255 scope global dynamic veth99')
-        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'link']).rstrip().decode('utf-8')
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'link'], universal_newlines=True).rstrip()
         self.assertRegex(output, 'inet .* scope link')
 
+    def test_dhcp_client_route_remove_on_renew(self):
+        self.copy_unit_to_networkd_unit_path('25-veth.netdev', 'dhcp-server-veth-peer.network',
+                                             'dhcp-client-ipv4-only-ipv6-disabled.network')
+        self.start_networkd(0)
+        self.wait_online(['veth-peer:carrier'])
+        self.start_dnsmasq(ipv4_range='192.168.5.100,192.168.5.199', lease_time='2m')
+        self.wait_online(['veth99:routable', 'veth-peer:routable'])
+
+        # test for issue #12490
+
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic'], universal_newlines=True).rstrip()
+        print(output)
+        self.assertRegex(output, 'inet 192.168.5.1[0-9]*/24 brd 192.168.5.255 scope global dynamic veth99')
+        address1=None
+        for line in output.splitlines():
+            if 'brd 192.168.5.255 scope global dynamic veth99' in line:
+                address1 = line.split()[1].split('/')[0]
+                break
+
+        output = subprocess.check_output(['ip', '-4', 'route', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
+        print(output)
+        self.assertRegex(output, f'default via 192.168.5.1 proto dhcp src {address1} metric 1024')
+        self.assertRegex(output, f'192.168.5.1 proto dhcp scope link src {address1} metric 1024')
+
+        self.stop_dnsmasq(dnsmasq_pid_file)
+        self.start_dnsmasq(ipv4_range='192.168.5.200,192.168.5.250', lease_time='2m')
+
+        print('Wait for the dynamic address to be expired')
+        time.sleep(130)
+
+        output = subprocess.check_output(['ip', '-4', 'address', 'show', 'dev', 'veth99', 'scope', 'global', 'dynamic'], universal_newlines=True).rstrip()
+        print(output)
+        self.assertRegex(output, 'inet 192.168.5.2[0-9]*/24 brd 192.168.5.255 scope global dynamic veth99')
+        address2=None
+        for line in output.splitlines():
+            if 'brd 192.168.5.255 scope global dynamic veth99' in line:
+                address2 = line.split()[1].split('/')[0]
+                break
+
+        self.assertNotEqual(address1, address2)
+
+        output = subprocess.check_output(['ip', '-4', 'route', 'show', 'dev', 'veth99'], universal_newlines=True).rstrip()
+        print(output)
+        self.assertNotRegex(output, f'default via 192.168.5.1 proto dhcp src {address1} metric 1024')
+        self.assertNotRegex(output, f'192.168.5.1 proto dhcp scope link src {address1} metric 1024')
+        self.assertRegex(output, f'default via 192.168.5.1 proto dhcp src {address2} metric 1024')
+        self.assertRegex(output, f'192.168.5.1 proto dhcp scope link src {address2} metric 1024')
+
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--build-dir', help='Path to build dir', dest='build_dir')
+    parser.add_argument('--networkd', help='Path to systemd-networkd', dest='networkd_bin')
+    parser.add_argument('--wait-online', help='Path to systemd-networkd-wait-online', dest='wait_online_bin')
+    parser.add_argument('--networkctl', help='Path to networkctl', dest='networkctl_bin')
+    parser.add_argument('--valgrind', help='Enable valgrind', dest='use_valgrind', type=bool, nargs='?', const=True, default=use_valgrind)
+    parser.add_argument('--debug', help='Generate debugging logs', dest='enable_debug', type=bool, nargs='?', const=True, default=enable_debug)
+    ns, args = parser.parse_known_args(namespace=unittest)
+
+    if ns.build_dir:
+        if ns.networkd_bin or ns.wait_online_bin or ns.networkctl_bin:
+            print('WARNING: --networkd, --wait-online, or --networkctl options are ignored when --build-dir is specified.')
+        networkd_bin = os.path.join(ns.build_dir, 'systemd-networkd')
+        wait_online_bin = os.path.join(ns.build_dir, 'systemd-networkd-wait-online')
+        networkctl_bin = os.path.join(ns.build_dir, 'networkctl')
+    else:
+        if ns.networkd_bin:
+            networkd_bin = ns.networkd_bin
+        if ns.wait_online_bin:
+            wait_online_bin = ns.wait_online_bin
+        if ns.networkctl_bin:
+            networkctl_bin = ns.networkctl_bin
+
+    use_valgrind = ns.use_valgrind
+    enable_debug = ns.enable_debug
+
+    if use_valgrind:
+        networkctl_cmd = ['valgrind', '--track-origins=yes', '--leak-check=full', '--show-leak-kinds=all', networkctl_bin]
+        wait_online_cmd = ['valgrind', '--track-origins=yes', '--leak-check=full', '--show-leak-kinds=all', wait_online_bin]
+    else:
+        networkctl_cmd = [networkctl_bin]
+        wait_online_cmd = [wait_online_bin]
+
+    if enable_debug:
+        env.update({ 'SYSTEMD_LOG_LEVEL' : 'debug' })
+
+    sys.argv[1:] = args
     unittest.main(testRunner=unittest.TextTestRunner(stream=sys.stdout,
                                                      verbosity=3))