From: Ondrej Lichtner olichtne@redhat.com

The rescan_devices method historically used the NetUtils.scan_netdevs method to externally dump the current configuration state of network devices.

This introduced several race condition issues due to having 2 sockets/communication channels for the same information. The main problem was that messages received on the main InterfaceManager._nl_socket should be deprecated when a rescan_devices is called, however in some situations some messages may already be being processed and saved in a temporary list when a rescan is called.

This could lead to removing a device during the rescan_devices call, then adding it back when handling of nl_socket messages resumes, leading to all sorts of double remove issues.

The redesign to address this issue is to remove the second channel created by the scan_netdevs method. The direct netlink socket connection which we use to listen for notifications can also be used to directly request the dump of the configuration state and add it to the end of the FIFO message queue in the correct order, resolving any possible race conditions.

Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Slave/InterfaceManager.py | 90 +++++++++------------------------- 1 file changed, 23 insertions(+), 67 deletions(-)

diff --git a/lnst/Slave/InterfaceManager.py b/lnst/Slave/InterfaceManager.py index 62f9222..13be2bc 100644 --- a/lnst/Slave/InterfaceManager.py +++ b/lnst/Slave/InterfaceManager.py @@ -16,6 +16,7 @@ import re import select import socket import logging +from collections import deque from lnst.Slave.NetConfigCommon import get_option from lnst.Common.NetUtils import normalize_hwaddr from lnst.Common.NetUtils import scan_netdevs @@ -25,12 +26,15 @@ from lnst.Common.DeviceError import DeviceNotFound, DeviceConfigError, DeviceDel from lnst.Common.InterfaceManagerError import InterfaceManagerError from lnst.Slave.DevlinkManager import DevlinkManager from pyroute2 import IPRSocket +from pyroute2.netlink import NLM_F_REQUEST, NLM_F_DUMP from pyroute2.netlink.rtnl import RTMGRP_IPV4_IFADDR from pyroute2.netlink.rtnl import RTMGRP_IPV6_IFADDR from pyroute2.netlink.rtnl import RTMGRP_LINK from pyroute2.netlink.rtnl import RTM_NEWLINK +from pyroute2.netlink.rtnl import RTM_GETLINK from pyroute2.netlink.rtnl import RTM_DELLINK from pyroute2.netlink.rtnl import RTM_NEWADDR +from pyroute2.netlink.rtnl import RTM_GETADDR from pyroute2.netlink.rtnl import RTM_DELADDR

NL_GROUPS = RTMGRP_IPV4_IFADDR | RTMGRP_IPV6_IFADDR | RTMGRP_LINK @@ -44,6 +48,8 @@ class InterfaceManager(object): self._nl_socket = IPRSocket() self._nl_socket.bind(groups=NL_GROUPS)

+ self._msg_queue = deque() + #TODO split DevlinkManager away from the InterfaceManager #self._dl_manager = DevlinkManager()

@@ -71,78 +77,31 @@ class InterfaceManager(object): def get_nl_socket(self): return self._nl_socket

- def get_netlink_messages(self): + def pull_netlink_messages_into_queue(self): try: - rl, wl, xl = select.select([self._nl_socket], [], [], 0) - try: - f = rl[0] - except: - return [] - return f.get() - except (IndexError, select.error): - return [] + while True: + rl, wl, xl = select.select([self._nl_socket], [], [], 0) + if not len(rl): + break + self._msg_queue.extend(self._nl_socket.get()) except socket.error: self.reconnect_netlink() return []

def rescan_devices(self): - #since we're rescanning all devices we need to dump all the messages - #currently in the Queue of the nl socket - so we don't later update - #devices with outdated messages - self.get_netlink_messages() - - devices_to_remove = list(self._devices.keys()) - devs = scan_netdevs() - for dev in devs: - if dev['index'] not in self._devices: - device = self._device_classes["Device"](self) - device._init_netlink(dev['netlink_msg']) - self._devices[dev['index']] = device - - update_msg = {"type": "dev_created", - "dev_data": device._get_if_data()} - self._server_handler.send_data_to_ctl(update_msg) - - device._disable() - else: - self._devices[dev['index']]._update_netlink(dev['netlink_msg']) - try: - devices_to_remove.remove(dev['index']) - except ValueError: - # we may have multiple updates for the same device, it's - # okay not to find the device in devices_to_remove - pass - - self._devices[dev['index']]._clear_ips() - for addr_msg in dev['ip_addrs']: - self._devices[dev['index']]._update_netlink(addr_msg) - for i in devices_to_remove: - if i not in self._devices: - #TODO - #this is a workaround fix for when the device to remove was - #already removed indirectly by the previous update loop - #the fix works for now but should be refactored at some point - continue - dev_name = self._devices[i].name - logging.debug("Deleting Device with ifindex %d, name %s because "\ - "it doesn't exist anymore." % (i, dev_name)) - - self._devices[i]._deleted = True - del self._devices[i] - - del_msg = {"type": "dev_deleted", - "ifindex": i} - self._server_handler.send_data_to_ctl(del_msg) - - # self._dl_manager.rescan_ports() - # for device in self._devices.values(): - # dl_port = self._dl_manager.get_port(device.name) - # device._set_devlink(dl_port) + self._nl_socket.put( + None, RTM_GETLINK, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) + self._nl_socket.put( + None, RTM_GETADDR, msg_flags=NLM_F_REQUEST | NLM_F_DUMP + ) + self.handle_netlink_msgs()

def handle_netlink_msgs(self): - msgs = self.get_netlink_messages() + self.pull_netlink_messages_into_queue()

- for msg in msgs: + while len(self._msg_queue): + msg = self._msg_queue.popleft() self._handle_netlink_msg(msg)

# self._dl_manager.rescan_ports() @@ -153,10 +112,7 @@ class InterfaceManager(object): def _handle_netlink_msg(self, msg): if msg['header']['type'] in [RTM_NEWLINK, RTM_NEWADDR, RTM_DELADDR]: if msg['index'] in self._devices: - try: - self._devices[msg['index']]._update_netlink(msg) - except DeviceDeleted: - return + self._devices[msg['index']]._update_netlink(msg) elif msg['header']['type'] == RTM_NEWLINK: dev = self._device_classes["Device"](self) dev._init_netlink(msg)

From: Ondrej Lichtner olichtne@redhat.com

Refactoring the create_device method a little enables us to completely remove the dependency of the module on scan_netdevs. The method should now only be used in the legacy NmConfigDevice module that should get removed soon.

Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Slave/InterfaceManager.py | 29 ++++++++++++++++++++--------- lnst/Slave/NetTestSlave.py | 2 -- 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/lnst/Slave/InterfaceManager.py b/lnst/Slave/InterfaceManager.py index 13be2bc..cc9f2d5 100644 --- a/lnst/Slave/InterfaceManager.py +++ b/lnst/Slave/InterfaceManager.py @@ -19,7 +19,6 @@ import logging from collections import deque from lnst.Slave.NetConfigCommon import get_option from lnst.Common.NetUtils import normalize_hwaddr -from lnst.Common.NetUtils import scan_netdevs from lnst.Common.ExecCmd import exec_cmd from lnst.Common.ConnectionHandler import recv_data from lnst.Common.DeviceError import DeviceNotFound, DeviceConfigError, DeviceDeleted @@ -89,13 +88,16 @@ class InterfaceManager(object): return []

def rescan_devices(self): + self.request_netlink_dump() + self.handle_netlink_msgs() + + def request_netlink_dump(self): self._nl_socket.put( None, RTM_GETLINK, msg_flags=NLM_F_REQUEST | NLM_F_DUMP ) self._nl_socket.put( None, RTM_GETADDR, msg_flags=NLM_F_REQUEST | NLM_F_DUMP ) - self.handle_netlink_msgs()

def handle_netlink_msgs(self): self.pull_netlink_messages_into_queue() @@ -196,14 +198,23 @@ class InterfaceManager(object): device._create() device._bulk_enabled = False

- devs = scan_netdevs() - for dev in devs: - if dev["name"] == device.name: - device._init_netlink(dev['netlink_msg']) - self._devices[dev['index']] = device - return device + self.request_netlink_dump() + self.pull_netlink_messages_into_queue()

- return None + device_found = False + while len(self._msg_queue): + msg = self._msg_queue.popleft() + if msg.get_attr("IFLA_IFNAME") == device.name: + device_found = True + device._init_netlink(msg) + self._devices[msg['index']] = device + else: + self._handle_netlink_msg(msg) + + if device_found: + return device + else: + raise DeviceError("Device creation failed")

def replace_dev(self, if_id, dev): del self._devices[if_id] diff --git a/lnst/Slave/NetTestSlave.py b/lnst/Slave/NetTestSlave.py index 27f5cce..4f5fc9f 100644 --- a/lnst/Slave/NetTestSlave.py +++ b/lnst/Slave/NetTestSlave.py @@ -319,8 +319,6 @@ class SlaveMethods:

def create_device(self, clsname, args=[], kwargs={}): dev = self._if_manager.create_device(clsname, args, kwargs) - if dev is None: - raise Exception("Device creation failed") return {"ifindex": dev.ifindex, "name": dev.name}

def start_packet_capture(self, filt):

On Fri, Oct 11, 2019 at 04:43:26PM +0200, olichtne@redhat.com wrote:

From: Ondrej Lichtner olichtne@redhat.com

The _ipr_wrapper method implemented in the base Device class can safely be used for the destroy operation of a SoftDevice. The methods are very similar so this is just refactoring to reduce code duplication.

Signed-off-by: Ondrej Lichtner olichtne@redhat.com

lnst/Devices/SoftDevice.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/lnst/Devices/SoftDevice.py b/lnst/Devices/SoftDevice.py index 7a92c24..3373f1d 100644 --- a/lnst/Devices/SoftDevice.py +++ b/lnst/Devices/SoftDevice.py @@ -69,14 +69,5 @@ class SoftDevice(Device): self.name, str(e)))

 def destroy(self):

•    with pyroute2.IPRoute() as ipr:
  

•        try:
  

•            logging.debug("Performing Netlink operation del for device {} {}".
  

•                    format(self.ifindex, self.name))
  

•            ipr.link("del", index=self.ifindex)
  

•            self._if_manager.rescan_devices()
  

•        except Exception as e:
  

•            log_exc_traceback()
  

•            raise DeviceConfigError("Deleting link {} failed: {}".format(
  

•                self.name, str(e)))
  

•    self._ipr_wrapper("link", "del", index=self.ifindex)
     return True
  

-- 2.23.0

patchset pushed.

-Ondrej