lvm2-commits June 2022

lvm2-commits@lists.fedorahosted.org

4 participants
25 discussions

main - lvmdbustest: Add test for property "Get"
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=d0f94e763d6e6bca491c7… Commit: d0f94e763d6e6bca491c74cea93ec3062f32757e Parent: 01ef2f2525e8733f01443987b2c6f99882e04f3d Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Tue Jun 7 11:47:27 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbustest: Add test for property "Get" We typically use "GetAll", so add test for "Get" and check values. --- test/dbus/lvmdbustest.py | 14 ++++++++++++++ test/dbus/testlib.py | 12 +++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/test/dbus/lvmdbustest.py b/test/dbus/lvmdbustest.py index 12f1741b1..e6f6d4891 100755 --- a/test/dbus/lvmdbustest.py +++ b/test/dbus/lvmdbustest.py @@ -677,6 +677,20 @@ class TestDbusService(unittest.TestCase): EOD), vg, LV_BASE_INT) self._validate_lookup("%s/%s" % (vg.Name, lv_name), lv.object_path) + def test_prop_get(self): + lv_name = lv_n() + vg = self._vg_create().Vg + lv = self._test_lv_create( + vg.LvCreate, + (dbus.String(lv_name), dbus.UInt64(mib(4)), + dbus.Array([], signature='(ott)'), dbus.Int32(g_tmo), + EOD), vg, LV_BASE_INT) + ri = RemoteInterface(lv.dbus_object, interface=LV_COMMON_INT, introspect=False) + + ri.update() + for prop_name in ri.get_property_names(): + self.assertEqual(ri.get_property_value(prop_name), getattr(ri, prop_name)) + def test_lv_create_job(self): lv_name = lv_n() vg = self._vg_create().Vg diff --git a/test/dbus/testlib.py b/test/dbus/testlib.py index 75a367d40..b793b67a8 100644 --- a/test/dbus/testlib.py +++ b/test/dbus/testlib.py @@ -183,6 +183,7 @@ class RemoteInterface(object): verify_type( vl, self.introspect[self.interface] ['properties'][kl]['p_type']) + self.p_name[kl] = True setattr(self, kl, vl) @property @@ -196,6 +197,7 @@ class RemoteInterface(object): self.interface = interface self.introspect = introspect self.tmo = 0 + self.p_name = {} if timelimit >= 0: self.tmo = float(timelimit) @@ -213,7 +215,7 @@ class RemoteInterface(object): def _wrapper(self, _method_name, *args, **kwargs): - # Lets see how long a method takes to execute, in call cases we should + # Let's see how long a method takes to execute, in call cases we should # return something when the time limit has been reached. start = time.time() result = getattr(self.dbus_interface, _method_name)(*args, **kwargs) @@ -241,6 +243,14 @@ class RemoteInterface(object): def update(self): self._set_props() + def get_property_names(self): + return self.p_name.keys() + + def get_property_value(self, name): + prop_interface = dbus.Interface( + self.dbus_object, 'org.freedesktop.DBus.Properties') + return prop_interface.Get(self.interface, name) + class ClientProxy(object):

1 0

main - lvmdbusd: Remove try/except for mkfifo
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=01ef2f2525e8733f01443… Commit: 01ef2f2525e8733f01443987b2c6f99882e04f3d Parent: 0d957dcacc3118e9f2d4e83830d6c93b8955fd5f Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Tue Jun 7 08:21:03 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbusd: Remove try/except for mkfifo We should never run into this error condition when using tempfile.mkdtemp. --- daemons/lvmdbusd/lvm_shell_proxy.py.in | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/daemons/lvmdbusd/lvm_shell_proxy.py.in b/daemons/lvmdbusd/lvm_shell_proxy.py.in index fa2415fcf..5802d2d70 100644 --- a/daemons/lvmdbusd/lvm_shell_proxy.py.in +++ b/daemons/lvmdbusd/lvm_shell_proxy.py.in @@ -122,11 +122,8 @@ class LVMShellProxy(object): tmp_dir = tempfile.mkdtemp(prefix="lvmdbus_") tmp_file = "%s/lvmdbus_report" % (tmp_dir) - try: - # Let's create a fifo for the report output - os.mkfifo(tmp_file, 0o600) - except FileExistsError: - pass + # Create a fifo for the report output + os.mkfifo(tmp_file, 0o600) # Open the fifo for use to read and for lvm child process to write to. self.report_fd = os.open(tmp_file, os.O_NONBLOCK)

1 0

main - lvmdusd: Remove non lvm JSON output support
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=0d957dcacc3118e9f2d4e… Commit: 0d957dcacc3118e9f2d4e83830d6c93b8955fd5f Parent: 73121e3f0712d228e713b71468484ca55d62d93b Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Tue Jun 7 08:20:06 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdusd: Remove non lvm JSON output support --- daemons/lvmdbusd/cmdhandler.py | 123 +---------------------------------------- daemons/lvmdbusd/lvmdb.py.in | 117 +++------------------------------------ daemons/lvmdbusd/main.py | 13 ++++- 3 files changed, 20 insertions(+), 233 deletions(-) diff --git a/daemons/lvmdbusd/cmdhandler.py b/daemons/lvmdbusd/cmdhandler.py index b88bd9896..8fbcd2e3a 100644 --- a/daemons/lvmdbusd/cmdhandler.py +++ b/daemons/lvmdbusd/cmdhandler.py @@ -220,38 +220,6 @@ def _dc(cmd, args): return c -def parse(out): - rc = [] - - for line in out.split('\n'): - # This line includes separators, so process them - if SEP in line: - elem = line.split(SEP) - cleaned_elem = [] - for e in elem: - e = e.strip() - cleaned_elem.append(e) - - if len(cleaned_elem) > 1: - rc.append(cleaned_elem) - else: - t = line.strip() - if len(t) > 0: - rc.append(t) - return rc - - -def parse_column_names(out, column_names): - lines = parse(out) - rc = [] - - for i in range(0, len(lines)): - d = dict(list(zip(column_names, lines[i]))) - rc.append(d) - - return rc - - def options_to_cli_args(options): rc = [] for k, v in list(dict(options).items()): @@ -637,46 +605,6 @@ def lvm_full_report_json(): return None -def pv_retrieve_with_segs(device=None): - d = [] - err = "" - out = "" - rc = 0 - - columns = ['pv_name', 'pv_uuid', 'pv_fmt', 'pv_size', 'pv_free', - 'pv_used', 'dev_size', 'pv_mda_size', 'pv_mda_free', - 'pv_ba_start', 'pv_ba_size', 'pe_start', 'pv_pe_count', - 'pv_pe_alloc_count', 'pv_attr', 'pv_tags', 'vg_name', - 'vg_uuid', 'pvseg_start', 'pvseg_size', 'segtype', 'pv_missing'] - - # Lvm has some issues where it returns failure when querying pvs when other - # operations are in process, see: - # https://bugzilla.redhat.com/show_bug.cgi?id=1274085 - for i in range(0, 10): - cmd = _dc('pvs', ['-o', ','.join(columns)]) - - if device: - cmd.extend(device) - - rc, out, err = call(cmd) - - if rc == 0: - d = parse_column_names(out, columns) - break - else: - time.sleep(0.2) - log_debug("LVM Bug workaround, retrying pvs command...") - - if rc != 0: - msg = "We were unable to get pvs to return without error after " \ - "trying 10 times, RC=%d, STDERR=(%s), STDOUT=(%s)" % \ - (rc, err, out) - log_error(msg) - raise RuntimeError(msg) - - return d - - def pv_resize(device, size_bytes, create_options): cmd = ['pvresize'] @@ -831,53 +759,6 @@ def activate_deactivate(op, name, activate, control_flags, options): return call(cmd) -def vg_retrieve(vg_specific): - if vg_specific: - assert isinstance(vg_specific, list) - - columns = ['vg_name', 'vg_uuid', 'vg_fmt', 'vg_size', 'vg_free', - 'vg_sysid', 'vg_extent_size', 'vg_extent_count', - 'vg_free_count', 'vg_profile', 'max_lv', 'max_pv', - 'pv_count', 'lv_count', 'snap_count', 'vg_seqno', - 'vg_mda_count', 'vg_mda_free', 'vg_mda_size', - 'vg_mda_used_count', 'vg_attr', 'vg_tags'] - - cmd = _dc('vgs', ['-o', ','.join(columns)]) - - if vg_specific: - cmd.extend(vg_specific) - - d = [] - rc, out, err = call(cmd) - if rc == 0: - d = parse_column_names(out, columns) - - return d - - -def lv_retrieve_with_segments(): - columns = ['lv_uuid', 'lv_name', 'lv_path', 'lv_size', - 'vg_name', 'pool_lv_uuid', 'pool_lv', 'origin_uuid', - 'origin', 'data_percent', - 'lv_attr', 'lv_tags', 'vg_uuid', 'lv_active', 'data_lv', - 'metadata_lv', 'seg_pe_ranges', 'segtype', 'lv_parent', - 'lv_role', 'lv_layout', - 'snap_percent', 'metadata_percent', 'copy_percent', - 'sync_percent', 'lv_metadata_size', 'move_pv', 'move_pv_uuid'] - - cmd = _dc('lvs', ['-a', '-o', ','.join(columns)]) - rc, out, err = call(cmd) - - d = [] - - if rc == 0: - d = parse_column_names(out, columns) - - return d - - if __name__ == '__main__': - pv_data = pv_retrieve_with_segs() - - for p in pv_data: - print(str(p)) + # Leave this for future debug as needed + pass diff --git a/daemons/lvmdbusd/lvmdb.py.in b/daemons/lvmdbusd/lvmdb.py.in index 2e7aba914..87474d0f3 100644 --- a/daemons/lvmdbusd/lvmdb.py.in +++ b/daemons/lvmdbusd/lvmdb.py.in @@ -13,14 +13,13 @@ from collections import OrderedDict import pprint as prettyprint import os -import sys from lvmdbusd import cmdhandler from lvmdbusd.utils import log_debug, log_error class DataStore(object): - def __init__(self, usejson=True, vdo_support=False): + def __init__(self, vdo_support=False): self.pvs = {} self.vgs = {} self.lvs = {} @@ -35,41 +34,9 @@ class DataStore(object): self.lvs_in_vgs = {} self.pvs_in_vgs = {} - # self.refresh() self.num_refreshes = 0 - - if usejson: - self.json = cmdhandler.supports_json() - else: - self.json = usejson - self.vdo_support = vdo_support - @staticmethod - def _insert_record(table, key, record, allowed_multiple): - if key in table: - existing = table[key] - - for rec_k, rec_v in record.items(): - if rec_k in allowed_multiple: - # This column name allows us to store multiple value for - # each type - if not isinstance(existing[rec_k], list): - existing_value = existing[rec_k] - existing[rec_k] = [existing_value, rec_v] - else: - existing[rec_k].append(rec_v) - else: - # If something is not expected to have changing values - # lets ensure that - if existing[rec_k] != rec_v: - raise RuntimeError( - "existing[%s]=%s != %s" % - (rec_k, str(existing[rec_k]), - str(rec_v))) - else: - table[key] = record - @staticmethod def _pvs_parse_common(c_pvs, c_pvs_in_vgs, c_lookup): for p in c_pvs.values(): @@ -84,22 +51,6 @@ class DataStore(object): # Lookup for translating between /dev/<name> and pv uuid c_lookup[p['pv_name']] = p['pv_uuid'] - @staticmethod - def _parse_pvs(_pvs): - pvs = sorted(_pvs, key=lambda pk: pk['pv_name']) - - c_pvs = OrderedDict() - c_lookup = {} - c_pvs_in_vgs = {} - - for p in pvs: - DataStore._insert_record( - c_pvs, p['pv_uuid'], p, - ['pvseg_start', 'pvseg_size', 'segtype']) - - DataStore._pvs_parse_common(c_pvs, c_pvs_in_vgs, c_lookup) - return c_pvs, c_lookup, c_pvs_in_vgs - @staticmethod def _parse_pvs_json(_all): @@ -141,28 +92,6 @@ class DataStore(object): return c_pvs, c_lookup, c_pvs_in_vgs - @staticmethod - def _parse_vgs(_vgs): - vgs = sorted(_vgs, key=lambda vk: vk['vg_uuid']) - - c_vgs = OrderedDict() - c_lookup = {} - - for i in vgs: - vg_name = i['vg_name'] - - # Lvm allows duplicate vg names. When this occurs, each subsequent - # matching VG name will be called vg_name:vg_uuid. Note: ':' is an - # invalid character for lvm VG names - if vg_name in c_lookup: - vg_name = "%s:%s" % (vg_name, i['vg_uuid']) - i['vg_name'] = vg_name - - c_lookup[vg_name] = i['vg_uuid'] - DataStore._insert_record(c_vgs, i['vg_uuid'], i, []) - - return c_vgs, c_lookup - @staticmethod def _parse_vgs_json(_all): @@ -227,22 +156,6 @@ class DataStore(object): return c_lvs, c_lvs_in_vgs, c_lvs_hidden, c_lv_full_lookup - @staticmethod - def _parse_lvs(_lvs): - lvs = sorted(_lvs, key=lambda vk: vk['lv_name']) - - c_lvs = OrderedDict() - c_lv_full_lookup = OrderedDict() - - for i in lvs: - full_name = "%s/%s" % (i['vg_name'], i['lv_name']) - c_lv_full_lookup[full_name] = i['lv_uuid'] - DataStore._insert_record( - c_lvs, i['lv_uuid'], i, - ['seg_pe_ranges', 'segtype']) - - return DataStore._parse_lvs_common(c_lvs, c_lv_full_lookup) - def _parse_lvs_json(self, _all): c_lvs = OrderedDict() @@ -401,22 +314,12 @@ class DataStore(object): log_debug("lvmdb - refresh entry") # Grab everything first then parse it - if self.json: - # Do a single lvm retrieve for everything in json - a = cmdhandler.lvm_full_report_json() - - _pvs, _pvs_lookup, _pvs_in_vgs = self._parse_pvs_json(a) - _vgs, _vgs_lookup = self._parse_vgs_json(a) - _lvs, _lvs_in_vgs, _lvs_hidden, _lvs_lookup = self._parse_lvs_json(a) - - else: - _raw_pvs = cmdhandler.pv_retrieve_with_segs() - _raw_vgs = cmdhandler.vg_retrieve(None) - _raw_lvs = cmdhandler.lv_retrieve_with_segments() + # Do a single lvm retrieve for everything in json + a = cmdhandler.lvm_full_report_json() - _pvs, _pvs_lookup, _pvs_in_vgs = self._parse_pvs(_raw_pvs) - _vgs, _vgs_lookup = self._parse_vgs(_raw_vgs) - _lvs, _lvs_in_vgs, _lvs_hidden, _lvs_lookup = self._parse_lvs(_raw_lvs) + _pvs, _pvs_lookup, _pvs_in_vgs = self._parse_pvs_json(a) + _vgs, _vgs_lookup = self._parse_vgs_json(a) + _lvs, _lvs_in_vgs, _lvs_hidden, _lvs_lookup = self._parse_lvs_json(a) # Set all self.pvs = _pvs @@ -527,13 +430,7 @@ class DataStore(object): if __name__ == "__main__": pp = prettyprint.PrettyPrinter(indent=4) - use_json = False - - if len(sys.argv) != 1: - print(len(sys.argv)) - use_json = True - - ds = DataStore(use_json) + ds = DataStore() ds.refresh() print("PVS") diff --git a/daemons/lvmdbusd/main.py b/daemons/lvmdbusd/main.py index 1e717ef69..597441a9e 100644 --- a/daemons/lvmdbusd/main.py +++ b/daemons/lvmdbusd/main.py @@ -29,7 +29,7 @@ from .utils import log_debug, log_error import argparse import os import sys -from .cmdhandler import LvmFlightRecorder, supports_vdo +from .cmdhandler import LvmFlightRecorder, supports_vdo, supports_json from .request import RequestEntry @@ -116,6 +116,15 @@ def main(): os.environ["LC_ALL"] = "C" cfg.args = parser.parse_args() + + if not cfg.args.use_json: + log_error("Daemon no longer supports lvm without JSON support, exiting!") + sys.exit(1) + else: + if not supports_json(): + log_error("Un-supported version of LVM, daemon requires JSON output, exiting!") + sys.exit(1) + cfg.create_request_entry = RequestEntry # We create a flight recorder in cmdhandler too, but we replace it here @@ -157,7 +166,7 @@ def main(): cfg.om = Lvm(BASE_OBJ_PATH) cfg.om.register_object(Manager(MANAGER_OBJ_PATH)) - cfg.db = lvmdb.DataStore(cfg.args.use_json, cfg.vdo_support) + cfg.db = lvmdb.DataStore(vdo_support=cfg.vdo_support) # Using a thread to process requests, we cannot hang the dbus library # thread that is handling the dbus interface

1 0

main - lvmdbustest: Increase number of LVs
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=73121e3f0712d228e713b… Commit: 73121e3f0712d228e713b71468484ca55d62d93b Parent: 8fa8dfdb8c3b5310a5b43c99fd088f4f076b5903 Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Mon Jun 6 10:01:18 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbustest: Increase number of LVs As storage is getting faster, we need to create more LVs to pass this test. --- test/dbus/lvmdbustest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dbus/lvmdbustest.py b/test/dbus/lvmdbustest.py index ae02c5fef..12f1741b1 100755 --- a/test/dbus/lvmdbustest.py +++ b/test/dbus/lvmdbustest.py @@ -1170,7 +1170,7 @@ class TestDbusService(unittest.TestCase): return # This may not pass - for i in [64, 128]: + for i in [128, 256]: yes = self._test_expired_timer(i) if yes: break

1 0

main - lvmdbustest: Correct comment spelling/grammar
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=8fa8dfdb8c3b5310a5b43… Commit: 8fa8dfdb8c3b5310a5b43c99fd088f4f076b5903 Parent: 55059e002abb3d858e4c53ac0da078ca853ff1f4 Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Mon Jun 6 09:59:59 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbustest: Correct comment spelling/grammar --- test/dbus/lvmdbustest.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/dbus/lvmdbustest.py b/test/dbus/lvmdbustest.py index e16a484b7..ae02c5fef 100755 --- a/test/dbus/lvmdbustest.py +++ b/test/dbus/lvmdbustest.py @@ -153,7 +153,7 @@ def get_objects(): rc[interface].append(proxy) # At this point we have a full population of everything, we now need to - # prune the the objects if we are filtering PVs with a sub selection. + # prune the objects if we are filtering PVs with a sub selection. return _prune(rc, pv_device_list), bus @@ -1111,7 +1111,7 @@ class TestDbusService(unittest.TestCase): def _test_expired_timer(self, num_lvs): rc = False - # In small configurations lvm is pretty snappy, so lets create a VG + # In small configurations lvm is pretty snappy, so let's create a VG # add a number of LVs and then remove the VG and all the contained # LVs which appears to consistently run a little slow. @@ -1781,7 +1781,7 @@ class TestDbusService(unittest.TestCase): self.assertTrue(ec == 0, "%s exit code = %d" % (operation, ec)) def test_external_vg_create(self): - # We need to ensure that if a user creates something outside of lvm + # We need to ensure that if a user creates something outside lvm # dbus service that things are sequenced correctly so that if a dbus # user calls into the service they will find the same information. vg_name = vg_n() @@ -1794,8 +1794,8 @@ class TestDbusService(unittest.TestCase): self._verify_existence(cmd, cmd[0], vg_name) def test_external_lv_create(self): - # Lets create a LV outside of service and see if we correctly handle - # it's inclusion + # Let's create a LV outside of service and see if we correctly handle + # its inclusion vg = self._vg_create().Vg lv_name = lv_n() full_name = "%s/%s" % (vg.Name, lv_name) @@ -1804,8 +1804,8 @@ class TestDbusService(unittest.TestCase): self._verify_existence(cmd, cmd[0], full_name) def test_external_pv_create(self): - # Lets create a PV outside of service and see if we correctly handle - # it's inclusion + # Let's create a PV outside of service and see if we correctly handle + # its inclusion target = self.objs[PV_INT][0] # Remove the PV @@ -1851,7 +1851,7 @@ class TestDbusService(unittest.TestCase): # # NOTE: This needs an equivalent of aux extend_filter_LVMTEST # when run from lvm2 testsuite. See dbustest.sh. - # Also if developing locally with actual devices one can achieve this + # Also, if developing locally with actual devices one can achieve this # by editing lvm.conf with "devices/scan_lvs = 1" As testing # typically utilizes loopback, this test is skipped in # those environments. @@ -1866,7 +1866,7 @@ class TestDbusService(unittest.TestCase): pv_object_path = self._create_nested(pv_object_path) def test_pv_symlinks(self): - # Lets take one of our test PVs, pvremove it, find a symlink to it + # Let's take one of our test PVs, pvremove it, find a symlink to it # and re-create using the symlink to ensure we return an object # path to it. Additionally, we will take the symlink and do a lookup # (Manager.LookUpByLvmId) using it and the original device path to @@ -1888,7 +1888,7 @@ class TestDbusService(unittest.TestCase): rc = self._lookup(pv_device_path) self.assertEqual(rc, '/') - # Lets locate a symlink for it + # Let's locate a symlink for it devices = glob('/dev/disk/*/*') rp_pv_device_path = os.path.realpath(pv_device_path) for d in devices: @@ -1953,7 +1953,7 @@ class TestDbusService(unittest.TestCase): raise unittest.SkipTest('vdo not supported') # Do this twice to ensure we are providing the correct flags to force - # the operation when if finds an existing vdo signature, which likely + # the operation when it finds an existing vdo signature, which likely # shouldn't exist. for _ in range(0, 2): vg, _, _ = self._create_vdo_pool_and_lv()

1 0

main - lvmdbustest: Test job remove path when job not complete
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=55059e002abb3d858e4c5… Commit: 55059e002abb3d858e4c53ac0da078ca853ff1f4 Parent: d393436727014b9eddf71009065f65a59ff3837c Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Mon Jun 6 09:58:39 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbustest: Test job remove path when job not complete --- test/dbus/lvmdbustest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/dbus/lvmdbustest.py b/test/dbus/lvmdbustest.py index d876a1748..e16a484b7 100755 --- a/test/dbus/lvmdbustest.py +++ b/test/dbus/lvmdbustest.py @@ -886,8 +886,19 @@ class TestDbusService(unittest.TestCase): j.Remove() break + else: + # Most of the time we will get this exception as expected, but there is + # a race condition between checking if it's complete and removing it (we want to try to remove while + # it's not complete to raise the exception) + # Thus, we can't reliably use self.assertRaises. + # We have included it here to test this path in the daemon. + try: + j.Remove() + except dbus.exceptions.DBusException: + pass if j.Wait(1): + self.assertTrue(j.Wait(0)) j.update() self.assertTrue(j.Complete)

1 0

main - lvmdbusd: Correct grammar in lvm shell proxy comments
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=d393436727014b9eddf71… Commit: d393436727014b9eddf71009065f65a59ff3837c Parent: 691494268502ddb20da2a14568984c0fa4f29f50 Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Mon Jun 6 09:57:20 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbusd: Correct grammar in lvm shell proxy comments --- daemons/lvmdbusd/lvm_shell_proxy.py.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemons/lvmdbusd/lvm_shell_proxy.py.in b/daemons/lvmdbusd/lvm_shell_proxy.py.in index e106ca36f..fa2415fcf 100644 --- a/daemons/lvmdbusd/lvm_shell_proxy.py.in +++ b/daemons/lvmdbusd/lvm_shell_proxy.py.in @@ -123,7 +123,7 @@ class LVMShellProxy(object): tmp_file = "%s/lvmdbus_report" % (tmp_dir) try: - # Lets create fifo for the report output + # Let's create a fifo for the report output os.mkfifo(tmp_file, 0o600) except FileExistsError: pass @@ -162,7 +162,7 @@ class LVMShellProxy(object): except: raise finally: - # These will get deleted when the FD count goes to zero so we + # These will get deleted when the FD count goes to zero, so we # can be sure to clean up correctly no matter how we finish os.unlink(tmp_file) os.rmdir(tmp_dir)

1 0

main - lvmdbusd: Don't require "lvm> " prompt for shell
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=691494268502ddb20da2a… Commit: 691494268502ddb20da2a14568984c0fa4f29f50 Parent: eee89a941eb4e63865356cfe9e513c24cfa8e0f9 Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Mon Jun 6 09:56:32 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbusd: Don't require "lvm> " prompt for shell Depending on how lvm is compiled, it may not present the "lvm> " prompt when using the lvm shell. Don't require it to be present. Addresses: https://bugzilla.redhat.com/show_bug.cgi?id=2090391 --- daemons/lvmdbusd/lvm_shell_proxy.py.in | 83 ++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/daemons/lvmdbusd/lvm_shell_proxy.py.in b/daemons/lvmdbusd/lvm_shell_proxy.py.in index 1a5051a92..e106ca36f 100644 --- a/daemons/lvmdbusd/lvm_shell_proxy.py.in +++ b/daemons/lvmdbusd/lvm_shell_proxy.py.in @@ -19,7 +19,6 @@ import sys import tempfile import time import select -import copy try: import simplejson as json @@ -31,8 +30,6 @@ from lvmdbusd.cfg import LVM_CMD from lvmdbusd.utils import log_debug, log_error, add_no_notify, make_non_block,\ read_decoded -SHELL_PROMPT = "lvm> " - def _quote_arg(arg): if len(shlex.split(arg)) > 1: @@ -43,10 +40,11 @@ def _quote_arg(arg): class LVMShellProxy(object): - # Read until we get prompt back and a result - # @param: no_output Caller expects no output to report FD - # Returns stdout, report, stderr (report is JSON!) - def _read_until_prompt(self, no_output=False): + # Read REPORT FD until we have a complete and valid JSON record or give + # up trying to get one. + # + # Returns stdout, report (JSON), stderr + def _read_response(self): stdout = "" report = "" stderr = "" @@ -58,6 +56,7 @@ class LVMShellProxy(object): # Try reading from all FDs to prevent one from filling up and causing # a hang. Keep reading until we get the prompt back and the report # FD does not contain valid JSON + while keep_reading: try: rd_fd = [ @@ -78,32 +77,33 @@ class LVMShellProxy(object): if self.lvm_shell.poll() is not None: raise Exception(self.lvm_shell.returncode, "%s" % stderr) - if stdout.endswith(SHELL_PROMPT): - if no_output: - keep_reading = False - else: - cur_report_len = len(report) - if cur_report_len != 0: - # Only bother to parse if we have more data - if prev_report_len != cur_report_len: - prev_report_len = cur_report_len - # Parse the JSON if it's good we are done, - # if not we will try to read some more. - try: - report_json = json.loads(report) - keep_reading = False - except ValueError: - pass - - if keep_reading: - extra_passes -= 1 - if extra_passes <= 0: - if len(report): - raise ValueError("Invalid json: %s" % - report) - else: - raise ValueError( - "lvm returned no JSON output!") + cur_report_len = len(report) + if cur_report_len != 0: + # Only bother to parse if we have more data and the last 2 characters match expected + # complete JSON, prevents excessive JSON parsing attempts + if prev_report_len != cur_report_len and report[-2:] == "}\n": + prev_report_len = cur_report_len + + # Parse the JSON if it's good we are done, + # if not we will try to read some more. + try: + report_json = json.loads(report) + keep_reading = False + except ValueError: + pass + + # As long as lvm is spewing something on one of the FDs we will + # keep trying. If we get a few timeouts with no activity, and + # we don't have valid JSON, we will raise an error. + if len(ready) == 0 and keep_reading: + extra_passes -= 1 + if extra_passes <= 0: + if len(report): + raise ValueError("Invalid json: %s" % + report) + else: + raise ValueError( + "lvm returned no JSON output!") except IOError as ioe: log_debug(str(ioe)) @@ -118,7 +118,6 @@ class LVMShellProxy(object): self.lvm_shell.stdin.flush() def __init__(self): - # Create a temp directory tmp_dir = tempfile.mkdtemp(prefix="lvmdbus_") tmp_file = "%s/lvmdbus_report" % (tmp_dir) @@ -139,6 +138,11 @@ class LVMShellProxy(object): local_env = {"LC_ALL": "C", "LVM_REPORT_FD": "%s" % lvm_fd, "LVM_COMMAND_PROFILE": "lvmdbusd", "LVM_LOG_FILE_MAX_LINES": "0"} + # If any env variables contain LVM we will propagate them too + for k, v in os.environ.items(): + if "LVM" in k: + local_env[k] = v + # run the lvm shell self.lvm_shell = subprocess.Popen( [LVM_CMD], @@ -152,10 +156,9 @@ class LVMShellProxy(object): # Close our copy of the lvm_fd, child process is open in its process space os.close(lvm_fd) - # wait for the first prompt - errors = self._read_until_prompt(no_output=True)[2] - if errors and len(errors): - raise RuntimeError(errors) + # Assume we are ready as we may not get the lvm prompt message depending on + # if we are using readline or editline. + except: raise finally: @@ -169,7 +172,7 @@ class LVMShellProxy(object): self._write_cmd('lastlog\n') # read everything from the STDOUT to the next prompt - stdout, report_json, stderr = self._read_until_prompt() + stdout, report_json, stderr = self._read_response() if 'log' in report_json: error_msg = "" # Walk the entire log array and build an error string @@ -203,7 +206,7 @@ class LVMShellProxy(object): self._write_cmd(cmd) # read everything from the STDOUT to the next prompt - stdout, report_json, stderr = self._read_until_prompt() + stdout, report_json, stderr = self._read_response() # Parse the report to see what happened if 'log' in report_json:

1 0

main - lvmdbusd: Job prop. Get/GetAll exec. immediately
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=eee89a941eb4e63865356… Commit: eee89a941eb4e63865356cfe9e513c24cfa8e0f9 Parent: 7a2090655d3ab5abde83b981594ed527e2a7f1f7 Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Mon Jun 6 09:51:54 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbusd: Job prop. Get/GetAll exec. immediately This allows API user the ability to check on the status of a long running job without blocking in the request queue. --- daemons/lvmdbusd/job.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/daemons/lvmdbusd/job.py b/daemons/lvmdbusd/job.py index 988b1147a..7629cafc7 100644 --- a/daemons/lvmdbusd/job.py +++ b/daemons/lvmdbusd/job.py @@ -226,3 +226,21 @@ class Job(AutomatedProperties): def Uuid(self): import uuid return uuid.uuid1() + + # Override the property "getters" implementation for the job interface, so a user can query a job while the queue + # is processing items. Originally all the property get methods were this way, but we changed this in + # e53454d6de07de56736303dd2157c3859f6fa848 + + # Properties + # noinspection PyUnusedLocal + @dbus.service.method(dbus_interface=dbus.PROPERTIES_IFACE, + in_signature='ss', out_signature='v') + def Get(self, interface_name, property_name): + # Note: If we get an exception in this handler we won't know about it, + # only the side effect of no returned value! + return AutomatedProperties._get_prop(self, interface_name, property_name) + + @dbus.service.method(dbus_interface=dbus.PROPERTIES_IFACE, + in_signature='s', out_signature='a{sv}') + def GetAll(self, interface_name): + return AutomatedProperties._get_all_prop(self, interface_name)

1 0

main - lvmdbusd: Remove the use of sub shell for lvm shell
by Tony Asleson 30 Jun '22

30 Jun '22

Gitweb: https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=7a2090655d3ab5abde83b… Commit: 7a2090655d3ab5abde83b981594ed527e2a7f1f7 Parent: b3d7aff6a3a8fd55790f61b9b0b33d599841030b Author: Tony Asleson <tasleson(a)redhat.com> AuthorDate: Thu May 26 10:44:02 2022 -0500 Committer: Tony Asleson <tasleson(a)redhat.com> CommitterDate: Thu Jun 30 10:55:16 2022 -0500 lvmdbusd: Remove the use of sub shell for lvm shell This reduces the number of processes and improves security. --- daemons/lvmdbusd/lvm_shell_proxy.py.in | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/daemons/lvmdbusd/lvm_shell_proxy.py.in b/daemons/lvmdbusd/lvm_shell_proxy.py.in index 40639442c..1a5051a92 100644 --- a/daemons/lvmdbusd/lvm_shell_proxy.py.in +++ b/daemons/lvmdbusd/lvm_shell_proxy.py.in @@ -129,31 +129,29 @@ class LVMShellProxy(object): except FileExistsError: pass - # We have to open non-blocking as the other side isn't open until - # we actually fork the process. + # Open the fifo for use to read and for lvm child process to write to. self.report_fd = os.open(tmp_file, os.O_NONBLOCK) self.report_stream = os.fdopen(self.report_fd, 'rb', 0) + lvm_fd = os.open(tmp_file, os.O_WRONLY) - # Setup the environment for using our own socket for reporting - local_env = {} - local_env["LC_ALL"] = "C" - local_env["LVM_REPORT_FD"] = "32" - local_env["LVM_COMMAND_PROFILE"] = "lvmdbusd" - - # Disable the abort logic if lvm logs too much, which easily happens - # when utilizing the lvm shell. - local_env["LVM_LOG_FILE_MAX_LINES"] = "0" + # Set up the environment for using our own socket for reporting and disable the abort + # logic if lvm logs too much, which easily happens when utilizing the lvm shell. + local_env = {"LC_ALL": "C", "LVM_REPORT_FD": "%s" % lvm_fd, "LVM_COMMAND_PROFILE": "lvmdbusd", + "LVM_LOG_FILE_MAX_LINES": "0"} # run the lvm shell self.lvm_shell = subprocess.Popen( - [LVM_CMD + " 32>%s" % tmp_file], + [LVM_CMD], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=local_env, - stderr=subprocess.PIPE, close_fds=True, shell=True) + stderr=subprocess.PIPE, close_fds=True, pass_fds=(lvm_fd,), shell=False) try: make_non_block(self.lvm_shell.stdout) make_non_block(self.lvm_shell.stderr) + # Close our copy of the lvm_fd, child process is open in its process space + os.close(lvm_fd) + # wait for the first prompt errors = self._read_until_prompt(no_output=True)[2] if errors and len(errors):

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

lvm2-commits June 2022