From a98f505c8fff01a22eb0d983490e66d1c5620df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 31 Jul 2019 17:42:20 +0200 Subject: [PATCH 1/2] Use 128k blocks when importing volume data Cloning volume was handled earlier, now od the same on importing volumes (any storage). Fixes QubesOS/qubes-issues#5134 --- qubes-rpc/admin.vm.volume.Import | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes-rpc/admin.vm.volume.Import b/qubes-rpc/admin.vm.volume.Import index 4bfaa70f..f41ee987 100755 --- a/qubes-rpc/admin.vm.volume.Import +++ b/qubes-rpc/admin.vm.volume.Import @@ -43,7 +43,7 @@ size=$(tail -c +3 "$tmpfile"|cut -d ' ' -f 1) path=$(tail -c +3 "$tmpfile"|cut -d ' ' -f 2) # now process stdin into this path -if sudo dd bs=4k of="$path" count="$size" iflag=count_bytes,fullblock \ +if sudo dd bs=128K of="$path" count="$size" iflag=count_bytes,fullblock \ conv=sparse,notrunc,nocreat,fdatasync status=none; then status="ok" else From 39d64eabc82e315ff8c96d318907c78a413dc7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 31 Jul 2019 17:56:36 +0200 Subject: [PATCH 2/2] api/stats: improve cpu_usage normalization, add cpu_usage_raw Give raw cpu_time value, instead of normalized one (to number of vcpus), as documented. Move the normalization to cpu_usage calculation. At the same time, add cpu_usage_raw without it, if anyone needs it. QubesOS/qubes-issues#4531 --- qubes/api/admin.py | 3 ++- qubes/app.py | 17 ++++++++++------- qubes/tests/api_admin.py | 12 ++++++++++++ qubes/tests/app.py | 20 +++++++++++++------- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index 343fbd41..e395edd7 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1561,7 +1561,8 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI): self.send_event(name, 'vm-stats', memory_kb=int(vm_info['memory_kb']), cpu_time=int(vm_info['cpu_time'] / 1000000), - cpu_usage=int(vm_info['cpu_usage'])) + cpu_usage=int(vm_info['cpu_usage']), + cpu_usage_raw=int(vm_info['cpu_usage_raw'])) return info_time, info diff --git a/qubes/app.py b/qubes/app.py index 9f6b4332..087825db 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -306,7 +306,8 @@ class QubesHost: Return a tuple of (measurements_time, measurements), where measurements is a dictionary with key: domid, value: dict: - cpu_time - absolute CPU usage (seconds since its startup) - - cpu_usage - CPU usage in % + - cpu_usage_raw - CPU usage in % + - cpu_usage - CPU usage in % (normalized to number of vcpus) - memory_kb - current memory assigned, in kb This function requires Xen hypervisor. @@ -350,17 +351,19 @@ class QubesHost: domid = vm['domid'] current[domid] = {} current[domid]['memory_kb'] = vm['mem_kb'] - current[domid]['cpu_time'] = int( - vm['cpu_time'] / max(vm['online_vcpus'], 1)) + current[domid]['cpu_time'] = int(vm['cpu_time']) + vcpus = max(vm['online_vcpus'], 1) if domid in previous: - current[domid]['cpu_usage'] = int( + current[domid]['cpu_usage_raw'] = int( (current[domid]['cpu_time'] - previous[domid]['cpu_time']) / 1000 ** 3 * 100 / (current_time - previous_time)) - if current[domid]['cpu_usage'] < 0: + if current[domid]['cpu_usage_raw'] < 0: # VM has been rebooted - current[domid]['cpu_usage'] = 0 + current[domid]['cpu_usage_raw'] = 0 else: - current[domid]['cpu_usage'] = 0 + current[domid]['cpu_usage_raw'] = 0 + current[domid]['cpu_usage'] = \ + int(current[domid]['cpu_usage_raw'] / vcpus) return (current_time, current) diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index ac577048..8637e7bc 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -2129,18 +2129,22 @@ class TC_00_VMs(AdminAPITestCase): 0: { 'cpu_time': 243951379111104 // 8, 'cpu_usage': 0, + 'cpu_usage_raw': 0, 'memory_kb': 3733212, }, 1: { 'cpu_time': 2849496569205, 'cpu_usage': 0, + 'cpu_usage_raw': 0, 'memory_kb': 303916, }, } stats2 = copy.deepcopy(stats1) stats2[0]['cpu_time'] += 100000000 stats2[0]['cpu_usage'] = 10 + stats2[0]['cpu_usage_raw'] = 10 stats2[1]['cpu_usage'] = 5 + stats2[1]['cpu_usage_raw'] = 5 self.app.host.get_vm_stats = unittest.mock.Mock() self.app.host.get_vm_stats.side_effect = [ (0, stats1), (1, stats2), @@ -2181,18 +2185,22 @@ class TC_00_VMs(AdminAPITestCase): unittest.mock.call('dom0', 'vm-stats', cpu_time=stats1[0]['cpu_time'] // 1000000, cpu_usage=stats1[0]['cpu_usage'], + cpu_usage_raw=stats1[0]['cpu_usage_raw'], memory_kb=stats1[0]['memory_kb']), unittest.mock.call('test-template', 'vm-stats', cpu_time=stats1[1]['cpu_time'] // 1000000, cpu_usage=stats1[1]['cpu_usage'], + cpu_usage_raw=stats1[1]['cpu_usage_raw'], memory_kb=stats1[1]['memory_kb']), unittest.mock.call('dom0', 'vm-stats', cpu_time=stats2[0]['cpu_time'] // 1000000, cpu_usage=stats2[0]['cpu_usage'], + cpu_usage_raw=stats2[0]['cpu_usage_raw'], memory_kb=stats2[0]['memory_kb']), unittest.mock.call('test-template', 'vm-stats', cpu_time=stats2[1]['cpu_time'] // 1000000, cpu_usage=stats2[1]['cpu_usage'], + cpu_usage_raw=stats2[1]['cpu_usage_raw'], memory_kb=stats2[1]['memory_kb']), ]) @@ -2203,11 +2211,13 @@ class TC_00_VMs(AdminAPITestCase): 2: { 'cpu_time': 2849496569205, 'cpu_usage': 0, + 'cpu_usage_raw': 0, 'memory_kb': 303916, }, } stats2 = copy.deepcopy(stats1) stats2[2]['cpu_usage'] = 5 + stats2[2]['cpu_usage_raw'] = 5 self.app.host.get_vm_stats = unittest.mock.Mock() self.app.host.get_vm_stats.side_effect = [ (0, stats1), (1, stats2), @@ -2248,10 +2258,12 @@ class TC_00_VMs(AdminAPITestCase): unittest.mock.call('test-vm1', 'vm-stats', cpu_time=stats1[2]['cpu_time'] // 1000000, cpu_usage=stats1[2]['cpu_usage'], + cpu_usage_raw=stats1[2]['cpu_usage_raw'], memory_kb=stats1[2]['memory_kb']), unittest.mock.call('test-vm1', 'vm-stats', cpu_time=stats2[2]['cpu_time'] // 1000000, cpu_usage=stats2[2]['cpu_usage'], + cpu_usage_raw=stats2[2]['cpu_usage_raw'], memory_kb=stats2[2]['memory_kb']), ]) diff --git a/qubes/tests/app.py b/qubes/tests/app.py index 63b630c9..0f06b641 100644 --- a/qubes/tests/app.py +++ b/qubes/tests/app.py @@ -80,18 +80,21 @@ class TC_20_QubesHost(qubes.tests.QubesTestCase): self.assertIsNotNone(info_time) expected_info = { 0: { - 'cpu_time': 243951379111104//8, + 'cpu_time': 243951379111104, 'cpu_usage': 0, + 'cpu_usage_raw': 0, 'memory_kb': 3733212, }, 1: { 'cpu_time': 2849496569205, 'cpu_usage': 0, + 'cpu_usage_raw': 0, 'memory_kb': 303916, }, 11: { - 'cpu_time': 249658663079978//8, + 'cpu_time': 249658663079978, 'cpu_usage': 0, + 'cpu_usage_raw': 0, 'memory_kb': 3782668, }, } @@ -104,25 +107,28 @@ class TC_20_QubesHost(qubes.tests.QubesTestCase): prev_time, prev_info = self.qubes_host.get_vm_stats() prev_time -= 1 - prev_info[0]['cpu_time'] -= 10**8 - prev_info[1]['cpu_time'] -= 10**9 - prev_info[11]['cpu_time'] -= 125 * 10**6 + prev_info[0]['cpu_time'] -= 8*10**8 # 0.8s + prev_info[1]['cpu_time'] -= 10**9 # 1s + prev_info[11]['cpu_time'] -= 10**9 # 1s info_time, info = self.qubes_host.get_vm_stats(prev_time, prev_info) self.assertIsNotNone(info_time) expected_info = { 0: { - 'cpu_time': 243951379111104//8, + 'cpu_time': 243951379111104, 'cpu_usage': 9, + 'cpu_usage_raw': 79, 'memory_kb': 3733212, }, 1: { 'cpu_time': 2849496569205, 'cpu_usage': 99, + 'cpu_usage_raw': 99, 'memory_kb': 303916, }, 11: { - 'cpu_time': 249658663079978//8, + 'cpu_time': 249658663079978, 'cpu_usage': 12, + 'cpu_usage_raw': 99, 'memory_kb': 3782668, }, }