Merge remote-tracking branch 'origin/pr/278'

* origin/pr/278:
  Added admin.pool.UsageDetails API method
  Add metadata info to LVM AdminAPI

Makefile

@@ -21,6 +21,7 @@ ADMIN_API_METHODS_SIMPLE = \
 	admin.label.Remove \
 	admin.pool.Add \
 	admin.pool.Info \
+	admin.pool.UsageDetails \
 	admin.pool.List \
 	admin.pool.ListDrivers \
 	admin.pool.Remove \

qubes/api/admin.py

@@ -622,6 +622,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
         self.fire_event_for_permission(pool=pool)
 
         other_info = ''
+        # Deprecated: remove this when all tools using this call are updated
         pool_size = pool.size
         if pool_size is not None:
             other_info += 'size={}\n'.format(pool_size)
@@ -641,6 +642,26 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
             for prop, val in sorted(pool.config.items())) + \
             other_info
 
+    @qubes.api.method('admin.pool.UsageDetails', no_payload=True,
+        scope='global', read=True)
+    @asyncio.coroutine
+    def pool_usage(self):
+        self.enforce(self.dest.name == 'dom0')
+        self.enforce(self.arg in self.app.pools.keys())
+
+        pool = self.app.pools[self.arg]
+
+        self.fire_event_for_permission(pool=pool)
+
+        usage = ''
+
+        pool_details = pool.usage_details
+
+        for name in sorted(pool_details):
+            usage += '{}={}\n'.format(name, pool_details[name])
+
+        return usage
+
     @qubes.api.method('admin.pool.Add',
         scope='global', write=True)
     @asyncio.coroutine

qubes/storage/__init__.py

@@ -795,6 +795,17 @@ class Pool:
     def usage(self):
         ''' Space used in the pool in bytes, or None if unknown '''
 
+    @property
+    def usage_details(self):
+        """Detailed information about pool usage as a dictionary
+        Contains data_usage for usage in bytes and data_size for pool
+        size; other implementations may add more implementation-specific
+        detail"""
+        return {
+            'data_usage': self.usage,
+            'data_size': self.size
+        }
+
     def _not_implemented(self, method_name):
         ''' Helper for emitting helpful `NotImplementedError` exceptions '''
         msg = "Pool driver {!s} has {!s}() not implemented"

qubes/storage/lvm.py

@@ -195,10 +195,28 @@ class ThinPool(qubes.storage.Pool):
         except KeyError:
             return 0
 
+    @property
+    def usage_details(self):
+        result = {}
+        result['data_size'] = self.size
+        result['data_usage'] = self.usage
+
+        try:
+            metadata_size = qubes.storage.lvm.size_cache[
+                self.volume_group + '/' + self.thin_pool]['metadata_size']
+            metadata_usage = qubes.storage.lvm.size_cache[
+                self.volume_group + '/' + self.thin_pool]['metadata_usage']
+        except KeyError:
+            metadata_size = 0
+            metadata_usage = 0
+        result['metadata_size'] = metadata_size
+        result['metadata_usage'] = metadata_usage
+
+        return result
 
 _init_cache_cmd = ['lvs', '--noheadings', '-o',
-   'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin',
-   '--units', 'b', '--separator', ';']
+   'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin,lv_metadata_size,'
+   'metadata_percent', '--units', 'b', '--separator', ';']
 
 def _parse_lvm_cache(lvm_output):
     result = {}
@@ -206,14 +224,20 @@ def _parse_lvm_cache(lvm_output):
     for line in lvm_output.splitlines():
         line = line.decode().strip()
         pool_name, pool_lv, name, size, usage_percent, attr, \
-            origin = line.split(';', 6)
+            origin, metadata_size, metadata_percent = line.split(';', 8)
         if '' in [pool_name, name, size, usage_percent]:
             continue
         name = pool_name + "/" + name
         size = int(size[:-1])  # Remove 'B' suffix
         usage = int(size / 100 * float(usage_percent))
+        if metadata_size:
+            metadata_size = int(metadata_size[:-1])
+            metadata_usage = int(metadata_size / 100 * float(metadata_percent))
+        else:
+            metadata_usage = None
         result[name] = {'size': size, 'usage': usage, 'pool_lv': pool_lv,
-            'attr': attr, 'origin': origin}
+            'attr': attr, 'origin': origin, 'metadata_size': metadata_size,
+                        'metadata_usage': metadata_usage}
 
     return result
 

qubes/tests/api_admin.py

@@ -595,7 +595,7 @@ class TC_00_VMs(AdminAPITestCase):
         self.app.pools = {
             'pool1': unittest.mock.Mock(config={
                 'param1': 'value1', 'param2': 'value2'},
-                size=None, usage=None),
+                size=None, usage=None, usage_details={}),
         }
         self.app.pools['pool1'].included_in.return_value = None
         value = self.call_mgmt_func(b'admin.pool.Info', b'dom0', b'pool1')
@@ -622,6 +622,23 @@ class TC_00_VMs(AdminAPITestCase):
             '\nincluded_in=pool1\n')
         self.assertFalse(self.app.save.called)
 
+    def test_153_pool_usage(self):
+        self.app.pools = {
+            'pool1': unittest.mock.Mock(config={
+                'param1': 'value1', 'param2': 'value2'},
+                usage_details={
+                    'data_usage': 102400,
+                    'data_size': 204800,
+                    'metadata_size': 1024,
+                    'metadata_usage': 50})
+        }
+        self.app.pools['pool1'].included_in.return_value = None
+        value = self.call_mgmt_func(b'admin.pool.UsageDetails', b'dom0', b'pool1')
+
+        self.assertEqual(value,
+                         'data_size=204800\ndata_usage=102400\nmetadata_size=1024\nmetadata_usage=50\n')
+        self.assertFalse(self.app.save.called)
+
     @unittest.mock.patch('qubes.storage.pool_drivers')
     @unittest.mock.patch('qubes.storage.driver_parameters')
     def test_160_pool_add(self, mock_parameters, mock_drivers):

qubes/tests/storage_lvm.py

@@ -997,6 +997,38 @@ class TC_00_ThinPool(ThinPoolBase):
         self.assertNotIn(volume1, list(self.pool.volumes))
         self.assertNotIn(volume1, list(self.pool.volumes))
 
+    def test_110_metadata_size(self):
+        with self.assertNotRaises(NotImplementedError):
+            usage = self.pool.usage_details
+
+        metadata_size = usage['metadata_size']
+        environ = os.environ.copy()
+        environ['LC_ALL'] = 'C.utf8'
+        pool_size = subprocess.check_output(['sudo', 'lvs', '--noheadings',
+                                             '-o', 'lv_metadata_size',
+                                             '--units', 'b',
+                                             self.pool.volume_group + '/' + self.pool.thin_pool],
+                                            env=environ)
+        self.assertEqual(metadata_size, int(pool_size.strip()[:-1]))
+
+    def test_111_metadata_usage(self):
+        with self.assertNotRaises(NotImplementedError):
+            usage = self.pool.usage_details
+
+        metadata_usage = usage['metadata_usage']
+        environ = os.environ.copy()
+        environ['LC_ALL'] = 'C.utf8'
+
+        pool_info = subprocess.check_output(['sudo', 'lvs', '--noheadings',
+            '-o', 'lv_metadata_size,metadata_percent',
+            '--units', 'b', self.pool.volume_group + '/' + self.pool.thin_pool],
+            env=environ)
+        pool_size, pool_usage = pool_info.strip().split()
+        pool_size = int(pool_size[:-1])
+        pool_usage = float(pool_usage)
+        self.assertEqual(metadata_usage, int(pool_size * pool_usage / 100))
+
+
 @skipUnlessLvmPoolExists
 class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase):
     ''' Sanity tests for :py:class:`qubes.storage.lvm.ThinPool` '''