From c9dece98f25e8169c1e8458a28875d10f95888e6 Mon Sep 17 00:00:00 2001 From: Daniel Fremont Date: Tue, 16 Apr 2024 16:26:23 -0700 Subject: [PATCH 1/2] fix ScenicSampler with more than 10 objects, again; improve docs --- src/verifai/features/features.py | 16 ++++++++++- src/verifai/samplers/scenic_sampler.py | 39 ++++++++++++++++++++++---- tests/scenic/test_scenic.py | 8 +++++- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/verifai/features/features.py b/src/verifai/features/features.py index 973a18c..8634ceb 100644 --- a/src/verifai/features/features.py +++ b/src/verifai/features/features.py @@ -721,7 +721,21 @@ def __repr__(self): return f'ScalarArray({self.domain}, {self.shape})' class Struct(Domain): - """A domain consisting of named sub-domains.""" + """A domain consisting of named sub-domains. + + The order of the sub-domains is arbitrary: two Structs are considered equal + if they have the same named sub-domains, regardless of order. As the order + is an implementation detail, accessing the values of sub-domains in points + sampled from a Struct should be done by name: + + >>> struct = Struct({'a': Box((0, 1)), 'b': Box((2, 3))}) + >>> point = struct.uniformPoint() + >>> point.b + (2.20215292046797,) + + Within a given version of VerifAI, the sub-domain order is consistent, so + that the order of columns in error tables is also consistent. + """ def __init__(self, domains): self.namedDomains = tuple(sorted(domains.items(), key=lambda i: i[0])) diff --git a/src/verifai/samplers/scenic_sampler.py b/src/verifai/samplers/scenic_sampler.py index 17e0add..bcea7fa 100644 --- a/src/verifai/samplers/scenic_sampler.py +++ b/src/verifai/samplers/scenic_sampler.py @@ -194,7 +194,10 @@ def spaceForScenario(scenario, ignoredProperties): assert scenario.egoObject is scenario.objects[0] doms = (domainForObject(obj, ignoredProperties) for obj in scenario.objects) - objects = Struct({ f'object{i}': dom for i, dom in enumerate(doms) }) + objects = Struct({ + ScenicSampler.nameForObject(i): dom + for i, dom in enumerate(doms) + }) # create domains for global parameters paramDoms = {} @@ -279,15 +282,30 @@ def nextSample(self, feedback=None): return self.pointForScene(self.lastScene) def pointForScene(self, scene): - """Convert a sampled Scenic :obj:`Scene` to a point in our feature space.""" + """Convert a sampled Scenic :obj:`~scenic.core.scenarios.Scene` to a point in our feature space. + + The `FeatureSpace` used by this sampler consists of 2 features: + + * ``objects``, which is a `Struct` consisting of attributes ``object0``, + ``object1``, etc. with the properties of the corresponding objects + in the Scenic program. The names of these attributes may change in a + future version of VerifAI: use the `nameForObject` function to + generate them. + * ``params``, which is a `Struct` storing the values of the + :term:`global parameters` of the Scenic program (use + `paramDictForSample` to extract them). + """ lengths, dom = self.space.domains assert lengths is None assert scene.egoObject is scene.objects[0] objDomain = dom.domainNamed['objects'] assert len(objDomain.domains) == len(scene.objects) - objects = (pointForObject(objDomain.domainNamed[f'object{i}'], obj) - for i, obj in enumerate(scene.objects)) - objPoint = objDomain.makePoint(*objects) + objects = { + self.nameForObject(i): + pointForObject(objDomain.domainNamed[self.nameForObject(i)], obj) + for i, obj in enumerate(scene.objects) + } + objPoint = objDomain.makePoint(**objects) paramDomain = dom.domainNamed['params'] params = {} @@ -298,8 +316,17 @@ def pointForScene(self, scene): return self.space.makePoint(objects=objPoint, params=paramPoint) + @staticmethod + def nameForObject(i): + """Name used in the `FeatureSpace` for the Scenic object with index i. + + That is, if ``scene`` is a :obj:`~scenic.core.scenarios.Scene`, the object + ``scene.objects[i]``. + """ + return f'object{i}' + def paramDictForSample(self, sample): - """Recover the dict of global parameters from a `ScenicSampler` sample.""" + """Recover the dict of :term:`global parameters` from a `ScenicSampler` sample.""" params = sample.params._asdict() corrected = {} for newName, quotedParam in self.quotedParams.items(): diff --git a/tests/scenic/test_scenic.py b/tests/scenic/test_scenic.py index fbe9ea2..2b3cf78 100644 --- a/tests/scenic/test_scenic.py +++ b/tests/scenic/test_scenic.py @@ -71,9 +71,15 @@ def test_object_order(new_Object): sample = sampler.nextSample() objects = sample.objects assert len(objects) == 11 - for i, obj in enumerate(objects): + for i in range(len(objects)): + name = ScenicSampler.nameForObject(i) + obj = getattr(objects, name) assert obj.position[:2] == pytest.approx((2*i, 0)) + flat = sampler.space.flatten(sample) + unflat = sampler.space.unflatten(flat) + assert unflat == sample + ## Active sampling def test_active_sampling(new_Object): From 77a9868254dfeb49a724082fe7411a72b7c824b0 Mon Sep 17 00:00:00 2001 From: Daniel Fremont Date: Tue, 16 Apr 2024 16:28:35 -0700 Subject: [PATCH 2/2] bump version number --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 55b6c07..aaaf9a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "verifai" -version = "2.1.0" +version = "2.1.1" description = "A toolkit for the formal design and analysis of systems that include artificial intelligence (AI) and machine learning (ML) components." authors = [ { name = "Tommaso Dreossi" },