From 1275703ca08b3b44cd5513f0d0798ae24be2d571 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 09:03:34 +0100 Subject: [PATCH 01/14] all imports relative --- autofit/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/autofit/__init__.py b/autofit/__init__.py index b5e1db99f..41c9020bf 100644 --- a/autofit/__init__.py +++ b/autofit/__init__.py @@ -9,7 +9,7 @@ dir(conf) from . import exc -from autofit.optimize.non_linear.samples import AbstractSamples, MCMCSamples, NestedSamplerSamples +from .optimize.non_linear.samples import AbstractSamples, MCMCSamples, NestedSamplerSamples from .aggregator import Aggregator, PhaseOutput from .mapper import * from .mapper import link @@ -24,8 +24,8 @@ from .mapper.prior_model.annotation import AnnotationPriorModel from .mapper.prior_model.collection import CollectionPriorModel from .mapper.prior_model.collection import CollectionPriorModel as Collection -from autofit.mapper.prior.deferred import DeferredArgument -from autofit.mapper.prior.deferred import DeferredInstance +from .mapper.prior.deferred import DeferredArgument +from .mapper.prior.deferred import DeferredInstance from .mapper.prior_model.dimension_type import DimensionType, map_types from .mapper.prior_model.prior_model import PriorModel from .mapper.prior_model.prior_model import PriorModel as Model @@ -40,9 +40,9 @@ from .optimize.non_linear.non_linear import Analysis from .optimize.non_linear.non_linear import NonLinearOptimizer from .optimize.non_linear.emcee import Emcee -from autofit.optimize.non_linear.paths import Paths -from autofit.optimize.non_linear.paths import make_path -from autofit.optimize.non_linear.paths import convert_paths +from .optimize.non_linear.paths import Paths +from .optimize.non_linear.paths import make_path +from .optimize.non_linear.paths import convert_paths from .optimize.non_linear.non_linear import Result from .text import formatter, samples_text from .tools import * @@ -54,8 +54,8 @@ from .tools.phase_property import PhaseProperty from .tools.pipeline import Pipeline from .tools.pipeline import ResultsCollection -from autofit.mapper.prior import AbstractPromise -from autofit.mapper.prior import last +from .mapper.prior import AbstractPromise +from .mapper.prior import last from .mapper.prior import * __version__ = '0.58.0' From 56274a153f3e606d102eb7dbb9eb91c0311e6819 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 09:19:55 +0100 Subject: [PATCH 02/14] adjusting import --- autofit/__init__.py | 9 +++++---- .../unit/mapper/model/test_model_mapper.py | 7 +++---- test_autofit/unit/mapper/promise/test_iteration.py | 6 +++--- test_autofit/unit/mapper/promise/test_promise.py | 10 +++++----- test_autofit/unit/mapper/test_assertion.py | 14 +++++++------- test_autofit/unit/mapper/test_prior_parsing.py | 8 ++++---- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/autofit/__init__.py b/autofit/__init__.py index 41c9020bf..9ce50195a 100644 --- a/autofit/__init__.py +++ b/autofit/__init__.py @@ -11,7 +11,6 @@ from . import exc from .optimize.non_linear.samples import AbstractSamples, MCMCSamples, NestedSamplerSamples from .aggregator import Aggregator, PhaseOutput -from .mapper import * from .mapper import link from .mapper.model import AbstractModel from .mapper.model import ModelInstance @@ -31,7 +30,6 @@ from .mapper.prior_model.prior_model import PriorModel as Model from .mapper.prior_model.util import PriorModelNameValue from .optimize.grid_search import GridSearch as OptimizerGridSearch -from .optimize import * from .optimize.non_linear.downhill_simplex import DownhillSimplex from .optimize.non_linear.nested_sampling.dynesty import DynestyStatic, DynestyDynamic from .optimize.grid_search import GridSearchResult @@ -45,7 +43,6 @@ from .optimize.non_linear.paths import convert_paths from .optimize.non_linear.non_linear import Result from .text import formatter, samples_text -from .tools import * from .tools import path_util from .tools.phase import AbstractPhase from .tools.phase import Phase @@ -56,6 +53,10 @@ from .tools.pipeline import ResultsCollection from .mapper.prior import AbstractPromise from .mapper.prior import last -from .mapper.prior import * +from .mapper.prior import GaussianPrior +from .mapper.prior import UniformPrior +from .mapper.prior import LogUniformPrior +from .mapper.prior import Prior +from .mapper import prior __version__ = '0.58.0' diff --git a/test_autofit/unit/mapper/model/test_model_mapper.py b/test_autofit/unit/mapper/model/test_model_mapper.py index 77f4d90cc..56b54c738 100644 --- a/test_autofit/unit/mapper/model/test_model_mapper.py +++ b/test_autofit/unit/mapper/model/test_model_mapper.py @@ -4,12 +4,12 @@ import pytest import autofit as af -from test_autofit.mock import MockClassGaussian import test_autofit.mock from autofit import exc from autofit.text import formatter as frm from test_autofit import mock from test_autofit.mock import GeometryProfile +from test_autofit.mock import MockClassGaussian dataset_path = "{}/../".format(os.path.dirname(os.path.realpath(__file__))) @@ -108,7 +108,7 @@ def test_with_instance(self): def test_with_promise(self): mm = af.ModelMapper() - mm.promise = af.Promise( + mm.promise = af.prior.Promise( af.Phase( phase_name="phase", analysis_class=None @@ -528,7 +528,6 @@ def test_log_priors_from_vector(self): assert log_priors == [0.125, 0.2] def test_random_vector_from_prior_within_limits(self): - np.random.seed(1) mapper = af.ModelMapper() @@ -864,7 +863,7 @@ def make_promise_mapper(): mapper = af.ModelMapper() mapper.galaxy = af.PriorModel( mock.Galaxy, - redshift=af.Promise( + redshift=af.prior.Promise( None, None, is_instance=False, diff --git a/test_autofit/unit/mapper/promise/test_iteration.py b/test_autofit/unit/mapper/promise/test_iteration.py index a59b16ed4..667b8d436 100644 --- a/test_autofit/unit/mapper/promise/test_iteration.py +++ b/test_autofit/unit/mapper/promise/test_iteration.py @@ -46,8 +46,8 @@ def test_index_type(self, phase): promise_0 = phase.result.model.collection[0] promise_1 = phase.result.model.collection[1] - assert isinstance(promise_0, af.Promise) - assert isinstance(promise_1, af.Promise) + assert isinstance(promise_0, af.prior.Promise) + assert isinstance(promise_1, af.prior.Promise) def test_index_populate_model(self, phase, prior_0, prior_1, results_collection): promise_0 = phase.result.model.collection[0] @@ -73,4 +73,4 @@ def test_iteration(self, phase): promises = list(phase.result.model.collection) assert len(promises) == 2 - assert all([isinstance(promise, af.Promise) for promise in promises]) + assert all([isinstance(promise, af.prior.Promise) for promise in promises]) diff --git a/test_autofit/unit/mapper/promise/test_promise.py b/test_autofit/unit/mapper/promise/test_promise.py index bf541ae76..b654a8dab 100644 --- a/test_autofit/unit/mapper/promise/test_promise.py +++ b/test_autofit/unit/mapper/promise/test_promise.py @@ -174,7 +174,7 @@ def test_does_not_contribute_to_prior_count( assert model.prior_count == 0 def test_model_promise(self, model_promise, phase): - assert isinstance(model_promise, af.Promise) + assert isinstance(model_promise, af.prior.Promise) assert model_promise.path == ("one", "redshift") assert model_promise.is_instance is False assert model_promise._phase is phase @@ -190,7 +190,7 @@ def test_optional_in_sub(self, collection, phase): assert result is None def test_instance_promise(self, instance_promise, phase): - assert isinstance(instance_promise, af.Promise) + assert isinstance(instance_promise, af.prior.Promise) assert instance_promise.path == ("one", "redshift") assert instance_promise.is_instance is True assert instance_promise._phase is phase @@ -239,13 +239,13 @@ def test_kwarg_promise(self, profile_promise, collection): def test_embedded_results(self, phase, collection): hyper_result = phase.result.hyper_result - assert isinstance(hyper_result, af.PromiseResult) + assert isinstance(hyper_result, af.prior.PromiseResult) model_promise = hyper_result.model instance_promise = hyper_result.instance - assert isinstance(model_promise.hyper_galaxy, af.Promise) - assert isinstance(instance_promise.hyper_galaxy, af.Promise) + assert isinstance(model_promise.hyper_galaxy, af.prior.Promise) + assert isinstance(instance_promise.hyper_galaxy, af.prior.Promise) model = model_promise.populate(collection) instance = instance_promise.populate(collection) diff --git a/test_autofit/unit/mapper/test_assertion.py b/test_autofit/unit/mapper/test_assertion.py index 21c995347..7cfe4156b 100644 --- a/test_autofit/unit/mapper/test_assertion.py +++ b/test_autofit/unit/mapper/test_assertion.py @@ -119,29 +119,29 @@ def make_model(collection): class TestPromiseAssertion: def test_less_than(self, promise_model, collection, model): promise = promise_model.axis_ratio < promise_model.phi - assert isinstance(promise, af.GreaterThanLessThanAssertion) + assert isinstance(promise, af.prior.GreaterThanLessThanAssertion) assertion = promise.populate(collection) - assert isinstance(assertion, af.GreaterThanLessThanAssertion) + assert isinstance(assertion, af.prior.GreaterThanLessThanAssertion) def test_greater_than(self, promise_model, collection, model): promise = promise_model.axis_ratio > promise_model.phi - assert isinstance(promise, af.GreaterThanLessThanAssertion) + assert isinstance(promise, af.prior.GreaterThanLessThanAssertion) def test_greater_than_equal(self, promise_model, collection, model): promise = promise_model.axis_ratio >= promise_model.phi - assert isinstance(promise, af.GreaterThanLessThanEqualAssertion) + assert isinstance(promise, af.prior.GreaterThanLessThanEqualAssertion) def test_integer_promise_assertion(self, promise_model, collection, model): promise = promise_model.axis_ratio > 1.0 - assert isinstance(promise, af.GreaterThanLessThanAssertion) + assert isinstance(promise, af.prior.GreaterThanLessThanAssertion) def test_compound_assertion(self, promise_model, collection, model): promise = (1.0 < promise_model.axis_ratio) < 1.0 - assert isinstance(promise, af.CompoundAssertion) + assert isinstance(promise, af.prior.CompoundAssertion) assertion = promise.populate(collection) - assert isinstance(assertion, af.CompoundAssertion) + assert isinstance(assertion, af.prior.CompoundAssertion) class TestModel: diff --git a/test_autofit/unit/mapper/test_prior_parsing.py b/test_autofit/unit/mapper/test_prior_parsing.py index 6aa4df165..9e6704865 100644 --- a/test_autofit/unit/mapper/test_prior_parsing.py +++ b/test_autofit/unit/mapper/test_prior_parsing.py @@ -52,21 +52,21 @@ def make_absolute_width_dict(): @pytest.fixture(name="relative_width_modifier") def make_relative_width_modifier(relative_width_dict): - return af.WidthModifier.from_dict(relative_width_dict) + return af.prior.WidthModifier.from_dict(relative_width_dict) @pytest.fixture(name="absolute_width_modifier") def make_absolute_width_modifier(absolute_width_dict): - return af.WidthModifier.from_dict(absolute_width_dict) + return af.prior.WidthModifier.from_dict(absolute_width_dict) class TestWidth: def test_relative(self, relative_width_modifier): - assert isinstance(relative_width_modifier, af.RelativeWidthModifier) + assert isinstance(relative_width_modifier, af.prior.RelativeWidthModifier) assert relative_width_modifier.value == 1.0 def test_absolute(self, absolute_width_modifier): - assert isinstance(absolute_width_modifier, af.AbsoluteWidthModifier) + assert isinstance(absolute_width_modifier, af.prior.AbsoluteWidthModifier) assert absolute_width_modifier.value == 2.0 From 5ed5eccb5c06f35dae8229d7bc483e890340d7af Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 09:22:48 +0100 Subject: [PATCH 03/14] no more general imports --- autofit/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/autofit/__init__.py b/autofit/__init__.py index 9ce50195a..7514cb90b 100644 --- a/autofit/__init__.py +++ b/autofit/__init__.py @@ -18,7 +18,6 @@ from .mapper.model_mapper import ModelMapper from .mapper.model_mapper import ModelMapper as Mapper from .mapper.model_object import ModelObject -from .mapper.prior_model import * from .mapper.prior_model.abstract import AbstractPriorModel from .mapper.prior_model.annotation import AnnotationPriorModel from .mapper.prior_model.collection import CollectionPriorModel From 5289f6596bad20afee972dd3ce0e46fff90e9618 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 09:41:22 +0100 Subject: [PATCH 04/14] match the as target --- scripts/edenise.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 scripts/edenise.py diff --git a/scripts/edenise.py b/scripts/edenise.py new file mode 100644 index 000000000..8c79c8c13 --- /dev/null +++ b/scripts/edenise.py @@ -0,0 +1,61 @@ +from pathlib import Path + +import pytest +import re + + +class Line: + def __init__(self, string): + self.string = string + + @property + def is_import(self): + return self.string.startswith("from") + + @property + def source(self): + return re.match(".* as (.+)", self.string).group(1) + + @property + def target(self): + pass + + +class Converter: + def __init__(self, source_directory): + self.source_directory = source_directory + with open(self._init_directory) as f: + lines = map(Line, f.readlines()) + + @property + def _init_directory(self): + return f"{self.source_directory}/__init__.py" + + +@pytest.fixture( + name="as_line" +) +def make_as_line(): + return Line( + "from .mapper.model import ModelInstance as Instance" + ) + + +class Test: + def test_line_is_import(self, as_line): + assert as_line.is_import + assert not Line(".mapper.model").is_import + + def test_source(self, as_line): + assert as_line.source == "Instance" + # assert Line( + # "from .mapper.model import ModelInstance" + # ).source == "ModelInstance" + + +if __name__ == "__main__": + root_directory = Path(__file__).parent.parent + + converter = Converter( + source_directory=root_directory / "autofit" + ) From 48328c2a78d9b7a650237d4b009474a242ffb517 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 09:50:11 +0100 Subject: [PATCH 05/14] sources --- scripts/edenise.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/edenise.py b/scripts/edenise.py index 8c79c8c13..065e9580c 100644 --- a/scripts/edenise.py +++ b/scripts/edenise.py @@ -1,7 +1,7 @@ +import re from pathlib import Path import pytest -import re class Line: @@ -14,7 +14,10 @@ def is_import(self): @property def source(self): - return re.match(".* as (.+)", self.string).group(1) + match = re.match(".* as (.+)", self.string) + if match is not None: + return match.group(1) + return re.match(".* import (.+)", self.string).group(1) @property def target(self): @@ -48,9 +51,9 @@ def test_line_is_import(self, as_line): def test_source(self, as_line): assert as_line.source == "Instance" - # assert Line( - # "from .mapper.model import ModelInstance" - # ).source == "ModelInstance" + assert Line( + "from .mapper.model import ModelInstance" + ).source == "ModelInstance" if __name__ == "__main__": From 7f4b9f4e4d0d708efbe25096ffb48e3aebf7a5c0 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 09:57:51 +0100 Subject: [PATCH 06/14] from source to target --- scripts/edenise.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/scripts/edenise.py b/scripts/edenise.py index 065e9580c..d1a193442 100644 --- a/scripts/edenise.py +++ b/scripts/edenise.py @@ -14,14 +14,23 @@ def is_import(self): @property def source(self): - match = re.match(".* as (.+)", self.string) + match = re.match("from .* as (.+)", self.string) if match is not None: return match.group(1) - return re.match(".* import (.+)", self.string).group(1) + return re.match("from .* import (.+)", self.string).group(1) @property def target(self): - pass + return self.string.replace( + f" as {self.source}", + "" + ).replace( + "from ", + "" + ).replace( + " import ", + "." + ) class Converter: @@ -44,16 +53,27 @@ def make_as_line(): ) +@pytest.fixture( + name="line" +) +def make_line(): + return Line( + "from .mapper.model import ModelInstance" + ) + + class Test: def test_line_is_import(self, as_line): assert as_line.is_import assert not Line(".mapper.model").is_import - def test_source(self, as_line): + def test_source(self, as_line, line): assert as_line.source == "Instance" - assert Line( - "from .mapper.model import ModelInstance" - ).source == "ModelInstance" + assert line.source == "ModelInstance" + + def test_target(self, as_line, line): + assert as_line.target == ".mapper.model.ModelInstance" + assert line.target == ".mapper.model.ModelInstance" if __name__ == "__main__": From 4afefa643b6f009f2618d3118e7fcd0af7c5df7d Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 10:12:49 +0100 Subject: [PATCH 07/14] conversion of strings --- scripts/edenise.py | 59 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/scripts/edenise.py b/scripts/edenise.py index d1a193442..d9445a167 100644 --- a/scripts/edenise.py +++ b/scripts/edenise.py @@ -30,18 +30,41 @@ def target(self): ).replace( " import ", "." + ).lstrip( + "." ) class Converter: - def __init__(self, source_directory): - self.source_directory = source_directory - with open(self._init_directory) as f: + def __init__(self, prefix, lines): + self.prefix = prefix + self.lines = lines + + @classmethod + def from_prefix_and_source_directory( + cls, + prefix, + source_directory + ): + source_directory = source_directory + with open( + f"{source_directory}/__init__.py" + ) as f: lines = map(Line, f.readlines()) - - @property - def _init_directory(self): - return f"{self.source_directory}/__init__.py" + return Converter(prefix, lines) + + def convert(self, string): + print(string) + for line in self.lines: + source = f"{self.prefix}.{line.source}" + target = f"{self.prefix}.{line.target}" + print(source) + print(target) + string = string.replace( + source, + target + ) + return string @pytest.fixture( @@ -72,13 +95,27 @@ def test_source(self, as_line, line): assert line.source == "ModelInstance" def test_target(self, as_line, line): - assert as_line.target == ".mapper.model.ModelInstance" - assert line.target == ".mapper.model.ModelInstance" + assert as_line.target == "mapper.model.ModelInstance" + assert line.target == "mapper.model.ModelInstance" + def test_replace(self, as_line, line): + converter = Converter( + "af", + [as_line, line] + ) + assert converter.convert( + "af.ModelInstance\naf.Instance" + ) == "af.mapper.model.ModelInstance\naf.mapper.model.ModelInstance" + -if __name__ == "__main__": +def main(): root_directory = Path(__file__).parent.parent - converter = Converter( + converter = Converter.from_prefix_and_source_directory( + prefix="af", source_directory=root_directory / "autofit" ) + + +if __name__ == "__main__": + main() From 3d1903b63678265ba4e1747ca04de858d61727b0 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 10:39:48 +0100 Subject: [PATCH 08/14] attempting to copy and convert --- scripts/edenise.py | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) mode change 100644 => 100755 scripts/edenise.py diff --git a/scripts/edenise.py b/scripts/edenise.py old mode 100644 new mode 100755 index d9445a167..6844d01dc --- a/scripts/edenise.py +++ b/scripts/edenise.py @@ -1,11 +1,17 @@ +#!/usr/bin/env python + import re -from pathlib import Path +import shutil +from os import path, walk import pytest class Line: def __init__(self, string): + if "*" in string: + print("Please ensure no imports in the __init__ contain a *") + exit(1) self.string = string @property @@ -54,12 +60,9 @@ def from_prefix_and_source_directory( return Converter(prefix, lines) def convert(self, string): - print(string) for line in self.lines: source = f"{self.prefix}.{line.source}" target = f"{self.prefix}.{line.target}" - print(source) - print(target) string = string.replace( source, target @@ -106,16 +109,41 @@ def test_replace(self, as_line, line): assert converter.convert( "af.ModelInstance\naf.Instance" ) == "af.mapper.model.ModelInstance\naf.mapper.model.ModelInstance" - + def main(): - root_directory = Path(__file__).parent.parent + root_directory = f"{path.dirname(path.realpath(__file__))}/.." + + name = "autofit" + prefix = "af" + + target_directory = f"{root_directory}/../{name}_eden" + + print(f"Creating {target_directory}...") + shutil.copytree( + root_directory, + target_directory, + symlinks=True + ) converter = Converter.from_prefix_and_source_directory( - prefix="af", - source_directory=root_directory / "autofit" + prefix=prefix, + source_directory=f"{root_directory}/{name}" ) + for root, _, files in walk(f"{target_directory}/test_{name}"): + for file in files: + if file.endswith(".py"): + with open(f"{root}/{file}", "r+") as f: + string = f.read() + f.seek(0) + f.write( + converter.convert( + string + ) + ) + f.truncate() + if __name__ == "__main__": main() From fa0874b3c1d58a4ab0c1e4c2b92af681a11e755a Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 10:55:28 +0100 Subject: [PATCH 09/14] mostly successful conversion --- scripts/edenise.py | 55 +++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/scripts/edenise.py b/scripts/edenise.py index 6844d01dc..5620c8841 100755 --- a/scripts/edenise.py +++ b/scripts/edenise.py @@ -12,7 +12,13 @@ def __init__(self, string): if "*" in string: print("Please ensure no imports in the __init__ contain a *") exit(1) - self.string = string + self.string = string.replace("\n", "") + + def __str__(self): + return f"{self.source} -> {self.target}" + + def __repr__(self): + return f"<{self.__class__.__name__} {self}>" @property def is_import(self): @@ -44,7 +50,10 @@ def target(self): class Converter: def __init__(self, prefix, lines): self.prefix = prefix - self.lines = lines + self.lines = list(filter( + lambda line: line.is_import, + lines + )) @classmethod def from_prefix_and_source_directory( @@ -88,27 +97,27 @@ def make_line(): ) -class Test: - def test_line_is_import(self, as_line): - assert as_line.is_import - assert not Line(".mapper.model").is_import - - def test_source(self, as_line, line): - assert as_line.source == "Instance" - assert line.source == "ModelInstance" - - def test_target(self, as_line, line): - assert as_line.target == "mapper.model.ModelInstance" - assert line.target == "mapper.model.ModelInstance" - - def test_replace(self, as_line, line): - converter = Converter( - "af", - [as_line, line] - ) - assert converter.convert( - "af.ModelInstance\naf.Instance" - ) == "af.mapper.model.ModelInstance\naf.mapper.model.ModelInstance" +# class Test: +# def test_line_is_import(self, as_line): +# assert as_line.is_import +# assert not Line(".mapper.model").is_import +# +# def test_source(self, as_line, line): +# assert as_line.source == "Instance" +# assert line.source == "ModelInstance" +# +# def test_target(self, as_line, line): +# assert as_line.target == "mapper.model.ModelInstance" +# assert line.target == "mapper.model.ModelInstance" +# +# def test_replace(self, as_line, line): +# converter = Converter( +# "af", +# [as_line, line] +# ) +# assert converter.convert( +# "af.ModelInstance\naf.Instance" +# ) == "af.mapper.model.ModelInstance\naf.mapper.model.ModelInstance" def main(): From 9f2901b76f476f5b95ec8cab65e33ca1f7dd1f8e Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 11:03:33 +0100 Subject: [PATCH 10/14] separate source, tests and script --- autofit/tools/edenise.py | 108 +++++++++++++++++ scripts/edenise.py | 148 +----------------------- test_autofit/unit/tools/test_edenise.py | 44 +++++++ 3 files changed, 157 insertions(+), 143 deletions(-) create mode 100644 autofit/tools/edenise.py create mode 100644 test_autofit/unit/tools/test_edenise.py diff --git a/autofit/tools/edenise.py b/autofit/tools/edenise.py new file mode 100644 index 000000000..be8d5b457 --- /dev/null +++ b/autofit/tools/edenise.py @@ -0,0 +1,108 @@ +import re +import shutil +from os import walk + + +class Line: + def __init__(self, string): + if "*" in string: + print("Please ensure no imports in the __init__ contain a *") + exit(1) + self.string = string.replace("\n", "") + + def __str__(self): + return f"{self.source} -> {self.target}" + + def __repr__(self): + return f"<{self.__class__.__name__} {self}>" + + @property + def is_import(self): + return self.string.startswith("from") + + @property + def source(self): + match = re.match("from .* as (.+)", self.string) + if match is not None: + return match.group(1) + return re.match("from .* import (.+)", self.string).group(1) + + @property + def target(self): + return self.string.replace( + f" as {self.source}", + "" + ).replace( + "from ", + "" + ).replace( + " import ", + "." + ).lstrip( + "." + ) + + +class Converter: + def __init__(self, prefix, lines): + self.prefix = prefix + self.lines = list(filter( + lambda line: line.is_import, + lines + )) + + @classmethod + def from_prefix_and_source_directory( + cls, + prefix, + source_directory + ): + source_directory = source_directory + with open( + f"{source_directory}/__init__.py" + ) as f: + lines = map(Line, f.readlines()) + return Converter(prefix, lines) + + def convert(self, string): + for line in self.lines: + source = f"{self.prefix}.{line.source}" + target = f"{self.prefix}.{line.target}" + string = string.replace( + source, + target + ) + return string + + +def edenise( + root_directory, + name, + prefix +): + target_directory = f"{root_directory}/../{name}_eden" + + print(f"Creating {target_directory}...") + shutil.copytree( + root_directory, + target_directory, + symlinks=True + ) + + converter = Converter.from_prefix_and_source_directory( + prefix=prefix, + source_directory=f"{root_directory}/{name}" + ) + + for root, _, files in walk(f"{target_directory}/test_{name}"): + for file in files: + if file.endswith(".py"): + with open(f"{root}/{file}", "r+") as f: + string = f.read() + f.seek(0) + f.write( + converter.convert( + string + ) + ) + f.truncate() diff --git a/scripts/edenise.py b/scripts/edenise.py index 5620c8841..55ba09999 100755 --- a/scripts/edenise.py +++ b/scripts/edenise.py @@ -1,123 +1,7 @@ #!/usr/bin/env python -import re -import shutil -from os import path, walk - -import pytest - - -class Line: - def __init__(self, string): - if "*" in string: - print("Please ensure no imports in the __init__ contain a *") - exit(1) - self.string = string.replace("\n", "") - - def __str__(self): - return f"{self.source} -> {self.target}" - - def __repr__(self): - return f"<{self.__class__.__name__} {self}>" - - @property - def is_import(self): - return self.string.startswith("from") - - @property - def source(self): - match = re.match("from .* as (.+)", self.string) - if match is not None: - return match.group(1) - return re.match("from .* import (.+)", self.string).group(1) - - @property - def target(self): - return self.string.replace( - f" as {self.source}", - "" - ).replace( - "from ", - "" - ).replace( - " import ", - "." - ).lstrip( - "." - ) - - -class Converter: - def __init__(self, prefix, lines): - self.prefix = prefix - self.lines = list(filter( - lambda line: line.is_import, - lines - )) - - @classmethod - def from_prefix_and_source_directory( - cls, - prefix, - source_directory - ): - source_directory = source_directory - with open( - f"{source_directory}/__init__.py" - ) as f: - lines = map(Line, f.readlines()) - return Converter(prefix, lines) - - def convert(self, string): - for line in self.lines: - source = f"{self.prefix}.{line.source}" - target = f"{self.prefix}.{line.target}" - string = string.replace( - source, - target - ) - return string - - -@pytest.fixture( - name="as_line" -) -def make_as_line(): - return Line( - "from .mapper.model import ModelInstance as Instance" - ) - - -@pytest.fixture( - name="line" -) -def make_line(): - return Line( - "from .mapper.model import ModelInstance" - ) - - -# class Test: -# def test_line_is_import(self, as_line): -# assert as_line.is_import -# assert not Line(".mapper.model").is_import -# -# def test_source(self, as_line, line): -# assert as_line.source == "Instance" -# assert line.source == "ModelInstance" -# -# def test_target(self, as_line, line): -# assert as_line.target == "mapper.model.ModelInstance" -# assert line.target == "mapper.model.ModelInstance" -# -# def test_replace(self, as_line, line): -# converter = Converter( -# "af", -# [as_line, line] -# ) -# assert converter.convert( -# "af.ModelInstance\naf.Instance" -# ) == "af.mapper.model.ModelInstance\naf.mapper.model.ModelInstance" +from os import path +from autofit.tools import edenise def main(): @@ -125,34 +9,12 @@ def main(): name = "autofit" prefix = "af" - - target_directory = f"{root_directory}/../{name}_eden" - - print(f"Creating {target_directory}...") - shutil.copytree( + edenise.edenise( root_directory, - target_directory, - symlinks=True + name, + prefix ) - converter = Converter.from_prefix_and_source_directory( - prefix=prefix, - source_directory=f"{root_directory}/{name}" - ) - - for root, _, files in walk(f"{target_directory}/test_{name}"): - for file in files: - if file.endswith(".py"): - with open(f"{root}/{file}", "r+") as f: - string = f.read() - f.seek(0) - f.write( - converter.convert( - string - ) - ) - f.truncate() - if __name__ == "__main__": main() diff --git a/test_autofit/unit/tools/test_edenise.py b/test_autofit/unit/tools/test_edenise.py new file mode 100644 index 000000000..4b2b76e99 --- /dev/null +++ b/test_autofit/unit/tools/test_edenise.py @@ -0,0 +1,44 @@ +import pytest + +from autofit.tools.edenise import Line, Converter + + +@pytest.fixture( + name="as_line" +) +def make_as_line(): + return Line( + "from .mapper.model import ModelInstance as Instance" + ) + + +@pytest.fixture( + name="line" +) +def make_line(): + return Line( + "from .mapper.model import ModelInstance" + ) + + +class Test: + def test_line_is_import(self, as_line): + assert as_line.is_import + assert not Line(".mapper.model").is_import + + def test_source(self, as_line, line): + assert as_line.source == "Instance" + assert line.source == "ModelInstance" + + def test_target(self, as_line, line): + assert as_line.target == "mapper.model.ModelInstance" + assert line.target == "mapper.model.ModelInstance" + + def test_replace(self, as_line, line): + converter = Converter( + "af", + [as_line, line] + ) + assert converter.convert( + "af.ModelInstance\naf.Instance" + ) == "af.mapper.model.ModelInstance\naf.mapper.model.ModelInstance" From c6a96542125e4d0fd0f9f73fe83b466554988e8d Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 11:17:17 +0100 Subject: [PATCH 11/14] sort lines so that longer statements are replaced first. intermediate id placeholders prevent double replacement --- autofit/tools/edenise.py | 29 ++++++++++++++++++++----- test_autofit/unit/tools/test_edenise.py | 7 ++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/autofit/tools/edenise.py b/autofit/tools/edenise.py index be8d5b457..968cad006 100644 --- a/autofit/tools/edenise.py +++ b/autofit/tools/edenise.py @@ -1,6 +1,7 @@ import re import shutil from os import walk +from uuid import uuid1 class Line: @@ -9,6 +10,7 @@ def __init__(self, string): print("Please ensure no imports in the __init__ contain a *") exit(1) self.string = string.replace("\n", "") + self.id = str(uuid1()) def __str__(self): return f"{self.source} -> {self.target}" @@ -16,6 +18,15 @@ def __str__(self): def __repr__(self): return f"<{self.__class__.__name__} {self}>" + def __len__(self): + return len(self.source) + + def __lt__(self, other): + return len(self) < len(other) + + def __gt__(self, other): + return len(self) > len(other) + @property def is_import(self): return self.string.startswith("from") @@ -46,10 +57,13 @@ def target(self): class Converter: def __init__(self, prefix, lines): self.prefix = prefix - self.lines = list(filter( - lambda line: line.is_import, - lines - )) + self.lines = sorted( + filter( + lambda line: line.is_import, + lines + ), + reverse=True + ) @classmethod def from_prefix_and_source_directory( @@ -67,9 +81,14 @@ def from_prefix_and_source_directory( def convert(self, string): for line in self.lines: source = f"{self.prefix}.{line.source}" - target = f"{self.prefix}.{line.target}" string = string.replace( source, + line.id + ) + for line in self.lines: + target = f"{self.prefix}.{line.target}" + string = string.replace( + line.id, target ) return string diff --git a/test_autofit/unit/tools/test_edenise.py b/test_autofit/unit/tools/test_edenise.py index 4b2b76e99..ef9061962 100644 --- a/test_autofit/unit/tools/test_edenise.py +++ b/test_autofit/unit/tools/test_edenise.py @@ -34,6 +34,13 @@ def test_target(self, as_line, line): assert as_line.target == "mapper.model.ModelInstance" assert line.target == "mapper.model.ModelInstance" + def test_phase_property_line(self): + line = Line( + "from .tools.phase_property import PhaseProperty" + ) + assert line.source == "PhaseProperty" + assert line.target == "tools.phase_property.PhaseProperty" + def test_replace(self, as_line, line): converter = Converter( "af", From 9366fa08928c0a87f2ace588c96a81f8354cdf7f Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 11:37:49 +0100 Subject: [PATCH 12/14] autoconf extended into autofit, imports updated --- autofit/__init__.py | 6 +- autofit/conf.py | 1 + autofit/mapper/prior/prior.py | 2 +- test_autofit/conftest.py | 2 +- .../multinest/config/json_priors/mock.json | 4 +- .../config/json_priors/test_autofit.json | 80 +++++++++++++------ .../optimize/nested_sampler/test_dynesty.py | 2 +- .../nested_sampler/test_multi_nest.py | 2 +- test_autofit/unit/optimize/test_emcee.py | 2 +- test_autofit/unit/optimize/test_non_linear.py | 2 +- 10 files changed, 65 insertions(+), 38 deletions(-) create mode 100644 autofit/conf.py diff --git a/autofit/__init__.py b/autofit/__init__.py index 7514cb90b..600db31f2 100644 --- a/autofit/__init__.py +++ b/autofit/__init__.py @@ -1,13 +1,11 @@ -from autoconf import conf -from autofit.mapper.model import path_instances_of_class -from autofit.mapper.prior_model.attribute_pair import ( +from .mapper.model import path_instances_of_class +from .mapper.prior_model.attribute_pair import ( cast_collection, AttributeNameValue, PriorNameValue, InstanceNameValue, ) -dir(conf) from . import exc from .optimize.non_linear.samples import AbstractSamples, MCMCSamples, NestedSamplerSamples from .aggregator import Aggregator, PhaseOutput diff --git a/autofit/conf.py b/autofit/conf.py new file mode 100644 index 000000000..e67960ff1 --- /dev/null +++ b/autofit/conf.py @@ -0,0 +1 @@ +from autoconf.conf import * \ No newline at end of file diff --git a/autofit/mapper/prior/prior.py b/autofit/mapper/prior/prior.py index 2e948d7e2..566460c2c 100644 --- a/autofit/mapper/prior/prior.py +++ b/autofit/mapper/prior/prior.py @@ -7,7 +7,7 @@ import numpy as np from scipy.special import erfcinv -from autoconf import conf +from autofit import conf from autofit import exc from autofit.mapper.model_object import ModelObject from autofit.mapper.prior.arithmetic import ArithmeticMixin diff --git a/test_autofit/conftest.py b/test_autofit/conftest.py index 95833788c..b68a23e53 100644 --- a/test_autofit/conftest.py +++ b/test_autofit/conftest.py @@ -3,7 +3,7 @@ import pytest import autofit as af -from autoconf import conf +from autofit import conf import shutil directory = path.dirname(path.realpath(__file__)) diff --git a/test_autofit/unit/optimize/nested_sampler/files/multinest/config/json_priors/mock.json b/test_autofit/unit/optimize/nested_sampler/files/multinest/config/json_priors/mock.json index 67a857e96..38d902074 100644 --- a/test_autofit/unit/optimize/nested_sampler/files/multinest/config/json_priors/mock.json +++ b/test_autofit/unit/optimize/nested_sampler/files/multinest/config/json_priors/mock.json @@ -612,9 +612,7 @@ } }, "Tracer": { - "grid": { - "type": "Deferred" - } + "grid.type": "Deferred" }, "Circle": { "circumference": { diff --git a/test_autofit/unit/optimize/nested_sampler/files/multinest/config/json_priors/test_autofit.json b/test_autofit/unit/optimize/nested_sampler/files/multinest/config/json_priors/test_autofit.json index b4b14be08..3bca9903a 100644 --- a/test_autofit/unit/optimize/nested_sampler/files/multinest/config/json_priors/test_autofit.json +++ b/test_autofit/unit/optimize/nested_sampler/files/multinest/config/json_priors/test_autofit.json @@ -1,33 +1,63 @@ { - "mock.Tracer": { - "lens_galaxy": { - "type": "Uniform", - "lower_limit": 0.0, - "upper_limit": 1.0, - "width_modifier": { - "type": "Absolute", - "value": 0.2 + "mock": { + "Tracer": { + "lens_galaxy": { + "type": "Uniform", + "lower_limit": 0.0, + "upper_limit": 1.0, + "width_modifier": { + "type": "Absolute", + "value": 0.2 + }, + "gaussian_limits": { + "lower": 0.0, + "upper": 1.0 + } }, - "gaussian_limits": { - "lower": 0.0, - "upper": 1.0 + "source_galaxy": { + "type": "Uniform", + "lower_limit": 0.0, + "upper_limit": 1.0, + "width_modifier": { + "type": "Absolute", + "value": 0.2 + }, + "gaussian_limits": { + "lower": 0.0, + "upper": 1.0 + } + }, + "grid": { + "type": "Deferred" } }, - "source_galaxy": { - "type": "Uniform", - "lower_limit": 0.0, - "upper_limit": 1.0, - "width_modifier": { - "type": "Absolute", - "value": 0.2 + "MockClassNLOx2": { + "one": { + "type": "Uniform", + "lower_limit": 0.0, + "upper_limit": 1.0, + "width_modifier": { + "type": "Absolute", + "value": 0.2 + }, + "gaussian_limits": { + "lower": 0.0, + "upper": 1.0 + } }, - "gaussian_limits": { - "lower": 0.0, - "upper": 1.0 + "two": { + "type": "Uniform", + "lower_limit": 0.0, + "upper_limit": 1.0, + "width_modifier": { + "type": "Absolute", + "value": 0.2 + }, + "gaussian_limits": { + "lower": 0.0, + "upper": 1.0 + } } - }, - "grid": { - "type": "Deferred" } } -} +} \ No newline at end of file diff --git a/test_autofit/unit/optimize/nested_sampler/test_dynesty.py b/test_autofit/unit/optimize/nested_sampler/test_dynesty.py index 35a085b25..fda4772fc 100644 --- a/test_autofit/unit/optimize/nested_sampler/test_dynesty.py +++ b/test_autofit/unit/optimize/nested_sampler/test_dynesty.py @@ -3,7 +3,7 @@ import pytest from autofit import Paths -from autoconf import conf +from autofit import conf import autofit as af import pickle from test_autofit.mock import MockClassNLOx4 diff --git a/test_autofit/unit/optimize/nested_sampler/test_multi_nest.py b/test_autofit/unit/optimize/nested_sampler/test_multi_nest.py index 5470c7e43..11e70992a 100644 --- a/test_autofit/unit/optimize/nested_sampler/test_multi_nest.py +++ b/test_autofit/unit/optimize/nested_sampler/test_multi_nest.py @@ -4,7 +4,7 @@ import pytest -from autoconf import conf +from autofit import conf import autofit as af from autofit import Paths from autofit.optimize.non_linear.nested_sampling import multi_nest as mn diff --git a/test_autofit/unit/optimize/test_emcee.py b/test_autofit/unit/optimize/test_emcee.py index 60f907afb..a1073e3dd 100644 --- a/test_autofit/unit/optimize/test_emcee.py +++ b/test_autofit/unit/optimize/test_emcee.py @@ -1,7 +1,7 @@ import os import pytest -from autoconf import conf +from autofit import conf import autofit as af from autofit import Paths from test_autofit.mock import MockClassNLOx4 diff --git a/test_autofit/unit/optimize/test_non_linear.py b/test_autofit/unit/optimize/test_non_linear.py index 1f4ea2913..c8c77199e 100644 --- a/test_autofit/unit/optimize/test_non_linear.py +++ b/test_autofit/unit/optimize/test_non_linear.py @@ -4,7 +4,7 @@ import pytest import autofit as af -from autoconf import conf +from autofit import conf from autofit import Paths from autofit.optimize.non_linear.mock_nlo import MockSamples from test_autofit.mock import ( From dab0094ab9eded59ca8b1ad40dd7b0abcca1aacd Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 11:40:08 +0100 Subject: [PATCH 13/14] clean out the init --- autofit/tools/edenise.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autofit/tools/edenise.py b/autofit/tools/edenise.py index 968cad006..b26b8fa57 100644 --- a/autofit/tools/edenise.py +++ b/autofit/tools/edenise.py @@ -125,3 +125,5 @@ def edenise( ) ) f.truncate() + + open(f"{target_directory}/{name}/__init__.py", "w+").close() From 7ed410daddd32729709c24b0589f340a208d2d89 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 May 2020 11:48:52 +0100 Subject: [PATCH 14/14] converted to cli --- scripts/edenise.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/edenise.py b/scripts/edenise.py index 55ba09999..ca7edc618 100755 --- a/scripts/edenise.py +++ b/scripts/edenise.py @@ -1,19 +1,22 @@ #!/usr/bin/env python -from os import path +from sys import argv + from autofit.tools import edenise def main(): - root_directory = f"{path.dirname(path.realpath(__file__))}/.." - - name = "autofit" - prefix = "af" - edenise.edenise( - root_directory, - name, - prefix - ) + try: + root_directory, name, prefix = argv[1:] + edenise.edenise( + root_directory, + name, + prefix + ) + except ValueError: + print("Usage: ./edenise.py root_directory project_name import_prefix") + print("e.g.: ./edenise.py /path/to/autofit autofit af") + exit(1) if __name__ == "__main__":