From bec91aae0d8963f5f415f28f5733239ec88e4051 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:29:08 +0000 Subject: [PATCH 01/15] =?UTF-8?q?refactor:=20restructure=20chapters=20?= =?UTF-8?q?=E2=80=94=20search=20chaining=20folds=20into=20chapter=202,=20p?= =?UTF-8?q?ixelizations=20become=20chapter=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the HowToLens restructure: chapter 3 (search chaining) is dissolved — tutorials 1 and 2 move to the end of chapter 2 as tutorials 9 (search chaining) and 10 (prior passing); the x2-galaxies chaining tutorial is deleted (superseded by the new scaling-up chapter's multi-galaxy tutorial). Chapter 4 (pixelizations) is renamed chapter 3 with model_fit renumbered 5->6 to make room for a new tutorial 5 (bayesian_formalism, added next). An empty chapter_4_scaling_up_galaxies is scaffolded; the chapter 2 summary file moves past the new tutorials. File moves only; content updates land in follow-up commits. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- .../tutorial_10_prior_passing.ipynb} | 0 .../tutorial_9_search_chaining.ipynb} | 0 .../README.md | 0 .../tutorial_1_pixelizations.ipynb | 0 .../tutorial_2_mappers.ipynb | 0 .../tutorial_3_inversions.ipynb | 0 .../tutorial_4_bayesian_regularization.ipynb | 0 .../tutorial_6_model_fit.ipynb} | 0 notebooks/chapter_3_search_chaining/README.md | 12 - .../tutorial_3_x2_galaxies.ipynb | 518 ------------------ .../tutorial_10_prior_passing.py} | 0 ...tutorial_9_summary => tutorial_11_summary} | 0 .../tutorial_9_search_chaining.py} | 0 .../README.md | 0 .../__init__.py | 0 .../introduction | 0 .../tutorial_1_pixelizations.py | 0 .../tutorial_2_mappers.py | 0 .../tutorial_3_inversions.py | 0 .../tutorial_4_bayesian_regularization.py | 0 .../tutorial_6_model_fit.py} | 0 scripts/chapter_3_search_chaining/README.md | 12 - .../chapter_3_search_chaining/introduction | 12 - .../tutorial_3_x2_galaxies.py | 286 ---------- .../__init__.py | 0 25 files changed, 840 deletions(-) rename notebooks/{chapter_3_search_chaining/tutorial_2_prior_passing.ipynb => chapter_2_modeling/tutorial_10_prior_passing.ipynb} (100%) rename notebooks/{chapter_3_search_chaining/tutorial_1_search_chaining.ipynb => chapter_2_modeling/tutorial_9_search_chaining.ipynb} (100%) rename notebooks/{chapter_4_pixelizations => chapter_3_pixelizations}/README.md (100%) rename notebooks/{chapter_4_pixelizations => chapter_3_pixelizations}/tutorial_1_pixelizations.ipynb (100%) rename notebooks/{chapter_4_pixelizations => chapter_3_pixelizations}/tutorial_2_mappers.ipynb (100%) rename notebooks/{chapter_4_pixelizations => chapter_3_pixelizations}/tutorial_3_inversions.ipynb (100%) rename notebooks/{chapter_4_pixelizations => chapter_3_pixelizations}/tutorial_4_bayesian_regularization.ipynb (100%) rename notebooks/{chapter_4_pixelizations/tutorial_5_model_fit.ipynb => chapter_3_pixelizations/tutorial_6_model_fit.ipynb} (100%) delete mode 100644 notebooks/chapter_3_search_chaining/README.md delete mode 100644 notebooks/chapter_3_search_chaining/tutorial_3_x2_galaxies.ipynb rename scripts/{chapter_3_search_chaining/tutorial_2_prior_passing.py => chapter_2_modeling/tutorial_10_prior_passing.py} (100%) rename scripts/chapter_2_modeling/{tutorial_9_summary => tutorial_11_summary} (100%) rename scripts/{chapter_3_search_chaining/tutorial_1_search_chaining.py => chapter_2_modeling/tutorial_9_search_chaining.py} (100%) rename scripts/{chapter_4_pixelizations => chapter_3_pixelizations}/README.md (100%) rename scripts/{chapter_3_search_chaining => chapter_3_pixelizations}/__init__.py (100%) rename scripts/{chapter_4_pixelizations => chapter_3_pixelizations}/introduction (100%) rename scripts/{chapter_4_pixelizations => chapter_3_pixelizations}/tutorial_1_pixelizations.py (100%) rename scripts/{chapter_4_pixelizations => chapter_3_pixelizations}/tutorial_2_mappers.py (100%) rename scripts/{chapter_4_pixelizations => chapter_3_pixelizations}/tutorial_3_inversions.py (100%) rename scripts/{chapter_4_pixelizations => chapter_3_pixelizations}/tutorial_4_bayesian_regularization.py (100%) rename scripts/{chapter_4_pixelizations/tutorial_5_model_fit.py => chapter_3_pixelizations/tutorial_6_model_fit.py} (100%) delete mode 100644 scripts/chapter_3_search_chaining/README.md delete mode 100755 scripts/chapter_3_search_chaining/introduction delete mode 100644 scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py rename scripts/{chapter_4_pixelizations => chapter_4_scaling_up_galaxies}/__init__.py (100%) mode change 100755 => 100644 diff --git a/notebooks/chapter_3_search_chaining/tutorial_2_prior_passing.ipynb b/notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb similarity index 100% rename from notebooks/chapter_3_search_chaining/tutorial_2_prior_passing.ipynb rename to notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb diff --git a/notebooks/chapter_3_search_chaining/tutorial_1_search_chaining.ipynb b/notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb similarity index 100% rename from notebooks/chapter_3_search_chaining/tutorial_1_search_chaining.ipynb rename to notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb diff --git a/notebooks/chapter_4_pixelizations/README.md b/notebooks/chapter_3_pixelizations/README.md similarity index 100% rename from notebooks/chapter_4_pixelizations/README.md rename to notebooks/chapter_3_pixelizations/README.md diff --git a/notebooks/chapter_4_pixelizations/tutorial_1_pixelizations.ipynb b/notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb similarity index 100% rename from notebooks/chapter_4_pixelizations/tutorial_1_pixelizations.ipynb rename to notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb diff --git a/notebooks/chapter_4_pixelizations/tutorial_2_mappers.ipynb b/notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb similarity index 100% rename from notebooks/chapter_4_pixelizations/tutorial_2_mappers.ipynb rename to notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb diff --git a/notebooks/chapter_4_pixelizations/tutorial_3_inversions.ipynb b/notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb similarity index 100% rename from notebooks/chapter_4_pixelizations/tutorial_3_inversions.ipynb rename to notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb diff --git a/notebooks/chapter_4_pixelizations/tutorial_4_bayesian_regularization.ipynb b/notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb similarity index 100% rename from notebooks/chapter_4_pixelizations/tutorial_4_bayesian_regularization.ipynb rename to notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb diff --git a/notebooks/chapter_4_pixelizations/tutorial_5_model_fit.ipynb b/notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb similarity index 100% rename from notebooks/chapter_4_pixelizations/tutorial_5_model_fit.ipynb rename to notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb diff --git a/notebooks/chapter_3_search_chaining/README.md b/notebooks/chapter_3_search_chaining/README.md deleted file mode 100644 index c3c0ad9..0000000 --- a/notebooks/chapter_3_search_chaining/README.md +++ /dev/null @@ -1,12 +0,0 @@ -In chapter 3, we introduce non-linear search chaining, whereby modeling pipelines are composed which each fit a -different model. - -**Colab** links to every tutorial are included. - -# Files - -- [Tutorial 1: Search Chaining](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_search_chaining/tutorial_1_search_chaining.ipynb) — Breaking the modeling procedure into a chained sequence of model-fits. - -- [Tutorial 2: Prior Passing](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_search_chaining/tutorial_2_prior_passing.ipynb) — How the results of earlier searches are passed to later searches. - -- [Tutorial 3: x2 Galaxies](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_search_chaining/tutorial_3_x2_galaxies.ipynb) — Modeling two galaxies simultaneously using chained searches. diff --git a/notebooks/chapter_3_search_chaining/tutorial_3_x2_galaxies.ipynb b/notebooks/chapter_3_search_chaining/tutorial_3_x2_galaxies.ipynb deleted file mode 100644 index 37cffa7..0000000 --- a/notebooks/chapter_3_search_chaining/tutorial_3_x2_galaxies.ipynb +++ /dev/null @@ -1,518 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Tutorial 3: Two Galaxies\n", - "========================\n", - "\n", - "Up to now, all the images we've fitted had one galaxy. However, we saw in chapter 1 that our galaxies object can\n", - "consist of multiple galaxies which each contribute to the overall emission. Multi-galaxy systems are challenging to\n", - "model, because they add an extra 5-10 parameters to the non-linear search per galaxy and, more problematically, the\n", - "degeneracies between the parameters of the light profiles of the galaxies can be severe.\n", - "\n", - "However, we can still break their analysis down using multiple searches and give ourselves a shot at getting a good\n", - "model. Here, we're going to fit a double galaxy system, fitting as much about each individual galaxy before\n", - "fitting them simultaneously.\n", - "\n", - "Up to now, I've put a focus on an analysis being generag. The script we write in this example is going to be the\n", - "opposite, specific to the image we're modeling. Fitting multiple galaxies is really difficult and writing a\n", - "pipeline that we can generalize to many galaxies isn't currently possible.\n", - "\n", - "__Contents__\n", - "\n", - "- **Initial Setup:** Load the double-galaxy dataset and apply a mask.\n", - "- **Paths:** Set the output path for chained search results.\n", - "- **Search Chaining Approach:** Strategy for fitting two galaxies using a sequence of searches.\n", - "- **Model + Search + Analysis + Model-Fit (Search 1):** Fit the first galaxy alone.\n", - "- **Result (Search 1):** Inspect the result of fitting the first galaxy.\n", - "- **Model (Search 2):** Compose a model for the second galaxy, fixing the first.\n", - "- **Search + Analysis + Model-Fit (Search 2):** Fit the second galaxy.\n", - "- **Result (Search 2):** Inspect the result of fitting the second galaxy.\n", - "- **Model + Search + Analysis + Model-Fit (Search 4):** Fit both galaxies simultaneously.\n", - "- **Result (Search 3):** Inspect the final simultaneous fit result.\n", - "- **Wrap Up:** Summary of search chaining for multi-galaxy systems." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Google Colab Setup__\n", - "\n", - "This cell sets up the environment when the notebook is run on Google Colab: it installs the\n", - "required PyAuto packages, clones the workspace (configuration files and example datasets) and\n", - "points the configuration at it. If you are running the notebook elsewhere (e.g. locally via\n", - "your own installation) it does nothing, and you can run it safely.\n", - "\n", - "Colab tip: model-fits run much faster on a GPU \u2014 enable one via \"Runtime\" -> \"Change runtime\n", - "type\" -> \"Hardware accelerator\" before running the notebook." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " import google.colab\n", - "except ImportError:\n", - " from autogalaxy import setup_colab as _setup_colab\n", - "else:\n", - " import importlib\n", - " import subprocess\n", - " import sys\n", - "\n", - " subprocess.check_call(\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"autonerves\", \"--no-deps\"]\n", - " )\n", - " _setup_colab = importlib.import_module(\"autonerves.setup_colab\")\n", - "\n", - "_setup_colab.setup(\"howtogalaxy\")" - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", - "\n", - "from pathlib import Path\n", - "import autofit as af\n", - "import autogalaxy as ag\n", - "import autogalaxy.plot as aplt" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Initial Setup__\n", - "\n", - "we'll use new galaxying data, where:\n", - "\n", - " - There are two galaxy's whose `LightProfile`'s are both `Sersic`'s." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "dataset_name = \"sersic_x2\"\n", - "dataset_path = Path(\"dataset\") / \"imaging\" / dataset_name" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Dataset Auto-Simulation__\n", - "\n", - "If the dataset does not already exist on your system, it will be created by running the corresponding\n", - "simulator script. This ensures that all example scripts can be run without manually simulating data first." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "if ag.util.dataset.should_simulate(str(dataset_path)):\n", - " import subprocess\n", - " import sys\n", - "\n", - " subprocess.run(\n", - " [sys.executable, \"scripts/simulators/sersic_x2.py\"],\n", - " check=True,\n", - " )\n", - "\n", - "\n", - "dataset = ag.Imaging.from_fits(\n", - " data_path=dataset_path / \"data.fits\",\n", - " noise_map_path=dataset_path / \"noise_map.fits\",\n", - " psf_path=dataset_path / \"psf.fits\",\n", - " pixel_scales=0.05,\n", - ")" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Mask__\n", - "\n", - "We need to choose our mask for the analysis. Given the light of both galaxies is present in the image we'll need to \n", - "include all their light in the image, so lets use a large circular mask. \n", - "\n", - "We'll use this mask in all three searches, however you could imagine customizing it on a per-search basis to speed up\n", - "the analysis." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "mask = ag.Mask2D.circular(\n", - " shape_native=dataset.shape_native, pixel_scales=dataset.pixel_scales, radius=6.0\n", - ")\n", - "\n", - "dataset = dataset.apply_mask(mask=mask)\n", - "\n", - "over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from(\n", - " grid=dataset.grid,\n", - " sub_size_list=[8, 4, 2],\n", - " radial_list=[0.3, 0.6],\n", - " centre_list=[(0.0, -1.0), (0.0, 1.0)],\n", - ")\n", - "\n", - "dataset = dataset.apply_over_sampling(over_sample_size_lp=over_sample_size)\n", - "\n", - "aplt.subplot_imaging_dataset(dataset=dataset)\n" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Paths__\n", - "\n", - "All four searches will use the same `path_prefix`, so we write it here to avoid repetition." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "path_prefix = Path(\"howtogalaxy\") / \"chapter_3\" / \"tutorial_3_x2_galaxies\"" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Search Chaining Approach__\n", - "\n", - "Looking at the image, there are two blobs of light corresponding to the two galaxies. \n", - "\n", - "So, how can we break the modeling up? As follows:\n", - "\n", - " 1) Fit and subtract the light of the left galaxy individually.\n", - " 2) Fit and subtract the light of the right galaxy individually.\n", - " 3) Use these results to initialize a fit which fits both galaxy's simultaneously.\n", - "\n", - "So, with this in mind, we'll perform an analysis using 3 searches:\n", - "\n", - " 1) Fit the light of the galaxy on the left of the image, at coordinates (0.0\", -1.0\").\n", - " 2) Fit the light of the galaxy on the right of the image, at coordinates (0.0\", 1.0\").\n", - " 4) Fit all relevant parameters simultaneously, using priors from searches 1, and 2.\n", - "\n", - "__Model + Search + Analysis + Model-Fit (Search 1)__\n", - "\n", - "Search 1 we fit a model where:\n", - "\n", - " - The left galaxy's light is a parametric linear `DevVaucouleurs` bulge with fixed centre [3 parameters].\n", - "\n", - " - the right galaxy's light is omitted.\n", - "\n", - "The number of free parameters and therefore the dimensionality of non-linear parameter space is N=3.\n", - "\n", - "__Notes__\n", - "\n", - "The `DevVaucouleurs` is an `Sersic` profile with `sersic_index=4`.\n", - "\n", - "We fix the centre of its light to (0.0, -1.0), the pixel we know the left galaxy's light centre peaks.\n", - "\n", - "We use linear light profiles througout this script, given that the model is quite complex and this helps\n", - "simplify it." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "left_galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp_linear.DevVaucouleurs)\n", - "left_galaxy.bulge.centre_0 = 0.0\n", - "left_galaxy.bulge.centre_1 = -1.0\n", - "\n", - "model_1 = af.Collection(galaxies=af.Collection(left_galaxy=left_galaxy))" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `info` attribute shows the model in a readable format." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "print(model_1.info)" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Search + Analysis + Model-Fit (Search 1)__" - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "analysis_1 = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", - "\n", - "search_1 = af.Nautilus(\n", - " path_prefix=path_prefix,\n", - " name=\"search[1]__left_galaxy_light[bulge_linear]\",\n", - " unique_tag=dataset_name,\n", - " n_live=75,\n", - " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", - ")\n", - "\n", - "result_1 = search_1.fit(model=model_1, analysis=analysis_1)" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Result (Search 1)__\n", - "\n", - "The results which are used for prior passing are summarised in the `info` attribute." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "print(result_1.info)" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Model (Search 2)__\n", - "\n", - "Search 2 we fit a model where:\n", - "\n", - " - The left galaxy's light is a parametric linear `DevVaucouleurs` bulge [0 parameters: fixed from search 1].\n", - "\n", - " - The right galaxy's light is a parametric linear `DevVaucouleurs` bulge with a fixed centre [3 parameters].\n", - "\n", - " - The galaxy's mass galaxy are omitted.\n", - "\n", - "The number of free parameters and therefore the dimensionality of non-linear parameter space is N=3.\n", - "\n", - "We fix the centre of the right lens's light to (0.0, 1.0), the pixel we know the right galaxy's light centre peaks.\n", - "\n", - "We also pass the result of the `left_galaxy` from search ` as an `instance`, which should improve the fitting of the\n", - "right lens." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "right_galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp_linear.DevVaucouleurs)\n", - "right_galaxy.bulge.centre_0 = 0.0\n", - "right_galaxy.bulge.centre_1 = 1.0\n", - "\n", - "model_2 = af.Collection(\n", - " galaxies=af.Collection(\n", - " left_galaxy=result_1.instance.galaxies.left_galaxy, right_galaxy=right_galaxy\n", - " )\n", - ")" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `info` attribute shows the model, including how all priors are updated via prior passing." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "print(model_2.info)" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Search + Analysis + Model-Fit (Search 2)__" - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "analysis_2 = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", - "\n", - "search_2 = af.Nautilus(\n", - " path_prefix=path_prefix,\n", - " name=\"search[2]__right_galaxy_light[bulge_linear]\",\n", - " unique_tag=dataset_name,\n", - " n_live=75,\n", - " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", - ")\n", - "\n", - "result_2 = search_2.fit(model=model_2, analysis=analysis_2)" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Result (Search 2)__\n", - "The results which are used for prior passing are summarised in the `info` attribute." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "print(result_2.info)" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Model + Search + Analysis + Model-Fit (Search 4)__\n", - "\n", - "Search 4 we fit a model where:\n", - "\n", - " - The left galaxy's light is a parametric linear `Sersic` bulge with centre fixed [4 parameters: priors initialized \n", - " from search 1].\n", - "\n", - " - The right galaxy's light is a parametric linear `Sersic` bulge with centre fixed [4 parameters: priors initialized \n", - " from search 2].\n", - "\n", - "The number of free parameters and therefore the dimensionality of non-linear parameter space is N=8.\n", - "\n", - "We can use a special prior passing method to do this, called `take_attributes`. This scans the `DevVaucouleurs`\n", - "passed to the `take_attributes` method for all parameters which have the same name as the `Sersic` model,\n", - "and if their names are the same it passes their prior as a `model` (like we did above). Thus, it will locate all 6\n", - "parameters in common between the two profiles (centre, ell_comps, intensity, effective_radius) and pass those,\n", - "leaving the `sersic_index`'s priors as the default values.\n", - "\n", - "The `take_attributes` method is used in many examples of prior passing, when we pass a simpler parameterization of a\n", - "model to a more complex model. Another good example would be passing the result of a `IsothermalSph` to an\n", - "`Isothermal`." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "left_galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp_linear.Sersic)\n", - "left_galaxy.bulge.take_attributes(result_1.model.galaxies.left_galaxy.bulge)\n", - "\n", - "right_galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp_linear.Sersic)\n", - "right_galaxy.bulge.take_attributes(result_2.model.galaxies.right_galaxy.bulge)\n", - "\n", - "model_3 = af.Collection(\n", - " galaxies=af.Collection(left_galaxy=left_galaxy, right_galaxy=right_galaxy)\n", - ")\n", - "\n", - "analysis_3 = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", - "\n", - "search_3 = af.Nautilus(\n", - " path_prefix=path_prefix,\n", - " name=\"search[3]_light_x2[bulge_linear]\",\n", - " unique_tag=dataset_name,\n", - " n_live=100,\n", - " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", - ")\n", - "\n", - "result_3 = search_3.fit(model=model_3, analysis=analysis_3)" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Result (Search 3)__\n", - "\n", - "The final results are summarised in the `info` attribute." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "print(result_3.info)" - ], - "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Wrap Up__\n", - "\n", - "We have successfully fitted multiple galaxies, but fitting each one-by-one." - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file diff --git a/scripts/chapter_3_search_chaining/tutorial_2_prior_passing.py b/scripts/chapter_2_modeling/tutorial_10_prior_passing.py similarity index 100% rename from scripts/chapter_3_search_chaining/tutorial_2_prior_passing.py rename to scripts/chapter_2_modeling/tutorial_10_prior_passing.py diff --git a/scripts/chapter_2_modeling/tutorial_9_summary b/scripts/chapter_2_modeling/tutorial_11_summary similarity index 100% rename from scripts/chapter_2_modeling/tutorial_9_summary rename to scripts/chapter_2_modeling/tutorial_11_summary diff --git a/scripts/chapter_3_search_chaining/tutorial_1_search_chaining.py b/scripts/chapter_2_modeling/tutorial_9_search_chaining.py similarity index 100% rename from scripts/chapter_3_search_chaining/tutorial_1_search_chaining.py rename to scripts/chapter_2_modeling/tutorial_9_search_chaining.py diff --git a/scripts/chapter_4_pixelizations/README.md b/scripts/chapter_3_pixelizations/README.md similarity index 100% rename from scripts/chapter_4_pixelizations/README.md rename to scripts/chapter_3_pixelizations/README.md diff --git a/scripts/chapter_3_search_chaining/__init__.py b/scripts/chapter_3_pixelizations/__init__.py similarity index 100% rename from scripts/chapter_3_search_chaining/__init__.py rename to scripts/chapter_3_pixelizations/__init__.py diff --git a/scripts/chapter_4_pixelizations/introduction b/scripts/chapter_3_pixelizations/introduction similarity index 100% rename from scripts/chapter_4_pixelizations/introduction rename to scripts/chapter_3_pixelizations/introduction diff --git a/scripts/chapter_4_pixelizations/tutorial_1_pixelizations.py b/scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py similarity index 100% rename from scripts/chapter_4_pixelizations/tutorial_1_pixelizations.py rename to scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py diff --git a/scripts/chapter_4_pixelizations/tutorial_2_mappers.py b/scripts/chapter_3_pixelizations/tutorial_2_mappers.py similarity index 100% rename from scripts/chapter_4_pixelizations/tutorial_2_mappers.py rename to scripts/chapter_3_pixelizations/tutorial_2_mappers.py diff --git a/scripts/chapter_4_pixelizations/tutorial_3_inversions.py b/scripts/chapter_3_pixelizations/tutorial_3_inversions.py similarity index 100% rename from scripts/chapter_4_pixelizations/tutorial_3_inversions.py rename to scripts/chapter_3_pixelizations/tutorial_3_inversions.py diff --git a/scripts/chapter_4_pixelizations/tutorial_4_bayesian_regularization.py b/scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py similarity index 100% rename from scripts/chapter_4_pixelizations/tutorial_4_bayesian_regularization.py rename to scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py diff --git a/scripts/chapter_4_pixelizations/tutorial_5_model_fit.py b/scripts/chapter_3_pixelizations/tutorial_6_model_fit.py similarity index 100% rename from scripts/chapter_4_pixelizations/tutorial_5_model_fit.py rename to scripts/chapter_3_pixelizations/tutorial_6_model_fit.py diff --git a/scripts/chapter_3_search_chaining/README.md b/scripts/chapter_3_search_chaining/README.md deleted file mode 100644 index c3c0ad9..0000000 --- a/scripts/chapter_3_search_chaining/README.md +++ /dev/null @@ -1,12 +0,0 @@ -In chapter 3, we introduce non-linear search chaining, whereby modeling pipelines are composed which each fit a -different model. - -**Colab** links to every tutorial are included. - -# Files - -- [Tutorial 1: Search Chaining](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_search_chaining/tutorial_1_search_chaining.ipynb) — Breaking the modeling procedure into a chained sequence of model-fits. - -- [Tutorial 2: Prior Passing](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_search_chaining/tutorial_2_prior_passing.ipynb) — How the results of earlier searches are passed to later searches. - -- [Tutorial 3: x2 Galaxies](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_search_chaining/tutorial_3_x2_galaxies.ipynb) — Modeling two galaxies simultaneously using chained searches. diff --git a/scripts/chapter_3_search_chaining/introduction b/scripts/chapter_3_search_chaining/introduction deleted file mode 100755 index 24f9842..0000000 --- a/scripts/chapter_3_search_chaining/introduction +++ /dev/null @@ -1,12 +0,0 @@ -You are now familiar and have a clear understanding of modeling. In this chapter, we'll -introduce search chaining, a concept we mentioned in the previous chapter. Search chaining allows us to generically link -searches together, so that we can seamlessly navigate the complex non-linear parameter spaces that come when fitting -realistic models. - -With these pipelines, you'll be able to: - -1) Fit a lens mass model light model to an image of a strongly lensed source. -2) Additionally fit the galaxy's light, if it is present. -3) Write customized pipelines for galaxy systems with multiple lens galaxies or source galaxies. -4) Customize pipelines such that the priors on parameters during the fit are adjusted to provide a more robust or - efficient fit. \ No newline at end of file diff --git a/scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py b/scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py deleted file mode 100644 index 577dc41..0000000 --- a/scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py +++ /dev/null @@ -1,286 +0,0 @@ -""" -Tutorial 3: Two Galaxies -======================== - -Up to now, all the images we've fitted had one galaxy. However, we saw in chapter 1 that our galaxies object can -consist of multiple galaxies which each contribute to the overall emission. Multi-galaxy systems are challenging to -model, because they add an extra 5-10 parameters to the non-linear search per galaxy and, more problematically, the -degeneracies between the parameters of the light profiles of the galaxies can be severe. - -However, we can still break their analysis down using multiple searches and give ourselves a shot at getting a good -model. Here, we're going to fit a double galaxy system, fitting as much about each individual galaxy before -fitting them simultaneously. - -Up to now, I've put a focus on an analysis being generag. The script we write in this example is going to be the -opposite, specific to the image we're modeling. Fitting multiple galaxies is really difficult and writing a -pipeline that we can generalize to many galaxies isn't currently possible. - -__Contents__ - -- **Initial Setup:** Load the double-galaxy dataset and apply a mask. -- **Paths:** Set the output path for chained search results. -- **Search Chaining Approach:** Strategy for fitting two galaxies using a sequence of searches. -- **Model + Search + Analysis + Model-Fit (Search 1):** Fit the first galaxy alone. -- **Result (Search 1):** Inspect the result of fitting the first galaxy. -- **Model (Search 2):** Compose a model for the second galaxy, fixing the first. -- **Search + Analysis + Model-Fit (Search 2):** Fit the second galaxy. -- **Result (Search 2):** Inspect the result of fitting the second galaxy. -- **Model + Search + Analysis + Model-Fit (Search 4):** Fit both galaxies simultaneously. -- **Result (Search 3):** Inspect the final simultaneous fit result. -- **Wrap Up:** Summary of search chaining for multi-galaxy systems. -""" - -# from autogalaxy import setup_notebook; setup_notebook() - -from pathlib import Path -import autofit as af -import autogalaxy as ag -import autogalaxy.plot as aplt - -""" -__Initial Setup__ - -we'll use new galaxying data, where: - - - There are two galaxy's whose `LightProfile`'s are both `Sersic`'s. -""" -dataset_name = "sersic_x2" -dataset_path = Path("dataset") / "imaging" / dataset_name - -""" -__Dataset Auto-Simulation__ - -If the dataset does not already exist on your system, it will be created by running the corresponding -simulator script. This ensures that all example scripts can be run without manually simulating data first. -""" -if ag.util.dataset.should_simulate(str(dataset_path)): - import subprocess - import sys - - subprocess.run( - [sys.executable, "scripts/simulators/sersic_x2.py"], - check=True, - ) - - -dataset = ag.Imaging.from_fits( - data_path=dataset_path / "data.fits", - noise_map_path=dataset_path / "noise_map.fits", - psf_path=dataset_path / "psf.fits", - pixel_scales=0.05, -) - -""" -__Mask__ - -We need to choose our mask for the analysis. Given the light of both galaxies is present in the image we'll need to -include all their light in the image, so lets use a large circular mask. - -We'll use this mask in all three searches, however you could imagine customizing it on a per-search basis to speed up -the analysis. -""" -mask = ag.Mask2D.circular( - shape_native=dataset.shape_native, pixel_scales=dataset.pixel_scales, radius=6.0 -) - -dataset = dataset.apply_mask(mask=mask) - -over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from( - grid=dataset.grid, - sub_size_list=[8, 4, 2], - radial_list=[0.3, 0.6], - centre_list=[(0.0, -1.0), (0.0, 1.0)], -) - -dataset = dataset.apply_over_sampling(over_sample_size_lp=over_sample_size) - -aplt.subplot_imaging_dataset(dataset=dataset) - - -""" -__Paths__ - -All four searches will use the same `path_prefix`, so we write it here to avoid repetition. -""" -path_prefix = Path("howtogalaxy") / "chapter_3" / "tutorial_3_x2_galaxies" - -""" -__Search Chaining Approach__ - -Looking at the image, there are two blobs of light corresponding to the two galaxies. - -So, how can we break the modeling up? As follows: - - 1) Fit and subtract the light of the left galaxy individually. - 2) Fit and subtract the light of the right galaxy individually. - 3) Use these results to initialize a fit which fits both galaxy's simultaneously. - -So, with this in mind, we'll perform an analysis using 3 searches: - - 1) Fit the light of the galaxy on the left of the image, at coordinates (0.0", -1.0"). - 2) Fit the light of the galaxy on the right of the image, at coordinates (0.0", 1.0"). - 4) Fit all relevant parameters simultaneously, using priors from searches 1, and 2. - -__Model + Search + Analysis + Model-Fit (Search 1)__ - -Search 1 we fit a model where: - - - The left galaxy's light is a parametric linear `DevVaucouleurs` bulge with fixed centre [3 parameters]. - - - the right galaxy's light is omitted. - -The number of free parameters and therefore the dimensionality of non-linear parameter space is N=3. - -__Notes__ - -The `DevVaucouleurs` is an `Sersic` profile with `sersic_index=4`. - -We fix the centre of its light to (0.0, -1.0), the pixel we know the left galaxy's light centre peaks. - -We use linear light profiles througout this script, given that the model is quite complex and this helps -simplify it. -""" -left_galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp_linear.DevVaucouleurs) -left_galaxy.bulge.centre_0 = 0.0 -left_galaxy.bulge.centre_1 = -1.0 - -model_1 = af.Collection(galaxies=af.Collection(left_galaxy=left_galaxy)) - -""" -The `info` attribute shows the model in a readable format. -""" -print(model_1.info) - -""" -__Search + Analysis + Model-Fit (Search 1)__ -""" -analysis_1 = ag.AnalysisImaging(dataset=dataset, use_jax=True) - -search_1 = af.Nautilus( - path_prefix=path_prefix, - name="search[1]__left_galaxy_light[bulge_linear]", - unique_tag=dataset_name, - n_live=75, - n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. -) - -result_1 = search_1.fit(model=model_1, analysis=analysis_1) - -""" -__Result (Search 1)__ - -The results which are used for prior passing are summarised in the `info` attribute. -""" -print(result_1.info) - -""" -__Model (Search 2)__ - -Search 2 we fit a model where: - - - The left galaxy's light is a parametric linear `DevVaucouleurs` bulge [0 parameters: fixed from search 1]. - - - The right galaxy's light is a parametric linear `DevVaucouleurs` bulge with a fixed centre [3 parameters]. - - - The galaxy's mass galaxy are omitted. - -The number of free parameters and therefore the dimensionality of non-linear parameter space is N=3. - -We fix the centre of the right lens's light to (0.0, 1.0), the pixel we know the right galaxy's light centre peaks. - -We also pass the result of the `left_galaxy` from search ` as an `instance`, which should improve the fitting of the -right lens. -""" -right_galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp_linear.DevVaucouleurs) -right_galaxy.bulge.centre_0 = 0.0 -right_galaxy.bulge.centre_1 = 1.0 - -model_2 = af.Collection( - galaxies=af.Collection( - left_galaxy=result_1.instance.galaxies.left_galaxy, right_galaxy=right_galaxy - ) -) - -""" -The `info` attribute shows the model, including how all priors are updated via prior passing. -""" -print(model_2.info) - -""" -__Search + Analysis + Model-Fit (Search 2)__ -""" -analysis_2 = ag.AnalysisImaging(dataset=dataset, use_jax=True) - -search_2 = af.Nautilus( - path_prefix=path_prefix, - name="search[2]__right_galaxy_light[bulge_linear]", - unique_tag=dataset_name, - n_live=75, - n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. -) - -result_2 = search_2.fit(model=model_2, analysis=analysis_2) - -""" -__Result (Search 2)__ -The results which are used for prior passing are summarised in the `info` attribute. -""" -print(result_2.info) - -""" -__Model + Search + Analysis + Model-Fit (Search 4)__ - -Search 4 we fit a model where: - - - The left galaxy's light is a parametric linear `Sersic` bulge with centre fixed [4 parameters: priors initialized - from search 1]. - - - The right galaxy's light is a parametric linear `Sersic` bulge with centre fixed [4 parameters: priors initialized - from search 2]. - -The number of free parameters and therefore the dimensionality of non-linear parameter space is N=8. - -We can use a special prior passing method to do this, called `take_attributes`. This scans the `DevVaucouleurs` -passed to the `take_attributes` method for all parameters which have the same name as the `Sersic` model, -and if their names are the same it passes their prior as a `model` (like we did above). Thus, it will locate all 6 -parameters in common between the two profiles (centre, ell_comps, intensity, effective_radius) and pass those, -leaving the `sersic_index`'s priors as the default values. - -The `take_attributes` method is used in many examples of prior passing, when we pass a simpler parameterization of a -model to a more complex model. Another good example would be passing the result of a `IsothermalSph` to an -`Isothermal`. -""" -left_galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp_linear.Sersic) -left_galaxy.bulge.take_attributes(result_1.model.galaxies.left_galaxy.bulge) - -right_galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp_linear.Sersic) -right_galaxy.bulge.take_attributes(result_2.model.galaxies.right_galaxy.bulge) - -model_3 = af.Collection( - galaxies=af.Collection(left_galaxy=left_galaxy, right_galaxy=right_galaxy) -) - -analysis_3 = ag.AnalysisImaging(dataset=dataset, use_jax=True) - -search_3 = af.Nautilus( - path_prefix=path_prefix, - name="search[3]_light_x2[bulge_linear]", - unique_tag=dataset_name, - n_live=100, - n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. -) - -result_3 = search_3.fit(model=model_3, analysis=analysis_3) - -""" -__Result (Search 3)__ - -The final results are summarised in the `info` attribute. -""" -print(result_3.info) - -""" -__Wrap Up__ - -We have successfully fitted multiple galaxies, but fitting each one-by-one. -""" diff --git a/scripts/chapter_4_pixelizations/__init__.py b/scripts/chapter_4_scaling_up_galaxies/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/chapter_4_pixelizations/__init__.py rename to scripts/chapter_4_scaling_up_galaxies/__init__.py From 363794fe8437971c329d2fab6b2cca8add7cfb4e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:32:01 +0000 Subject: [PATCH 02/15] docs: update top-level chapter maps for the restructure Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- AGENTS.md | 7 ++++--- README.md | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 70d47a8..86a37c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,8 +10,9 @@ instructions for this repo. - `scripts/` — Runnable Python tutorial scripts: - `chapter_1_introduction/` — Grids, light profiles, galaxies, data, fitting - `chapter_2_modeling/` — Non-linear searches, Bayesian inference, galaxy modeling - - `chapter_3_search_chaining/` — Search chaining, prior passing, automated pipelines - - `chapter_4_pixelizations/` — Pixelized reconstruction, inversions, regularization + - `chapter_3_pixelizations/` — Pixelized reconstruction, inversions, regularization, the Bayesian + formalism + - `chapter_4_scaling_up_galaxies/` — Extra galaxies, blended multi-galaxy systems, cluster fields - `chapter_optional/` — Alternative searches and advanced topics - `simulators/` — Simulator scripts that generate the tutorial datasets at runtime - `notebooks/` — Jupyter versions, generated from `scripts/` (do not edit directly) @@ -36,7 +37,7 @@ fast smoke run). **Dataset realism:** automated runs **do** cap datasets — workspaces. (This paragraph previously claimed the opposite; the claim was untrue and went unnoticed because the chapters that break under the cap were never in the smoke list.) Tutorials must therefore work at **both** resolutions: never hardcode an index or a shape derived from the full-resolution -dataset. `chapter_4_pixelizations/tutorial_3_inversions.py` is the cautionary case — it sized its mesh +dataset. `chapter_3_pixelizations/tutorial_3_inversions.py` is the cautionary case — it sized its mesh from `dataset.shape_native`, giving 10000 mesh pixels at full resolution but 256 under the cap, so its fixed `pix_indexes` ran off the end. diff --git a/README.md b/README.md index 1d84496..a5c9d7f 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ more appropriate — they are concise and assume the concepts taught in **HowToG - `chapter_1_introduction` — An introduction to galaxy morphology and **PyAutoGalaxy**: grids, light profiles, galaxies, simulated imaging data, and fitting. - `chapter_2_modeling` — Bayesian inference, non-linear searches, and how to fit a galaxy model to CCD - imaging data with **PyAutoGalaxy**. -- `chapter_3_search_chaining` — Chaining multiple non-linear searches together to build automated galaxy - modeling pipelines for complex systems. -- `chapter_4_pixelizations` — Pixelized source reconstructions (inversions) for galaxies with irregular - morphologies. + imaging data with **PyAutoGalaxy**, ending with search chaining and automated pipelines. +- `chapter_3_pixelizations` — Pixelized reconstructions (inversions) for galaxies with irregular + morphologies, including the Bayesian formalism underpinning them. +- `chapter_4_scaling_up_galaxies` — Scaling galaxy modeling up beyond a single galaxy: extra galaxies, + blended multi-galaxy systems, and cluster fields. - `chapter_optional` — Optional tutorials on alternative non-linear searches and other advanced topics. **HowToGalaxy** currently sits at four chapters. Each chapter will take around a day to work through. From f33295f02507aeba4d65b1ba3fa9998f20f98795 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:34:03 +0000 Subject: [PATCH 03/15] docs: teach interferometer data in the chapter 1 data tutorial One docstring + code block loading and plotting an interferometer dataset (visibilities, uv-plane fitting, dirty images), stating HowToGalaxy will not cover interferometry further and pointing readers to the workspace. Backed by a new plain-galaxy interferometer simulator with a small synthetic uv-baseline set, wired in with the standard auto-simulate guard. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- .../chapter_1_introduction/tutorial_2_data.py | 57 +++++- scripts/simulators/interferometer.py | 167 ++++++++++++++++++ 2 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 scripts/simulators/interferometer.py diff --git a/scripts/chapter_1_introduction/tutorial_2_data.py b/scripts/chapter_1_introduction/tutorial_2_data.py index 2499ea8..68fe088 100644 --- a/scripts/chapter_1_introduction/tutorial_2_data.py +++ b/scripts/chapter_1_introduction/tutorial_2_data.py @@ -26,6 +26,7 @@ - **Background Sky:** Add background sky light that introduces noise across the entire image. - **Simulator:** Use the SimulatorImaging object to simulate imaging data with all effects combined. - **Output:** Save the simulated data to .fits files for use in future tutorials. +- **Interferometer Data:** Radio / sub-mm interferometers like ALMA observe visibilities in the uv-plane rather than images. - **Wrap Up:** Summary of how CCD imaging data is simulated. """ @@ -350,10 +351,64 @@ overwrite=True, ) +""" +__Interferometer Data__ + +CCD imaging is not the only type of data used to study galaxies. Radio and sub-mm interferometers, like the +Atacama Large Millimeter Array (ALMA) and the Jansky Very Large Array (JVLA), observe galaxies at wavelengths +where a CCD cannot. + +An interferometer does not observe an image of the galaxy. Each pair of antennas in the array measures a +"visibility", a Fourier component of the sky brightness, at a point in what is called the "uv-plane" set by the +separation of the two antennas. The dataset is therefore a set of complex visibilities in Fourier space, not a 2D +image, and its noise properties are very different from those of CCD data — there is no PSF convolution, Poisson +noise or background sky; instead each visibility has Gaussian noise. + +One could Fourier transform the visibilities into an image (called a "dirty image") and fit that, but the transform +correlates the noise between pixels, making the fit statistically incorrect. **PyAutoGalaxy** therefore fits galaxy +models directly in visibility space: the galaxy's image is evaluated in real space on a grid (defined by a +real-space mask), Fourier transformed to the uv-plane and compared with the observed visibilities there. + +Below, we load a simulated interferometer dataset (creating it first via the `scripts/simulators/interferometer.py` +script if it does not exist on your hard-disk, using the same auto-simulation idiom as later tutorials) and plot +its dirty images — the closest an interferometer dataset comes to the CCD images we simulated above. + +HowToGalaxy will not cover interferometry any further than this. The lecture series teaches galaxy modeling using +CCD imaging, and everything you learn transfers to visibility-space fitting. If you need to model interferometer +data, go to the `autogalaxy_workspace/scripts/interferometer` package, which is the dedicated resource for +uv-plane galaxy modeling. +""" +dataset_path = Path("dataset") / "interferometer" / "simple" + +if ag.util.dataset.should_simulate(str(dataset_path)): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/interferometer.py"], + check=True, + ) + +real_space_mask = ag.Mask2D.circular( + shape_native=(100, 100), + pixel_scales=0.1, + radius=3.0, +) + +dataset = ag.Interferometer.from_fits( + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + uv_wavelengths_path=dataset_path / "uv_wavelengths.fits", + real_space_mask=real_space_mask, + transformer_class=ag.TransformerDFT, +) + +aplt.subplot_interferometer_dirty_images(dataset=dataset) + """ __Wrap Up__ -In this tutorial, you learned how CCD imaging data of a galaxy is collected using real telescopes like the +In this tutorial, you learned how CCD imaging data of a galaxy is collected using real telescopes like the Hubble Space Telescope, and how to simulate this data using the `SimulatorImaging` object. Let's summarise what we've covered: diff --git a/scripts/simulators/interferometer.py b/scripts/simulators/interferometer.py new file mode 100644 index 0000000..751e9e8 --- /dev/null +++ b/scripts/simulators/interferometer.py @@ -0,0 +1,167 @@ +""" +Simulator: Interferometer +========================= + +This script simulates `Interferometer` data of a galaxy, as would be observed by a radio or sub-mm +interferometer like ALMA or the JVLA, where: + + - The galaxy's bulge is an `Sersic`. + +Unlike CCD imaging, an interferometer does not observe an image of the galaxy. It measures "visibilities", which +are the Fourier transform of the sky brightness sampled at a set of points in the "uv-plane", where each point +corresponds to the separation of a pair of antennas in the array. + +It is used to illustrate interferometer data in the HowToGalaxy lecture series. HowToGalaxy does not cover +interferometry beyond this glimpse; the `autogalaxy_workspace/scripts/interferometer` package is the dedicated +resource for uv-plane galaxy modeling. + +__Contents__ + +- **Dataset Paths:** Set the output path for the simulated dataset. +- **Simulate:** Simulate the image using a (y,x) real-space grid and a synthetic set of uv-plane baselines. +- **Galaxies:** Define the galaxy Sersic light profile used for simulation. +- **Output:** Save the simulated dataset to FITS files. +- **Visualize:** Output subplot and dirty-image PNGs of the simulated dataset. +- **Plane Output:** Save the Galaxies object as a JSON file. + +__Start Here Notebook__ + +If any code in this script is unclear, refer to the `autogalaxy_workspace/*/interferometer/simulator.ipynb` +notebook. +""" + +# from autogalaxy import setup_notebook; setup_notebook() + +from pathlib import Path +import numpy as np +import autogalaxy as ag +import autogalaxy.plot as aplt + +""" +__Dataset Paths__ + +The `dataset_type` describes the type of data being simulated and `dataset_name` gives it a descriptive name. +""" +dataset_type = "interferometer" +dataset_name = "simple" + +dataset_path = Path("dataset", dataset_type, dataset_name) + +""" +__Simulate__ + +For interferometer data, the galaxy's image is evaluated in real space on a (y,x) grid and then Fourier +transformed to the uv-plane, where it is compared to the observed visibilities. + +Interferometers do not observe galaxies in a way where over sampling is necessary, therefore the grid uses no +over sampling. +""" +grid = ag.Grid2D.uniform( + shape_native=(100, 100), + pixel_scales=0.1, +) + +""" +To perform the Fourier transform we need the wavelengths of the baselines, which describe where in the uv-plane +each visibility samples the sky's Fourier transform. + +For real data these are determined by the layout of the interferometer's antennas. The `autogalaxy_workspace` +bundles baselines of real instruments (e.g. the Square Mile Array (SMA) and ALMA). For this tutorial dataset we +instead draw a small synthetic set of baselines from a Gaussian distribution in the uv-plane, with a scale +comparable to the SMA's (a few hundred visibilities), keeping the simulation fast and self-contained. +""" +rng = np.random.default_rng(1) + +total_visibilities = 200 + +uv_wavelengths = rng.normal(loc=0.0, scale=1.0e5, size=(total_visibilities, 2)) + +""" +To simulate the interferometer dataset we first create a simulator, which defines the exposure time, noise levels +and Fourier transform method used in the simulation. + +We use the `TransformerDFT`, an exact Discrete Fourier Transform which is fast for datasets with a low number of +visibilities like this one. +""" +simulator = ag.SimulatorInterferometer( + uv_wavelengths=uv_wavelengths, + exposure_time=300.0, + noise_sigma=1000.0, + transformer_class=ag.TransformerDFT, +) + +""" +__Galaxies__ + +Setup the galaxy with a bulge (elliptical Sersic) for this simulation. +""" +galaxy = ag.Galaxy( + redshift=0.5, + bulge=ag.lp.Sersic( + centre=(0.0, 0.0), + ell_comps=ag.convert.ell_comps_from(axis_ratio=0.9, angle=45.0), + intensity=1.0, + effective_radius=0.8, + sersic_index=4.0, + ), +) + +""" +Use these galaxies to generate the image for the simulated `Interferometer` dataset. +""" +galaxies = ag.Galaxies(galaxies=[galaxy]) +aplt.plot_array(array=galaxies.image_2d_from(grid=grid), title="Image") + +""" +Pass the simulator galaxies, which creates the image plotted above and simulates it as an interferometer dataset. +""" +dataset = simulator.via_galaxies_from(galaxies=galaxies, grid=grid) + +""" +Plot the simulated interferometer dataset's dirty images before outputting it to fits. +""" +aplt.subplot_interferometer_dirty_images(dataset=dataset) + +""" +__Output__ + +Output the simulated dataset to the dataset path as .fits files. +""" +aplt.fits_interferometer( + dataset=dataset, + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + uv_wavelengths_path=dataset_path / "uv_wavelengths.fits", + overwrite=True, +) + +""" +__Visualize__ + +Output a subplot of the simulated dataset's dirty images and the galaxies quantities to the dataset path as +.png files. +""" +aplt.subplot_interferometer_dirty_images( + dataset=dataset, output_path=dataset_path, output_format="png" +) + +aplt.subplot_galaxies( + galaxies=galaxies, grid=grid, output_path=dataset_path, output_format="png" +) + +""" +__Plane Output__ + +Save the `Galaxies` in the dataset folder as a .json file, ensuring the true light profiles and galaxies +are safely stored and available to check how the dataset was simulated in the future. + +This can be loaded via the method `galaxies = ag.from_json()`. +""" +ag.output_to_json( + obj=galaxies, + file_path=Path(dataset_path, "galaxies.json"), +) + +""" +The dataset can be viewed in the folder `dataset/interferometer/simple`. +""" From 91c28e1d658075b1084dbf91dbc3ab079193ab63 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:34:12 +0000 Subject: [PATCH 04/15] docs: rewrite need for speed for the JAX era Replaces the outdated numba-centric story with the current stack: JAX JIT compilation of the likelihood (use_jax=True default), CPU vs GPU, VRAM checking, run-time arithmetic tied to tutorial 2's real numbers, and the galaxy-specific cost-per-evaluation breakdown (light profiles, over-sampling, PSF convolution, inversion linear algebra). Framing hands off to the new end-of-chapter search chaining tutorials. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- .../tutorial_8_need_for_speed.py | 246 ++++++++++++------ 1 file changed, 163 insertions(+), 83 deletions(-) diff --git a/scripts/chapter_2_modeling/tutorial_8_need_for_speed.py b/scripts/chapter_2_modeling/tutorial_8_need_for_speed.py index 89a257d..36ea7a4 100755 --- a/scripts/chapter_2_modeling/tutorial_8_need_for_speed.py +++ b/scripts/chapter_2_modeling/tutorial_8_need_for_speed.py @@ -1,83 +1,163 @@ -""" -Tutorial 8: Need For Speed -========================== - -In this chapter, we have learnt how to model galaxies and how to balance complexity and realism to ensure that we -infer a good model. - -__Contents__ - -- **Searching Non-linear Parameter Space:** How dimensionality, priors and settings drive search run-times. -- **Algorithmic Optimization:** How PyAutoGalaxy uses numba and JAX for fast computation. -- **Data Quantity:** How the number of image pixels affects run-time. -- **Wrap Up:** Summary of strategies for keeping model-fit run-times manageable. - -For fitting more complex models, the final challenge that we face is keeping the run-time low. One can easily end -up in a situation where a model-fit takes days, or longer, to fit just one image. For fitting complex models and high -resolution datasets this is somewhat unavoidable. However, it is worth us discussing what drives the long run-times of -the modeling process and how we might speed it up. - -__Searching Non-linear Parameter Space__ - -The time it takes for the non-linear search to sample parameter space and find the high likelihood models is driven by: - - - Dimensionality: A more complex parameter space (e.g. more parameters) takes longer to search. - - Priors: The broader the priors on each parameter the longer the search. - - Settings: Non-linear search settings which sample parameter space more thoroughly lead to longer run-times. - -When we use only one search to fit a model, we are somewhat restricted in how we can try to achieve faster run -times by changing these 3 aspects of the search. - -In the next chapter, we introduce 'non-linear search chaining', which fits a model using multiple searches that -are performed back-to-back. A key motivation for this is that it gives us a lot more flexibility in juggling the -dimensionality, priors and settings so as to perform faster and more efficient modeling. - -In the optional **HowToGalaxy** tutorial `chapter_optional/tutorial_searches.ipynb` we discuss other non-linear -searches supported by **HowToGalaxy** which use a different approach to sample parameter sample than `nautilus`. For -those familiar with statistical inference, this includes maximum likelihood estimators and MCMC algorithms. - -For galaxy modeling, there are maximum likelihood estimator methods (e.g. the Levenberg-Marquardt) that for simple -models (e.g. a single `Sersic`) can often reliable infer the maximum likelihood model, using 10 times or fewer -likelihood evaluations than `nautilus` therefore running over ten times or more faster). However, these methods do -not infer reliable errors and are subject to inferring local maximum. Nevertheless, users modeling large galaxy -samples may wish to investigate these methods. - -__Algorithmic Optimization__ - -Every operation **PyAutoGalaxy** performs to fit galaxy data with a model takes time, for example: - - - Computing the intensity values from a light profile. - - Convolving the galaxies image with the PSF to compare it to the data. - -One can therefore in principle make **PyAutoGalaxy** run faster by using more efficient algorithms. However, I am -confident that for many tasks and operations we have written code that is already very fast! - -I often get asked, given that **PyAutoGalaxy** is written in Python (a synonymously slow programming language), is it -not really slow? **PyAutoGalaxy** uses a library called `numba` to ensure that it runs fast, which recompiles Python -functions into C functions before **PyAutoGalaxy** runs. This gives us C-like speed, but in Python code. If you`ve got -your own code that needs speeding up, I strongly recommend that you look up Numba: - -http://numba.pydata.org/ - -Therefore, **PyAutoGalaxy** is pretty well optimized and there are no 'low hanging fruit' speed ups available by -writing the code in a different language. - -__Data Quantity__ - -The final factor driving run-speed is the quantity of data that is fitted. For every image-pixel that we fit, -we have to compute the light profile intensities and convolve it with the telescope's PSF. The larger that PSF is, -the more convolution operations we have to perform too. - -In the previous exercises, we used images with a pixel scale of 0.1". This value is relatively low resolution: most -Hubble Space Telescope images have a pixel scale of 0.05", which is four times the number of pixels! Some telescopes -observe at scales of 0.03" or, dare I say it, 0.01". At these resolutions things can *really* slow down, if we -do not think carefully about run speed beforehand. - -There are ways that we can reduce the number of image-pixels we fit, via masking. If we mask out more of the image, -we will fit fewer pixels and **PyAutoGalaxy** will run faster. If you want the best, most perfect model possible, -aggressive masking and cutting the data in this way is a bad idea, as discussed in tutorial 5. - -__Wrap Up__ - -This tutorial simply wanted to get you thinking about *why* a model takes as long to fit as it does. -""" +""" +Tutorial 8: Need For Speed +========================== + +In this chapter, we have learnt how to model galaxies and how to balance complexity and realism to ensure that we +infer a good model. + +For fitting more complex models, the final challenge that we face is keeping the run-time low. One can easily end +up in a situation where a model-fit takes days, or longer, to fit just one image. For fitting complex models and high +resolution datasets this is somewhat unavoidable. However, it is worth us discussing what drives the long run-times of +the modeling process and how we might speed it up. + +We have kept an eye on the log likelihood evaluation times and therefore expected overall run-time of the fits +performed in this chapter. Lets quickly remind ourselves of the main factors that drive the run-time, and how we +might reduce it. + +The overall run-time of a model-fit is, to a good approximation, a simple product of two numbers: + + - The number of log likelihood evaluations the non-linear search performs before it converges. + + - The time each individual log likelihood evaluation takes. + +Everything we discuss below reduces one (or both) of these numbers. + +__Contents__ + +- **Searching Non-linear Parameter Space:** What drives the number of likelihood evaluations a search performs. +- **Cost Per Evaluation:** What drives the time a single log likelihood evaluation takes. +- **JAX:** How **PyAutoGalaxy** compiles the likelihood function with JAX and runs it on CPUs and GPUs. +- **Run Time Estimation:** How to estimate the overall run-time of a fit before committing to it. +- **Data Quantity:** How the amount of data fitted, and the mask applied to it, drives run-times. +- **Wrap Up:** Summary of the script and next steps. + +__Searching Non-linear Parameter Space__ + +The time it takes for the non-linear search to sample parameter space and find the high likelihood models is driven by: + + - Dimensionality: A more complex parameter space (e.g. more parameters) takes longer to search. + - Priors: The broader the priors on each parameter the longer the search. + - Settings: Non-linear search settings which sample parameter space more thoroughly (e.g. a higher `n_live`) lead to + longer run-times. + +When we use only one search to fit a model, we are somewhat restricted in how we can try to achieve faster run +times by changing these 3 aspects of the search. We have already seen one powerful exception: the linear light +profiles of tutorial 5 remove every `intensity` parameter from the non-linear parameter space, reducing its +dimensionality and removing some of its most difficult degeneracies at the same time. + +The final two tutorials of this chapter (tutorials 9 and 10) introduce 'non-linear search chaining', which fits +a model using multiple searches that are performed back-to-back, passing the results of earlier searches to +later ones. A key motivation for this is that it gives us a lot more flexibility in juggling the dimensionality, +priors and settings so as to perform faster and more efficient modeling. + +In the optional **HowToGalaxy** tutorial `chapter_optional/tutorial_searches.py` we discuss other non-linear searches +supported by **PyAutoGalaxy** which use a different approach to sample parameter space than `Nautilus`. For those +familiar with statistical inference, this includes maximum likelihood estimators and MCMC algorithms. + +For galaxy modeling, there are maximum likelihood estimator methods (e.g. the Levenberg-Marquardt) that for simple +models (e.g. a single `Sersic`) can often reliably infer the maximum likelihood model, using ten times or fewer +likelihood evaluations than `Nautilus` and therefore running over ten times faster. However, these methods do +not infer reliable errors and are prone to becoming trapped in local maxima. Nevertheless, users modeling large +galaxy samples may wish to investigate these methods. + +__Cost Per Evaluation__ + +The second number is the time a single log likelihood evaluation takes, which is set by every operation +**PyAutoGalaxy** performs to fit the galaxy data with a model, for example: + + - Computing the intensity values of every light profile on the image's grid of (y,x) coordinates. + - Over-sampling: sub-dividing image pixels into finer sub-pixels where the light profile changes rapidly (e.g. the + centre of the galaxy), which multiplies the number of intensity evaluations performed. + - Convolving the model image of the galaxies with the telescope's PSF to compare it to the data. The larger the + PSF, the more expensive this convolution becomes. + - For linear light profiles, basis functions like the MGE of tutorial 5 and the pixelizations of chapter 3, solving + the linear algebra of the inversion, which computes the `intensity` values that best fit the data. + +More complex fitting techniques therefore cost more per evaluation: a couple of standard light profiles are cheap, +an MGE costs more (its inversion solves for many Gaussians), and pixelized reconstructions cost the most. +As we saw in tutorial 5, this trade often still favours the more advanced technique, because the simpler parameter +space it produces reduces the number of evaluations the search needs. + +__JAX__ + +How are these operations made fast? **PyAutoGalaxy** uses JAX (https://github.com/jax-ml/jax), Google's numerical +computing library, which just-in-time (JIT) compiles the entire log likelihood function into optimized machine code. + +JIT compilation means the first time the likelihood function is called, JAX traces through every operation it +performs (light profile evaluation, over-sampling, PSF convolution, the linear algebra of an inversion) and compiles +them into a single optimized program. This compilation is a one-off overhead, typically taking seconds to a couple +of minutes depending on model complexity. Every one of the tens of thousands of likelihood evaluations that follow +then reuses the compiled program and runs dramatically faster than ordinary Python. When a search begins you will +see a log message like `JAX jit compiling vectorized (vmap) likelihood function, could take seconds or minutes...` +-- that is the compilation happening. + +JAX's second superpower is that the same compiled code runs on either a CPU or a GPU. On a GPU, likelihood +evaluations are not only faster individually but can be batched, with many evaluated simultaneously (the +`n_batch` search input we have set throughout this chapter controls the size of these batches). GPU speed-ups +over CPU of an order of magnitude or more are common, and they grow with the size of the dataset and model. If you +have access to a GPU (even a modest laptop GPU), it is the single biggest speed-up available to you. + +You do not need to do anything to switch this on. If JAX is installed (`pip install autogalaxy[jax]`), every +`AnalysisImaging` object we have created in this chapter defaults to `use_jax=True` and the search compiles and +batches the likelihood function automatically. You can force the plain NumPy path with +`ag.AnalysisImaging(dataset=dataset, use_jax=False)` (or by setting the environment variable +`PYAUTO_DISABLE_JAX=1`), which is mainly useful for debugging, as NumPy errors and stack traces are much easier +to read than JAX ones. + +The one trade-off to keep in mind is the compilation overhead itself. For a full model-fit performing tens of +thousands of likelihood evaluations it is negligible. For a quick one-off calculation (e.g. fitting galaxies to +data once via a `FitImaging`, as we did in chapter 1) compilation can take longer than the calculation it speeds +up, which is why short interactive computations sometimes feel slower than you might expect the first time they run. + +Long-time users may remember that **PyAutoGalaxy** previously used `numba`, which recompiles individual Python +functions into fast machine code. JAX supersedes it: rather than accelerating functions one at a time, it compiles +and optimizes the likelihood function end-to-end, and adds GPU support and batching on top. + +Therefore, **PyAutoGalaxy** is pretty well optimized and there are no 'low hanging fruit' speed ups available by +writing the code in a different language. + +__Run Time Estimation__ + +Because run-time is the product of the two numbers above, we can estimate it before committing to a fit: + + - Time the log likelihood evaluation: perform a fit of the model to the data once (e.g. via a `FitImaging`) and + time it, remembering to discard the first call, which includes JIT compilation. + + - Estimate the number of evaluations: for `Nautilus` a reasonable rule of thumb is of order 10000 evaluations per + free parameter, with simpler parameter spaces (e.g. linear light profiles, chained searches with tight priors) + converging in fewer. + +Multiplying the two gives a ballpark overall run-time. In tutorial 2 we used exactly this arithmetic: a log +likelihood evaluation time of ~0.05 seconds and 50000-100000 evaluations gave expected run-times of roughly 30 +minutes on CPU and 10 minutes on GPU. If your own estimate comes out at days or weeks, that is the moment to +simplify the model, tighten the search settings, reach for search chaining or find a GPU -- before starting the +fit, not after. + +On a GPU, there is one other pre-flight check: `analysis.print_vram_use(model=model, batch_size=search.batch_size)` +estimates whether the fit will fit within the GPU's memory for your chosen batch size. + +__Data Quantity__ + +The final factor driving run-speed is the quantity of data that is fitted. For every image-pixel that we fit, +we have to compute the light profile intensities and convolve them with the telescope's PSF. The larger that PSF is, +the more convolution operations we have to perform too. + +In the previous exercises, we used images with a pixel scale of 0.1". This value is relatively low resolution: most +Hubble Space Telescope images have a pixel scale of 0.05", which is four times the number of pixels! Some telescopes +observe at scales of 0.03" or, dare I say it, 0.01". At these resolutions things can *really* slow down, if we +do not think carefully about run speed beforehand. + +There are ways that we can reduce the number of image-pixels we fit, via masking. If we mask out more of the image, +we will fit fewer pixels and **PyAutoGalaxy** will run faster. If you want the best, most perfect model possible, +aggressive masking and cutting the data in this way is a bad idea, as discussed in tutorial 6. + +__Wrap Up__ + +This tutorial simply wanted to get you thinking about *why* a model takes as long to fit as it does: the number +of likelihood evaluations the search performs, times the cost of each evaluation, with JAX's compiled likelihood +function (and a GPU, if you have one) driving the latter down as far as it will go. + +The remaining tutorials of this chapter attack the other half of the product: search chaining (tutorial 9) and prior +passing (tutorial 10) reduce the number of evaluations needed to fit complex models, laying the groundwork for the +pixelized reconstructions of chapter 3. +""" From 1a9c320c6b0dc2df94fa4a0ee740c59c993f9cd2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:36:00 +0000 Subject: [PATCH 05/15] =?UTF-8?q?docs:=20write=20chapter=204=20tutorial=20?= =?UTF-8?q?2=20=E2=80=94=20multi-galaxy=20blends?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The co-dominant blend regime: why fitting overlapping galaxies separately biases both, joint composition of the sersic_x2 pair with MGE light and centres fixed to the observed light, runtime-verified parameter accounting (14 -> 4 free parameters across the model choices), per-galaxy decomposition and photometry, and the flux-split degeneracy. The sersic_x2 simulator header now names its real chapter. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- .../tutorial_2_multi_galaxy.py | 410 ++++++++++++++++++ scripts/simulators/sersic_x2.py | 2 +- 2 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py diff --git a/scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py b/scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py new file mode 100644 index 0000000..ace7e27 --- /dev/null +++ b/scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py @@ -0,0 +1,410 @@ +""" +Tutorial 2: Multi-Galaxy Blends +=============================== + +In the previous tutorial, we learned how to deal with extra galaxies near the galaxy we care about — nuisance +objects whose light contaminates the data but which are not themselves the subject of our study. We removed their +emission from the analysis, or gave them a heavily restricted model, and the single galaxy we were studying +remained the star of the show. + +In this tutorial, we meet systems where that picture breaks down entirely: **multi-galaxy blends**, where two +(or more) galaxies of comparable brightness overlap on the sky and *every one of them* is a science subject. +Neither galaxy is a minor contaminant we can mask away or simplify — they are **co-dominant**, and each needs +its own free light model, fitted simultaneously in a single analysis. + +How do such systems arise physically? There are two main channels: + +- **Interacting pairs and mergers**: two galaxies at the same redshift caught orbiting one another or in the + act of merging. Their projected separation is small compared to the extent of their light, so their surface + brightness distributions genuinely overlap — and the interaction itself (tidal features, triggered star + formation, morphological disturbance) is often exactly the science we want to measure, which is why both + galaxies are subjects of the fit. + +- **Chance projections**: two physically unrelated galaxies at different redshifts that happen to lie along + nearly the same line of sight. There is no physical interaction, but on the image their light still blends, + and measuring either galaxy's morphology or photometry requires decomposing the blend. + +In both cases the modeling challenge is the same: the flux in each pixel is the sum of both galaxies' +emission (blurred by the PSF), so the only principled way to measure either galaxy is to fit a model in which +both galaxies are present at once. This tutorial shows how to compose and fit that model, how to stop its +parameter space growing out of control, and what the data can — and cannot — tell us about a blend. + +__Contents__ + +- **Initial Setup:** Load imaging of two blended galaxies, simulating it first if it is not on disk. +- **Mask:** Define a mask which encloses the light of both galaxies. +- **Over Sampling:** Centre the adaptive over sampling grid on every galaxy, not just one. +- **Why Not Fit Them Separately?:** Why masking out one galaxy and fitting the other biases both measurements. +- **Model:** Compose a model with one free light model per galaxy, and count its parameters. +- **Fixing the Centres:** Fix each galaxy's centre to its observed light centre, and why this is standard. +- **Model Fit:** Fit the two-galaxy model to the data with a non-linear search. +- **Result:** Decompose the blend into each galaxy's light and measure per-galaxy photometry. +- **Light Decomposition Degeneracy:** The total flux is well constrained, but its split between the galaxies is not. +- **Wrap Up:** Summary and the road to cluster fields. +""" + +from autogalaxy import jax_wrapper # Sets JAX environment before other imports + +# from autogalaxy import setup_notebook; setup_notebook() + +from pathlib import Path +import autofit as af +import autogalaxy as ag +import autogalaxy.plot as aplt + +""" +__Initial Setup__ + +We begin with `Imaging` of two galaxies, separated by 2.0", whose light blends together: + + - The first galaxy's bulge is a `Sersic` centred at (0.0", -1.0"), with effective radius 0.8" and Sersic + index 2.5. + - The second galaxy's bulge is a `Sersic` centred at (0.0", 1.0"), with effective radius 0.6" and Sersic + index 3.0. + +Because both galaxies have Sersic indices well above 1, their light falls off slowly with radius — each +galaxy's outer envelope extends far past the midpoint between them, so a significant fraction of the flux in +every central pixel comes from *both* galaxies at once. This is the blend we must decompose. + +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +dataset_name = "sersic_x2" +dataset_path = Path("dataset") / "imaging" / dataset_name + +if ag.util.dataset.should_simulate(str(dataset_path)): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/sersic_x2.py"], + check=True, + ) + +dataset = ag.Imaging.from_fits( + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + psf_path=dataset_path / "psf.fits", + pixel_scales=0.1, +) + +""" +When we plot the data, the difference from every dataset we have modeled so far is obvious: there are two +bright galaxies, and between them their light merges into a continuous bridge of emission. There is no radius +at which we could draw a boundary and say "this flux belongs to galaxy 0, that flux to galaxy 1". +""" +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Mask__ + +We define a 3.0" circular mask centred between the two galaxies. For a blended system, the mask must enclose +the light of **both** galaxies — the fit is going to decompose the blend, so it needs to see the whole blend. +A mask sized by eye around either galaxy individually would cut through the other galaxy's light, guaranteeing +a biased fit. +""" +mask_radius = 3.0 + +mask = ag.Mask2D.circular( + shape_native=dataset.shape_native, + pixel_scales=dataset.pixel_scales, + radius=mask_radius, +) + +dataset = dataset.apply_mask(mask=mask) + +""" +__Over Sampling__ + +We use the adaptive over sampling scheme introduced in earlier chapters, which evaluates the steep central +regions of a galaxy's light at high resolution. The one multi-galaxy specific point is that the adaptive grid +is centred on **every** galaxy, not just one — each galaxy has its own steep central light profile needing +accurate evaluation, and `centre_list` takes as many centres as we give it. + +The two centres below are the observed centres of the two galaxies, which for this simulated dataset we know +exactly. For real data you would measure them from the image itself — the `autogalaxy_workspace`'s +`multi_galaxy` package loads them from a `galaxy_centres.json` file, which a GUI in the workspace writes from +mouse clicks on the image. +""" +galaxy_centres = [(0.0, -1.0), (0.0, 1.0)] + +over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from( + grid=dataset.grid, + sub_size_list=[4, 2, 2], + radial_list=[0.3, 0.6], + centre_list=galaxy_centres, +) + +dataset = dataset.apply_over_sampling(over_sample_size_lp=over_sample_size) + +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Why Not Fit Them Separately?__ + +Before composing the joint model, it is worth asking the obvious question: why not just fit each galaxy on its +own? We could mask out galaxy 1, fit galaxy 0, then swap and repeat — turning one hard problem into two easy +ones, each identical to the single-galaxy fits of chapter 2. + +The answer is that this biases **both** measurements, for two compounding reasons: + +- **Contaminated flux**: whatever mask we draw, the pixels we keep still contain the other galaxy's light — + the blend has no clean boundary. The fit attributes that extra flux to the galaxy being modeled, inflating + its measured brightness and dragging its effective radius and Sersic index towards the contaminating + neighbour. The neighbour's light is brightest exactly on the side facing it, so the fitted ellipticity and + centre are skewed in that direction too. + +- **Amputated flux**: the mask that removes the neighbour also removes part of the modeled galaxy's own outer + envelope, precisely where the two overlap. The fit never sees that light, so it underestimates how far the + galaxy extends — biasing the effective radius and Sersic index in the *opposite* direction to the first + effect. The two biases do not cancel; they combine into measurements that are simply wrong, in ways that + depend on the mask choice. + +The joint fit has neither problem. Every pixel in the mask is modeled as the sum of both galaxies' light, so +no flux is wrongly attributed and none is thrown away. Decomposing the blend *is* the fit — this is the whole +point of the multi-galaxy regime, and it is why both galaxies must be in the model even if only one of them is +the galaxy we ultimately care about scientifically. +""" + +""" +__Model__ + +We now compose the model, and here the multi-galaxy regime shows its teeth: every galaxy needs its own free +light model, so the model has one `Galaxy` entry per galaxy. + +We build each galaxy in a loop over the observed centres and store them in a dictionary as `galaxy_0`, +`galaxy_1`, etc. This list-based composition scales to any number of blended galaxies, and it is the same API +the `autogalaxy_workspace`'s `multi_galaxy` package uses — so nothing needs re-learning later. + +For each galaxy's light we use a Multi Gaussian Expansion (MGE) made of linear light profiles, built by the +utility function `ag.model_util.mge_model_from` (this hides the longer `Basis` composition API we stepped +through in the linear profiles tutorial of chapter 2). Each galaxy's 20 Gaussians add just **4 non-linear +parameters** — a shared centre and shared elliptical components, with every `sigma` fixed to log-spaced values +and every `intensity` solved for by the linear inversion. + +The `centre_prior_is_uniform=True` input places a uniform prior of width 0.2" on each galaxy's centre, +centred on its observed light centre — so `galaxy_0`'s centre priors are centred on (0.0", -1.0") and +`galaxy_1`'s on (0.0", 1.0"). This is essential in a blend: if both galaxies had identical broad centre +priors, the model would not know which component is which, and the search would waste time exploring +solutions where the two galaxies have swapped places (or worse, piled on top of one another). + +The MGE choice matters more here than anywhere we have used it before, because the multi-galaxy regime +multiplies whatever light model we choose by the number of galaxies: + + - Two full `Sersic` bulges (`ag.lp.Sersic`): **14** free parameters (7 each). + - Two linear `Sersic` bulges (`ag.lp_linear.Sersic`, intensities solved for): **12** free parameters. + - Two MGEs: **8** free parameters (4 each) — and each MGE is far more flexible than a single Sersic, + capturing the asymmetries and radially-varying ellipticity that real (especially interacting!) galaxies + show. + +A light model that is both cheap and flexible is close to essential in this regime — this is exactly where +the MGE shines. On top of the parameter count, solving the intensities by linear algebra has a second, subtler +benefit for blends that we will return to at the end of this tutorial. +""" +galaxy_dict = {} + +for i, centre in enumerate(galaxy_centres): + + bulge = ag.model_util.mge_model_from( + mask_radius=mask_radius, + total_gaussians=20, + centre_prior_is_uniform=True, + centre=(centre[0], centre[1]), + sigma_min=dataset.pixel_scales[0] / 10.0, + ) + + galaxy_dict[f"galaxy_{i}"] = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge) + +model = af.Collection(galaxies=af.Collection(**galaxy_dict)) + +""" +The model's `info` shows `galaxy_0` and `galaxy_1` each carrying their own free MGE — the signature of the +multi-galaxy regime — and its `prior_count` gives the total number of free parameters. +""" +print(model.info) + +print(f"Free parameters (free centres): {model.prior_count}") + +""" +The count is **8**: each galaxy contributes its MGE's 2 centre parameters and 2 elliptical component +parameters. Every further blended galaxy will add 4 more — the growth is linear, and with plain Sersic light +profiles instead of MGEs it would grow at 7 per galaxy instead. + +__Fixing the Centres__ + +Before fitting, we make one change that is standard practice for blended systems: we **fix each galaxy's +centre to its observed light centre**, removing 2 free parameters per galaxy. + +Why? With a single galaxy, the data constrains the centre extremely well: it is simply the peak of the light, +and there is only one plausible culprit for every photon. With blended light this breaks down. Each pixel's +flux is the sum of both galaxies' light, so a small shift of one galaxy's centre can be compensated by changes +to the other galaxy's model — its ellipticity, its Gaussians' relative intensities, or a shift of its own +centre. The free centres become degenerate with one another and with the rest of the model, and the search +wanders these degeneracies, converging slowly and sometimes settling in unphysical corners of parameter space +where one model galaxy has drifted between the two real ones. + +Fixing each centre to the observed light peak resolves this at minimal cost. Unlike almost every other light +profile parameter, the centre is something we can measure directly and robustly from the image before any +fitting — a peak position is hard to fake, even in a blend. (Measuring a genuine offset — say, of a nucleus +displaced during an interaction — is real science, but it is a *follow-up* fit performed after a robust model +with fixed centres has been found.) + +The `centre_fixed` input of `mge_model_from` fixes every Gaussian's centre to the input tuple, so the centre +is no longer a free parameter with a prior. +""" +galaxy_dict = {} + +for i, centre in enumerate(galaxy_centres): + + bulge = ag.model_util.mge_model_from( + mask_radius=mask_radius, + total_gaussians=20, + centre_fixed=(centre[0], centre[1]), + sigma_min=dataset.pixel_scales[0] / 10.0, + ) + + galaxy_dict[f"galaxy_{i}"] = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge) + +model = af.Collection(galaxies=af.Collection(**galaxy_dict)) + +""" +Printing the model's `info` again, each centre is now listed as a fixed value with no prior, and the free +parameter count has dropped from 8 to **4** — just each galaxy's two elliptical components. Pause on that +number: we are about to decompose a fully blended pair of galaxies, capturing both morphologies with 40 +Gaussians, by searching a 4-dimensional parameter space. Compare it to the 14 dimensions two full Sersics +would have cost, and it is clear how the combination of linear profiles, the MGE and fixed centres is what +makes this regime tractable at all. +""" +print(model.info) + +print(f"Free parameters (fixed centres): {model.prior_count}") + +""" +__Model Fit__ + +We fit the model with the nested sampling algorithm `Nautilus`, as in previous chapters. Thanks to the MGE and +the fixed centres the parameter space is small and well behaved, so `n_live=100` (the chapter 2 default) is +ample. + +Note that the `AnalysisImaging` object is completely unchanged from single-galaxy fitting — the multi-galaxy +regime changed the model composition, and nothing else. +""" +search = af.Nautilus( + path_prefix=Path("howtogalaxy", "chapter_4"), + name="tutorial_2_multi_galaxy", + unique_tag=dataset_name, + n_live=100, + n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. +) + +analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True) + +print( + "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_4/tutorial_2_multi_galaxy" + " folder for live output of the results, images and model." + " This Jupyter notebook cell with progress once search has completed - this could take some time!" +) + +result = search.fit(model=model, analysis=analysis) + +""" +__Result__ + +The result's `info` shows the inferred model, with each galaxy's parameters listed under its own `galaxy_0` / +`galaxy_1` entry. +""" +print(result.info) + +""" +The fit subplot shows the joint model reproducing the full blend — both galaxies and the bridge of overlapping +light between them — with residuals at the noise level. +""" +aplt.subplot_fit_imaging(fit=result.max_log_likelihood_fit) + +""" +The core deliverable of a multi-galaxy fit is the **decomposition**: `subplot_fit_imaging_of_galaxy` shows, +for each galaxy in turn, its modeled light on its own and the data with the *other* galaxy's model subtracted. +This is the closest thing to an image of each galaxy as it would appear without its companion — something no +mask could ever have given us. +""" +for i in range(len(galaxy_centres)): + aplt.subplot_fit_imaging_of_galaxy(fit=result.max_log_likelihood_fit, galaxy_index=i) + +""" +With the blend decomposed, per-galaxy photometry is direct: each galaxy's model image contains only its own +light, so summing it gives that galaxy's flux uncontaminated by its neighbour. +""" +galaxies = result.max_log_likelihood_galaxies + +for i, galaxy in enumerate(galaxies): + image = galaxy.image_2d_from(grid=dataset.grids.lp) + print(f"galaxy_{i}: total model flux = {float(image.array.sum()):.3f}") + +""" +__Light Decomposition Degeneracy__ + +The photometry above comes with an important caveat, and it is the deepest lesson of the multi-galaxy regime. + +In the pixels where the two galaxies overlap, the data constrains only the **sum** of their light. A model +that brightens galaxy 0's outer envelope and dims galaxy 1's by the same amount produces a nearly identical +image, and therefore a nearly identical likelihood. The *total* flux of the blend is pinned down superbly — +every photon is accounted for — but the *split* of that flux between the two galaxies is constrained only by +the shapes of the profiles, which is a much weaker lever. The closer the pair and the larger their overlap, +the weaker it gets. + +You can see this degeneracy directly in the posterior. When you run this tutorial for real (rather than +skimming the generated output), open the search's corner plot — the `.png` in the output folder's `image` +directory, or plot it yourself with `aplt.corner_anesthetic(samples=result.samples)` — and focus on the panels +pairing `galaxy_0`'s parameters against `galaxy_1`'s. You will see tilted, anti-correlated contours: when one +galaxy's model claims more of the shared light, the other's gives it up. Each galaxy's flux, effective radius +and shape measurements inherit the full width of this degeneracy, so their error bars are larger — sometimes +much larger — than an equivalent isolated galaxy's would be. Honest science on blended systems propagates +these widened uncertainties rather than quoting the best-fit split as if it were certain, and this is +precisely why we fit blends with a sampler that maps the full posterior rather than an optimizer that returns +a single point: the best-fit sits somewhere on the degeneracy ridge and tells you nothing about the ridge's +length. + +This is also the subtler benefit of linear light profiles promised earlier. The flux ratio between the two +galaxies — the quantity most degenerate in a blend — lives in the `intensity` values, and the inversion solves +those exactly at every likelihood evaluation. The non-linear search never has to explore the intensity +degeneracy stochastically; it is handled by linear algebra, leaving the sampler to map only the (much milder) +degeneracies between the galaxies' shapes. Fitting a blend with standard (non-linear) light profiles forfeits +this, and is markedly slower and less reliable as a result. + +__Wrap Up__ + +In this tutorial, we modeled a system of two blended galaxies simultaneously. Let's summarise what we've +learnt: + +- **Co-dominant galaxies**: When two or more galaxies of comparable brightness overlap on the sky, every one + of them needs its own free light model — unlike the nuisance neighbours of the previous tutorial, none can + be masked away or simplified. Such blends arise from interacting pairs and mergers at one redshift, or + chance projections of unrelated galaxies along the line of sight. + +- **Separate fits are biased**: Masking one galaxy out and fitting the other contaminates the fit with the + neighbour's flux while amputating the target's own overlapping light — both measurements come out wrong. + The joint fit, in which every pixel is modeled as the sum of both galaxies, is the only principled + decomposition. + +- **Parameter accounting**: Two full Sersics would cost 14 free parameters; linear profiles cut this to 12, + the MGE to 8, and fixing each galaxy's centre to its observed light centre to just 4 — with the MGE's + intensities solved by linear algebra rather than sampled. Cheap, flexible light models are what make the + multi-galaxy regime tractable, because every additional galaxy multiplies the cost. + +- **Fixed centres**: Blended light makes free centres degenerate — one galaxy's centre shift can be absorbed + by changes to the other's model. Fixing each centre to the observed light peak, the one parameter we can + measure robustly before fitting, is the standard trick for taming this. + +- **Light decomposition degeneracy**: The data constrains the total flux of the blend far better than its + split between the galaxies, producing anti-correlated posteriors between the two galaxies' parameters and + inflating the uncertainties on all per-galaxy measurements. Map it with a sampler, and propagate it. + +Everything here scaled comfortably to two galaxies, and the loop-based composition would carry us to three or +four. But the ladder keeps climbing: in the next tutorial we reach **cluster fields**, where a brightest +cluster galaxy sits among tens to hundreds of member galaxies. Giving every member its own free model — even +a 2-parameter one — cannot scale that far, and modeling them one blend at a time would reintroduce every bias +we just eliminated. The answer is catalogue-driven composition: the member population is built automatically +from a catalogue of measured positions and photometry, and modeled collectively. That is where we go next. +""" diff --git a/scripts/simulators/sersic_x2.py b/scripts/simulators/sersic_x2.py index 2dd522b..21eab79 100644 --- a/scripts/simulators/sersic_x2.py +++ b/scripts/simulators/sersic_x2.py @@ -7,7 +7,7 @@ - The first galaxy's bulge is an `Sersic`. - The second galaxy's bulge is an `Sersic`. -This dataset is used in chapter 3 of the **HowToGalaxy** lectures. +This dataset is used in chapter 4 of the **HowToGalaxy** lectures. __Contents__ From 643d1126ebdcb095eac8e5407a11fb55544ca497 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:37:09 +0000 Subject: [PATCH 06/15] =?UTF-8?q?docs:=20write=20chapter=204=20tutorial=20?= =?UTF-8?q?3=20=E2=80=94=20cluster=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the scaling-up chapter and the series: what a cluster field looks like (BCG + members + intra-cluster light), why composition becomes catalogue-driven at this scale, the two-tier model (free MGE BCG, catalogue members sharing one intensity-scale parameter), fitting and per-member results access, and the closing ladder recap handing off to the workspace and HowToLens. Backed by a new cluster simulator mirroring the workspace's product set including the member CSV. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- .../tutorial_3_cluster.py | 403 ++++++++++++++++++ scripts/simulators/cluster.py | 229 ++++++++++ 2 files changed, 632 insertions(+) create mode 100644 scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py create mode 100644 scripts/simulators/cluster.py diff --git a/scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py b/scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py new file mode 100644 index 0000000..dbef76f --- /dev/null +++ b/scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py @@ -0,0 +1,403 @@ +""" +Tutorial 3: Cluster +=================== + +Throughout this chapter we have been scaling up: from a single galaxy with extra galaxies nearby, to blended +multi-galaxy systems where every galaxy received its own free light model. + +This tutorial climbs the final rung: **cluster fields**, the richest environments in which galaxies live. An +image of a galaxy cluster contains: + +- **A brightest cluster galaxy (BCG)**: the giant elliptical that sits at the cluster's centre. BCGs are the + most massive galaxies in the Universe, built up over billions of years by swallowing their neighbours, and + their extended envelopes hold a record of that assembly history. + +- **Tens to hundreds of member galaxies**: the cluster's population of (mostly elliptical) galaxies, + scattered across a field many times wider than the single-galaxy images of earlier chapters. + +- **Intra-cluster light (ICL)**: a faint, diffuse glow of stars stripped from member galaxies, filling the + space between them. It blends smoothly into the BCG's outer envelope, which is why careful modeling of the + BCG's light is the starting point of every ICL measurement. + +Why model all of this light? Three science cases drive the cluster regime: + +- **BCG growth**: the size and shape of a BCG's outer envelope constrains how much of its mass was accreted + from merging satellites — but measuring that envelope requires subtracting every member galaxy whose light + overlaps it. + +- **Intra-cluster light**: the ICL is only measurable once the BCG and members are modeled and removed; + what remains is the diffuse component. + +- **Member photometry**: the luminosity of each member, uncontaminated by its neighbours and the BCG's + envelope, is what luminosity functions and scaling relations are built from. + +(A quick note for readers heading towards gravitational lensing: galaxy clusters are also the Universe's +most powerful strong lenses, and the lensing version of this scale — where a catalogue like the one below +drives member *masses* and the fit is to point-source image positions rather than galaxy light — lives in +chapter 4 of the **HowToLens** lectures.) + +__The Scaling Problem__ + +The previous tutorials composed models the way we have all chapter: one `af.Model(ag.Galaxy)` per galaxy, +each with its own free light profile. A blend of two or three galaxies is fine. A cluster is not: + +- **Parameters**: ten members with a free spherical Sersic each is 40 free parameters before we even touch + the BCG; a hundred members is 400. No non-linear search can sample such a space reliably. + +- **Information**: the faint members do not contain enough signal to constrain four free parameters each — + most of those dimensions would be unconstrained noise. + +- **Practicality**: hand-writing Python model code for hundreds of galaxies is not sensible, and no + astronomer works that way. What an observer actually has is a **catalogue**: a table listing where each + member is and how bright it is. + +The cluster regime therefore changes how the model is composed. The BCG — the galaxy whose structure we +care most about — is modeled individually and richly, with a free MGE, exactly as a single galaxy would be. +The member population is driven by the catalogue: each member's centre is fixed to its catalogue position, +its shape is fixed to sensible values, and its intensity is TIED to its catalogue luminosity through a +single shared free normalization. Adding a member is a row append; the model's dimensionality does not grow +with the population. + +__Contents__ + +- **Dataset:** Load the simulated cluster field (1 BCG + 10 members), auto-simulating it if absent. +- **Member Catalogue:** Load the member centres and luminosities from `scaling_galaxies.csv`. +- **Masking:** Mask the wide cluster field and over-sample every galaxy's centre. +- **Model:** Compose the two-tier cluster model — free BCG MGE + catalogue-driven member tier. +- **Search + Analysis:** Configure the Nautilus non-linear search and the analysis. +- **Model Fit:** Run the fit. +- **Result:** Inspect the fit, including the member-subtracted BCG decomposition. +- **Per-Member Results:** Access each member's fitted light — catalogue-scale photometry. +- **Refinements:** Where the composition goes next — freeing tier shapes, promoting bright members. +- **Wrap Up:** The chapter and the **HowToGalaxy** lectures conclude. +""" + +from autogalaxy import jax_wrapper # Sets JAX environment before other imports + +# from autogalaxy import setup_notebook; setup_notebook() + +from pathlib import Path + +import autofit as af +import autogalaxy as ag +import autogalaxy.plot as aplt + +""" +__Dataset__ + +We fit a simulated cluster field kept deliberately small so it runs quickly — 1 BCG and 10 member galaxies +in a 25" x 25" field — but containing every ingredient of the cluster regime. A real cluster simply has more +members, and we will see that scaling the model up to hundreds of them does not add a single free parameter. +""" +dataset_name = "simple" +dataset_path = Path("dataset") / "cluster" / dataset_name + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if ag.util.dataset.should_simulate(str(dataset_path)): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/cluster.py"], + check=True, + ) + + +dataset = ag.Imaging.from_fits( + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + psf_path=dataset_path / "psf.fits", + pixel_scales=0.1, +) + +""" +Plotting the dataset shows what a cluster field looks like in imaging: the bright, extended BCG dominating +the centre, with the fainter members scattered across the frame. In real data the diffuse intra-cluster +light would fill the space between them. + +Note the field of view compared to earlier chapters — the members sit up to ~8" from the centre, so the +image is far larger than the ~6" cutouts we fitted before. +""" +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Member Catalogue__ + +The member population enters the model through a catalogue file, `scaling_galaxies.csv`, with one row per +member galaxy and three columns: + + - `y`, `x`: the member's centre in arc-seconds — where the galaxy is. + - `luminosity`: the member's luminosity — how bright it is. Any consistent luminosity units work, because + (as we will see below) only a shared normalization of the luminosities is fitted. + +This is exactly what an observer measures: a photometry catalogue. In a real analysis this CSV comes +straight from source-extraction software or a survey database — it is spreadsheet-editable, and adding a +member to the model means appending a row, not writing Python. + +We load it with `ag.galaxy_table_from_csv`, the catalogue-loading API. +""" +scaling_table = ag.galaxy_table_from_csv( + file_path=dataset_path / "scaling_galaxies.csv" +) + +member_centres = scaling_table.centres.in_list +member_luminosities = scaling_table.luminosities + +print(f"Members in catalogue: {len(member_centres)}") +print(f"First member centre: {member_centres[0]}") +print(f"First member luminosity: {float(member_luminosities[0])}") + +""" +The BCG's centre is recorded in its own file, `bcg_centres.json` — in a real cluster there may be more than +one dominant galaxy (e.g. a second bright elliptical) that deserves individual modeling, so their centres +are kept separate from the member catalogue. +""" +bcg_centres = ag.from_json(file_path=dataset_path / "bcg_centres.json") + +print(f"BCG centre: {bcg_centres[0]}") + +""" +__Masking__ + +We mask the field generously — the members span the frame, and the model must account for every galaxy +inside the mask (an unmodeled galaxy inside the mask would bias the fit, as the first tutorial of this +chapter showed). + +The over-sampling scheme is centred on every galaxy in the field — the BCG and all ten members — so each +galaxy's steep central light gradient is evaluated accurately. +""" +mask_radius = 11.0 + +mask = ag.Mask2D.circular( + shape_native=dataset.shape_native, + pixel_scales=dataset.pixel_scales, + radius=mask_radius, +) + +dataset = dataset.apply_mask(mask=mask) + +over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from( + grid=dataset.grid, + sub_size_list=[4, 2, 2], + radial_list=[0.3, 0.6], + centre_list=list(bcg_centres) + list(member_centres), +) + +dataset = dataset.apply_over_sampling(over_sample_size_lp=over_sample_size) + +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Model__ + +The model has two tiers, and this two-tier composition is the cluster regime's signature: + +**Tier 1 — the BCG, modeled richly.** The BCG is the science target, so it gets the most flexible light +model we have: a free Multi-Gaussian Expansion (the MGE introduced in chapter 2), composed via +`ag.model_util.mge_model_from`. Its 20 Gaussians can capture the BCG's extended envelope, radially-varying +ellipticity and isophotal twists — exactly the structures that BCG growth and intra-cluster light studies +measure. This costs the usual handful of non-linear parameters (centre + elliptical components; the +Gaussian intensities are solved linearly). + +**Tier 2 — the members, modeled cheaply from the catalogue.** Each member gets a spherical Sersic +(`SersicSph`) generated in a loop over the catalogue rows: + + - its `centre` is FIXED to the catalogue (y, x) position; + - its shape (`effective_radius`, `sersic_index`) is FIXED to values typical of cluster ellipticals; + - its `intensity` is TIED to the catalogue luminosity through one shared free parameter: + + intensity_i = intensity_scale * luminosity_i + +The single `intensity_scale` prior is defined once, outside the loop, and every member's intensity is an +arithmetic expression of it — the prior-linking arithmetic we have used throughout the lectures, now doing +population-scale work. The whole 10-member tier therefore contributes ONE free parameter to the non-linear +search, and would still contribute one with 200 members. + +Physically, `intensity_scale` is the conversion between the catalogue's luminosity units and the image's +flux units: the catalogue fixes each member's brightness *relative* to the others (which photometry measures +well), and the fit solves for the one overall normalization. +""" +# BCG: + +bulge = ag.model_util.mge_model_from( + mask_radius=3.0, + total_gaussians=20, + centre_prior_is_uniform=True, + centre=(bcg_centres[0][0], bcg_centres[0][1]), + sigma_min=dataset.pixel_scales[0] / 10.0, +) + +galaxy_dict = {"bcg": af.Model(ag.Galaxy, redshift=0.5, bulge=bulge)} + +# Members: one shared free normalization for the whole tier. + +intensity_scale = af.UniformPrior(lower_limit=0.0, upper_limit=10.0) + +for i, (centre, luminosity) in enumerate(zip(member_centres, member_luminosities)): + + bulge = af.Model(ag.lp.SersicSph) + bulge.centre = tuple(centre) + bulge.intensity = intensity_scale * float(luminosity) # tied to the catalogue + bulge.effective_radius = 0.6 + bulge.sersic_index = 3.0 + + galaxy_dict[f"member_{i}"] = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge) + +model = af.Collection(galaxies=af.Collection(**galaxy_dict)) + +""" +Printing `model.info` confirms the composition: the BCG's MGE parameters are free, every member's intensity +shows as tied to the single shared `intensity_scale`, and the total dimensionality stays small despite the +model containing 11 galaxies. This is the punchline of the cluster workflow — catalogues decouple the +model's physical richness from the dimensionality of the search. +""" +print(model.info) + +""" +__Search + Analysis__ + +The analysis is the same `AnalysisImaging` used on every rung of the ladder — the cluster regime changed how +the model is *composed*, not how it is *fitted*. We fit with the Nautilus nested sampler, as throughout the +lectures, which returns the full posterior: the errors on the BCG's structural parameters and on +`intensity_scale` are what a cluster paper reports. +""" +search = af.Nautilus( + path_prefix=Path("howtogalaxy", "chapter_4"), + name="tutorial_3_cluster", + unique_tag=dataset_name, + n_live=150, + n_batch=50, +) + +analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True) + +""" +__Model Fit__ + +Run the fit. Despite the model containing 11 galaxies, the parameter space is barely larger than a single +galaxy's — this is the catalogue tier doing its job. +""" +print( + "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_4/tutorial_3_cluster" + " folder for live output of the results, images and model." + " This Jupyter notebook cell with progress once search has completed - this could take some time!" +) + +result = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +__Result__ + +The `info` attribute shows the resulting model — the BCG's MGE parameters with their errors, and the single +inferred `intensity_scale` shared by the member tier. +""" +print(result.info) + +""" +Plotting the maximum likelihood fit shows the full cluster model reproducing the field: BCG and all ten +members fitted simultaneously. +""" +aplt.subplot_fit_imaging(fit=result.max_log_likelihood_fit) + +""" +The plot below is the one that motivates cluster light modeling: the fit decomposed by galaxy, showing the +BCG's light (galaxy index 0) in isolation — that is, with every member galaxy's light accounted for by its +own model component rather than contaminating the BCG's. + +This member-subtracted BCG is the measurement BCG-growth and intra-cluster light studies are built on: its +outer envelope can now be traced without the members' light biasing it, and whatever diffuse light the model +cannot attribute to any galaxy is the ICL candidate signal. +""" +aplt.subplot_fit_imaging_of_galaxy(fit=result.max_log_likelihood_fit, galaxy_index=0) + +""" +__Per-Member Results__ + +The member tier's single free parameter was `intensity_scale`, but the maximum likelihood instance contains +the fully-realised light profile of every member — the tie to the catalogue has been applied, so each member +carries the intensity its luminosity implies. Looping over the members tabulates the fitted photometry of +the whole population, exactly as a member-photometry analysis of a real cluster would. +""" +instance = result.max_log_likelihood_instance + +for i in range(len(member_centres)): + member = getattr(instance.galaxies, f"member_{i}") + print( + f"Member {i}: centre={member.bulge.centre}, " + f"intensity={float(member.bulge.intensity):.3f}" + ) + +""" +The simulator built every member with `intensity = luminosity` — a true shared normalization of 1.0. We can +check the fit recovered it by dividing any member's fitted intensity by its catalogue luminosity. + +(A real, full run recovers a value close to 1.0; if you are running this script in the fast test mode used +for automated checks, the search is bypassed and the printed value is a prior median, not meaningful.) +""" +recovered_scale = float(instance.galaxies.member_0.bulge.intensity) / float( + member_luminosities[0] +) + +print(f"Recovered member-tier intensity_scale = {recovered_scale:.3f} (truth: 1.0)") + +""" +__Refinements__ + +This tutorial used the simplest catalogue tier — fixed shapes, one shared normalization — which is the right +starting point. Real cluster analyses then reach for two refinements, both of which preserve the +row-append scaling: + +- **Free the tier's shape**: promote the shared `effective_radius` and `sersic_index` from fixed values to + parameters shared by the whole tier. The tier then costs 3 free parameters instead of 1 — still + independent of the member count. + +- **Promote bright members**: give the brightest few members their own free light models alongside the BCG, + since the data constrains them well. Promotion costs the full per-galaxy parameters, so promote sparingly + — brightest first, and only while the data keeps constraining them. + +Both are demonstrated in `autogalaxy_workspace/*/cluster/modeling.py`, which fits this same dataset. + +__Wrap Up__ + +This tutorial completes the chapter's ladder of scale, and with it the **HowToGalaxy** lectures. Lets recap +the ladder one final time: + +- **One galaxy** (chapters 1-3): light profiles, non-linear searches, linear profiles and the MGE, and + pixelized reconstructions — the core toolkit, applied to a single galaxy at the centre of its image. + +- **Extra galaxies**: nearby galaxies whose light contaminates the target's, included in the model so the + target's photometry is unbiased. + +- **Multi-galaxy blends**: systems of overlapping galaxies modeled simultaneously, each with its own free + light model. + +- **Cluster fields** (this tutorial): a BCG modeled richly plus a member population too numerous for free + models, driven instead by a catalogue — fixed centres, luminosity-tied intensities, one shared free + normalization. Model complexity decoupled from population size. + +The theme of the chapter has been that scaling up is about *composition*, not new fitting machinery: the +same `af.Model` / `af.Collection` API, the same `AnalysisImaging`, the same searches — arranged so that the +information in the data decides which parameters are free. + +And that theme closes the series: you now know how to simulate and fit galaxy imaging, compose models from +simple Sersics to MGEs and pixelizations, run and interpret non-linear searches, and scale all of it from +one galaxy to a cluster field. Where to go next: + +- `autogalaxy_workspace`: the destination for real science — the `imaging`, `multi_galaxy` and `cluster` + packages mirror this chapter's ladder with production-ready scripts (including interferometer data, + multi-wavelength fitting and ellipse fitting, which the lectures did not cover), and its `guides` cover + every API in depth. + +- **HowToLens**: the companion lecture series for strong gravitational lensing, where galaxies like these + become lenses — its chapter 4 climbs this same ladder on the mass side, with the cluster rung fitting + point-source image positions driven by the very same catalogue machinery. + +Congratulations on finishing the **HowToGalaxy** lectures — now go model some galaxies! +""" diff --git a/scripts/simulators/cluster.py b/scripts/simulators/cluster.py new file mode 100644 index 0000000..688d641 --- /dev/null +++ b/scripts/simulators/cluster.py @@ -0,0 +1,229 @@ +""" +Simulator: Cluster +================== + +This script simulates `Imaging` of a cluster field: a brightest cluster galaxy (BCG) surrounded by ten +lower-luminosity member galaxies. It is used in chapter 4 of the **HowToGalaxy** lectures, where the member +population is modeled via a **catalogue**: a CSV of member centres and luminosities whose photometry pins +the faint galaxies while only a shared normalization stays free. + +This script simulates `Imaging` of a cluster field where: + + - The BCG's bulge is an elliptical `Sersic` (de Vaucouleurs-like). + - Ten member galaxies have `SersicSph` light profiles whose intensities follow their catalogue + luminosities. + +__Contents__ + +- **Dataset Paths:** Set the output path for the simulated dataset. +- **Grid:** Create a 2D grid with adaptive over-sampling for simulation. +- **Galaxies:** Define the BCG and member galaxy light profiles used for simulation. +- **Output:** Save the simulated dataset to FITS files. +- **Member Catalogue CSV:** Write `scaling_galaxies.csv` (y, x, luminosity) — the tutorial's input. +- **Visualize:** Output subplot and image PNGs of the simulated dataset. +- **Plane Output:** Save the Galaxies object and BCG centre as JSON files. + +__Start Here Notebook__ + +If any code in this script is unclear, refer to the `simulators/simple.ipynb` notebook. +""" + +# from autogalaxy import setup_notebook; setup_notebook() + +import csv +from pathlib import Path +import autogalaxy as ag +import autogalaxy.plot as aplt + +""" +__Dataset Paths__ + +The `dataset_type` describes the type of data being simulated and `dataset_name` gives it a descriptive name. +""" +dataset_type = "cluster" +dataset_name = "simple" + +dataset_path = Path("dataset", dataset_type, dataset_name) + +""" +__Grid__ + +The cluster field spans a much wider area than the single-galaxy datasets of earlier chapters, so the grid +is 250 x 250 pixels (25" x 25" at 0.1" per pixel). + +The BCG sits at the centre of the field and the ten member galaxies are scattered across it. Their centres +and luminosities are defined here because both the light profiles and the member catalogue CSV are built +from them below. +""" +grid = ag.Grid2D.uniform( + shape_native=(250, 250), + pixel_scales=0.1, +) + +bcg_centre = (0.0, 0.0) + +member_centres = [ + (5.5, -6.5), + (-7.5, 3.0), + (3.0, 8.0), + (8.0, 5.0), + (-6.5, -8.0), + (-2.5, 6.5), + (7.0, -2.0), + (-8.0, 8.5), + (2.0, -8.5), + (-4.0, -3.5), +] + +member_luminosities = [0.40, 0.32, 0.25, 0.20, 0.16, 0.13, 0.10, 0.08, 0.06, 0.05] + +""" +Simulate the image using a (y,x) grid with the adaptive over sampling scheme, centred on every galaxy in +the field (the BCG and all ten members). +""" +over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from( + grid=grid, + sub_size_list=[32, 8, 2], + radial_list=[0.3, 0.6], + centre_list=[bcg_centre] + member_centres, +) + +grid = grid.apply_over_sampling(over_sample_size=over_sample_size) + +""" +Simulate a simple Gaussian PSF for the image. +""" +psf = ag.Convolver.from_gaussian( + shape_native=(11, 11), sigma=0.1, pixel_scales=grid.pixel_scales +) + +""" +Create the simulator for the imaging data, which defines the exposure time, background sky, noise levels and psf. +""" +simulator = ag.SimulatorImaging( + exposure_time=300.0, + psf=psf, + background_sky_level=0.1, + add_poisson_noise_to_data=True, +) + +""" +__Galaxies__ + +Setup the BCG: a bright, extended de Vaucouleurs-like Sersic at the cluster centre. In the chapter 4 +cluster tutorial it is the one galaxy modeled individually, with a free MGE. +""" +bcg = ag.Galaxy( + redshift=0.5, + bulge=ag.lp.Sersic( + centre=bcg_centre, + ell_comps=ag.convert.ell_comps_from(axis_ratio=0.8, angle=45.0), + intensity=1.5, + effective_radius=2.5, + sersic_index=4.0, + ), +) + +""" +Setup the ten member galaxies, whose central intensities equal their catalogue luminosities — so the +rendered image visibly traces the catalogue, and the tutorial's shared-normalization tier +(intensity = scale * luminosity) can recover the truth with `scale = 1`. +""" +members = [] +for centre, luminosity in zip(member_centres, member_luminosities): + members.append( + ag.Galaxy( + redshift=0.5, + bulge=ag.lp.SersicSph( + centre=centre, + intensity=luminosity, + effective_radius=0.6, + sersic_index=3.0, + ), + ) + ) + +""" +Use these galaxies to generate the image for the simulated `Imaging` dataset. +""" +galaxies = ag.Galaxies(galaxies=[bcg] + members) +aplt.plot_array(array=galaxies.image_2d_from(grid=grid), title="Image") + +""" +Pass the simulator galaxies, which creates the image which is simulated as an imaging dataset. +""" +dataset = simulator.via_galaxies_from(galaxies=galaxies, grid=grid) + +""" +Plot the simulated `Imaging` dataset before outputting it to fits. +""" +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Output__ + +Output the simulated dataset to the dataset path as .fits files. +""" +aplt.fits_imaging( + dataset=dataset, + data_path=dataset_path / "data.fits", + psf_path=dataset_path / "psf.fits", + noise_map_path=dataset_path / "noise_map.fits", + overwrite=True, +) + +""" +__Member Catalogue CSV__ + +Write the member catalogue to `scaling_galaxies.csv` with the three-column `y, x, luminosity` schema used +by the `autogalaxy_workspace` cluster package (and shared with the lensing workspace's cluster package, +where the same catalogue drives member MASSES via a scaling relation; here it drives member LIGHT). The +tutorial loads it with `ag.galaxy_table_from_csv` — the catalogue-loading API that makes the member +population a row-append away from scaling up. +""" +with open(dataset_path / "scaling_galaxies.csv", "w", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(["y", "x", "luminosity"]) + for centre, luminosity in zip(member_centres, member_luminosities): + writer.writerow([centre[0], centre[1], luminosity]) + +""" +__Visualize__ + +Output a subplot of the simulated dataset, the image and the galaxies quantities to the dataset path as .png files. +""" +aplt.subplot_imaging_dataset( + dataset=dataset, output_path=dataset_path, output_format="png" +) +aplt.plot_array( + array=dataset.data, title="Data", output_path=dataset_path, output_format="png" +) + +aplt.subplot_galaxies( + galaxies=galaxies, grid=grid, output_path=dataset_path, output_format="png" +) + +""" +__Plane Output__ + +Save the `Galaxies` in the dataset folder as a .json file, ensuring the true light profiles and galaxies +are safely stored and available to check how the dataset was simulated in the future. + +This can be loaded via the method `galaxies = ag.from_json()`. + +The BCG centre is also saved as a .json file, mirroring the real-cluster workflow where the BCG centre(s) +are recorded separately from the member catalogue. +""" +ag.output_to_json( + obj=galaxies, + file_path=Path(dataset_path, "galaxies.json"), +) + +ag.output_to_json( + obj=ag.Grid2DIrregular([bcg_centre]), + file_path=Path(dataset_path, "bcg_centres.json"), +) + +""" +The dataset can be viewed in the folder `dataset/cluster/simple`. +""" From 4adaeb7ab51789b87357c84fd6086b5a17081bd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:37:09 +0000 Subject: [PATCH 07/15] =?UTF-8?q?docs:=20write=20chapter=204=20tutorial=20?= =?UTF-8?q?1=20=E2=80=94=20extra=20galaxies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opens the scaling-up chapter with the crowded-fields ladder framing, then the two ways to handle an interloper galaxy in the field: noise scaling its light out of the fit, or modeling it explicitly through the extra_galaxies model interface with its centre fixed to the observed light. The decision framing is purely light contamination — no lensing. Backed by a new extra_galaxy simulator mirroring the workspace's product set (mask + centres artifacts included). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- .../tutorial_1_extra_galaxies.py | 435 ++++++++++++++++++ scripts/simulators/extra_galaxy.py | 238 ++++++++++ 2 files changed, 673 insertions(+) create mode 100644 scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py create mode 100644 scripts/simulators/extra_galaxy.py diff --git a/scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py b/scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py new file mode 100644 index 0000000..1548559 --- /dev/null +++ b/scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py @@ -0,0 +1,435 @@ +""" +Tutorial 1: Extra Galaxies +========================== + +Welcome to chapter 4 of **HowToGalaxy**, where we scale up galaxy modeling beyond a single galaxy. + +In chapters 1 to 3, every dataset we studied had the same simple anatomy: one galaxy, alone at the centre of the +image, whose light we modeled with parametric profiles, basis functions or pixelizations. This is the cleanest +possible configuration, and it was the right place to learn the fundamentals of light profiles, non-linear searches +and Bayesian inference. + +Real galaxies are rarely this tidy. Galaxies live in crowded fields: a galaxy may have a faint companion a few +arc-seconds away, be blended with a close neighbour of comparable brightness, or sit in a cluster containing +hundreds of members. The science of galaxy modeling scales up through this hierarchy, and so must our modeling: + +- **Extra galaxies (this tutorial)**: a single galaxy of interest, with one or more nearby interloper galaxies + whose light may contaminate the analysis. + +- **Multi-galaxy blends (next tutorial)**: systems where two or more galaxies are blended together and all are + subjects of the analysis, so no single galaxy can be called "the" galaxy. + +- **Cluster fields (final tutorial)**: images containing many galaxies, for example a brightest cluster galaxy + surrounded by a population of cluster members, which must be modeled together. + +The same ladder exists in strong gravitational lensing, where interlopers, multi-galaxy deflectors and cluster-scale +lenses complicate the analysis in analogous ways. The **HowToLens** lectures scale up lens modeling through an +equivalent chapter 4, using the same **PyAutoFit** API you will learn here. + +In this first tutorial, we take the first step up in scale: a galaxy with one extra galaxy nearby. We will learn +how to decide whether the extra galaxy matters, and the two approaches to dealing with it when it does: removing +its light from the data, or including it in the model. + +__Contents__ + +- **Initial Setup:** Load the imaging dataset of a galaxy with an extra galaxy and inspect the interloper. +- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. +- **The Decision:** The core question: does the extra galaxy's light overlap the region of the image we fit? +- **Mask:** Define a circular mask large enough to include the extra galaxy's emission. +- **Approach 1 Noise Scaling:** Remove the extra galaxy's light from the fit by scaling its data and noise values. +- **Noise Scaling Fit:** Fit a model to the noise-scaled data, without the extra galaxy in the model. +- **Approach 2 Extra Galaxies Model:** Include the extra galaxy's light in the model explicitly. +- **Extra Galaxy Centres:** Why the extra galaxy's centre is fixed to its observed light centre. +- **Extra Galaxies Model Composition:** Compose the model including the extra galaxy via the extra galaxies API. +- **Extra Galaxies Fit:** Fit the model which includes the extra galaxy. +- **Which Approach When:** Guidance on choosing between noise scaling and explicit modeling. +- **Wrap Up:** Summary of the script and next steps. +""" + +from autogalaxy import jax_wrapper # Sets JAX environment before other imports + +# from autogalaxy import setup_notebook; setup_notebook() + +from pathlib import Path +import autogalaxy as ag +import autogalaxy.plot as aplt +import autofit as af + +""" +__Initial Setup__ + +Lets load the `Imaging` dataset we'll fit in this tutorial. It is similar to the `simple__sersic` dataset used +throughout chapter 2, where: + + - The galaxy's bulge is an `Sersic`. + +However, there is one addition: an extra galaxy, with its own light (an `ExponentialSph`), located a few +arc-seconds from the main galaxy. +""" +dataset_name = "extra_galaxy" +dataset_path = Path("dataset") / "imaging" / dataset_name + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if ag.util.dataset.should_simulate(str(dataset_path)): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/extra_galaxy.py"], + check=True, + ) + +dataset = ag.Imaging.from_fits( + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + psf_path=dataset_path / "psf.fits", + pixel_scales=0.1, +) + +""" +When we plot the dataset, the familiar sight of a single galaxy at the centre of the image is joined by a blob of +light in the upper-right of the image. + +This is the extra galaxy. It is not the galaxy we are studying: it is an interloper, a galaxy that happens to lie +close to ours on the sky. Interlopers like this are extremely common in real imaging of galaxies, and everyone who +models galaxies has to decide what to do about them. +""" +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__The Decision__ + +The decision of what to do about an extra galaxy hinges on one question: + +**Does its light overlap the region of the image we fit?** + +The extra galaxy's emission may extend into the region of the image we fit. If it does, and we fit a model +containing only the main galaxy, the model has no component that can produce this emission. The fit will respond by +distorting the main galaxy's light profiles to soak it up — for example inflating the `effective_radius` or skewing +the `ell_comps` towards the interloper — biasing every parameter we infer about the galaxy's structure. + +If the extra galaxy's light does not reach the fitted region (it is faint, or far away, or both), we can simply +ignore it. When it does, we have two approaches: + +- **Approach 1 (remove its light)**: We can remove the extra galaxy's emission from the fit entirely, without + adding anything to the model. + +- **Approach 2 (model it explicitly)**: If its light blends too closely with the main galaxy's emission to cleanly + remove, we include the extra galaxy in the model, with its own light profile. + +We will now perform both, and at the end of the tutorial discuss when each is appropriate. + +__Mask__ + +We first define the circular mask used to fit the data. In chapter 2 we typically used a 2.5" - 3.0" mask, which +tightly contained the galaxy. + +Here, we use a larger 6.0" mask, so that the region containing the extra galaxy is included in the fit. If we +simply shrank the mask to exclude the extra galaxy, we would also throw away pixels containing the main galaxy's +outer emission, and the mask's hard edge could still cut through the extra galaxy's light. +""" +mask = ag.Mask2D.circular( + shape_native=dataset.shape_native, + pixel_scales=dataset.pixel_scales, + radius=6.0, +) + +dataset = dataset.apply_mask(mask=mask) + +""" +Plotting the masked dataset confirms the extra galaxy's emission is inside the mask, and will therefore impact the +model-fit unless we do something about it. +""" +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Approach 1 Noise Scaling__ + +Our first approach removes the extra galaxy's light from the fit. + +The most obvious way to do this would be to mask the extra galaxy's pixels, removing them from the fit entirely. +However, removing pixels changes the fit in subtle ways: their coordinates are no longer used when evaluating the +galaxy's light, and for certain models (e.g. the pixelized reconstructions of chapter 3) removing interior pixels +creates discontinuities in the pixelization that introduce unexpected systematics. + +Instead, we use **noise scaling**: the pixels stay in the fit, but their data values are set to zero and their +noise-map values are increased to very large values. A pixel with enormous noise contributes negligibly to the +likelihood, so the extra galaxy's light cannot influence the model, while the pixels themselves remain part of the +fit's geometry. + +To do this we need a mask of the extra galaxy's region. For real data, you would create this yourself by +inspecting the image (the `autogalaxy_workspace`'s `data_preparation` package includes a GUI for drawing it); for +this simulated dataset the simulator script has already output a `mask_extra_galaxies.fits` circle covering the +extra galaxy. + +We reload the dataset first, because noise scaling must be applied before the circular mask. +""" +dataset = ag.Imaging.from_fits( + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + psf_path=dataset_path / "psf.fits", + pixel_scales=0.1, +) + +mask_extra_galaxies = ag.Mask2D.from_fits( + file_path=dataset_path / "mask_extra_galaxies.fits", + pixel_scales=0.1, + invert=True, # Note that we invert the mask here as `True` means a pixel is scaled. +) + +dataset = dataset.apply_noise_scaling(mask=mask_extra_galaxies) + +mask = ag.Mask2D.circular( + shape_native=dataset.shape_native, + pixel_scales=dataset.pixel_scales, + radius=6.0, +) + +dataset = dataset.apply_mask(mask=mask) + +""" +Plotting the dataset shows the extra galaxy's emission has vanished: its data values are zero and the +signal-to-noise of its pixels is effectively zero, so the fit will simply ignore that region of the image. +""" +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Noise Scaling Fit__ + +We now fit this noise-scaled dataset with a model that does **not** include the extra galaxy. The model is the +same one we used in chapter 2's linear profiles tutorial: a linear `Sersic` bulge. +""" +bulge = af.Model(ag.lp_linear.Sersic) + +galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge) + +model = af.Collection(galaxies=af.Collection(galaxy=galaxy)) + +search = af.Nautilus( + path_prefix=Path("howtogalaxy", "chapter_4"), + name="tutorial_1_extra_galaxies_noise_scaling", + unique_tag=dataset_name, + n_live=100, + n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. +) + +analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True) + +print( + "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_4/tutorial_1_extra_galaxies_noise_scaling" + " folder for live output of the results, images and model." + " This Jupyter notebook cell with progress once search has completed - this could take some time!" +) + +result_noise_scaling = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +Plotting the maximum log likelihood fit shows the galaxy is fitted well, with the noise-scaled region contributing +nothing to the residuals. + +The extra galaxy's light has been dealt with, and no complexity was added to the model: the fit had exactly the +same free parameters as a fit to an isolated galaxy. This is the great appeal of noise scaling. +""" +aplt.subplot_fit_imaging(fit=result_noise_scaling.max_log_likelihood_fit) + +""" +__Approach 2 Extra Galaxies Model__ + +Our second approach includes the extra galaxy in the model, fitting its light explicitly so we no longer need to +remove it from the data. Its emission is subtracted by the model itself, including any faint light that spills +towards the main galaxy which a noise-scaling mask cannot cleanly separate. + +We reload the dataset and apply the 6.0" circular mask, but this time we do **not** apply noise scaling, because +the extra galaxy's emission is now something the model itself will fit. +""" +dataset = ag.Imaging.from_fits( + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + psf_path=dataset_path / "psf.fits", + pixel_scales=0.1, +) + +mask = ag.Mask2D.circular( + shape_native=dataset.shape_native, + pixel_scales=dataset.pixel_scales, + radius=6.0, +) + +dataset = dataset.apply_mask(mask=mask) + +""" +__Extra Galaxy Centres__ + +To include the extra galaxy in the model, we input the centre of its light, as observed in the image. + +In principle, we could add the extra galaxy to the model with a completely free centre, and let the non-linear +search figure out where it is. In practice this is a bad idea: the extra parameters make parameter space more +complex, and fits commonly go wrong in characteristic ways. For example, the extra galaxy's light profile may +wander off and try to fit part of the main galaxy's asymmetric emission instead of the interloper, leaving the +interloper unfitted and the main galaxy's light model biased. + +Fixing each extra galaxy's light centre to its observed light centre removes these failure modes and keeps the +model as simple as possible. The observed centre is an excellent estimate of the true centre, because a galaxy's +brightest pixel closely traces the peak of its light distribution. + +For real data you would measure these centres from the image (the `autogalaxy_workspace`'s `data_preparation` +package shows how, including a GUI for marking them); for this simulated dataset the simulator has output them to +a .json file, which we load below. +""" +extra_galaxies_centres = ag.Grid2DIrregular( + ag.from_json(file_path=dataset_path / "extra_galaxies_centres.json") +) + +print(extra_galaxies_centres) + +""" +__Extra Galaxies Model Composition__ + +We compose the main galaxy model exactly as before. +""" +bulge = af.Model(ag.lp_linear.Sersic) + +galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge) + +""" +We now compose the model of the extra galaxy, using the dedicated `extra_galaxies` modeling API. + +For each extra galaxy centre (there is only one here, but the loop below scales to any number) we create a `Galaxy` +model with: + + - A linear `SersicSph` light profile, with its `centre` fixed to the observed centre [2 free + parameters: `effective_radius` and `sersic_index`, as the `intensity` is solved for by the linear inversion]. + +The extra galaxies are grouped into their own `af.Collection`, which is passed to the overall model via its +`extra_galaxies` input, alongside the `galaxies` collection containing the main galaxy. This is the same API used +throughout the `autogalaxy_workspace` for extra galaxies, and it is how **PyAutoGalaxy** knows these galaxies are +nuisance components of the fit rather than the subject of the analysis. +""" +# Extra Galaxies: + +extra_galaxies_list = [] + +for extra_galaxy_centre in extra_galaxies_centres: + + extra_galaxy = af.Model( + ag.Galaxy, + redshift=0.5, + bulge=ag.lp_linear.SersicSph, + ) + + extra_galaxy.bulge.centre = extra_galaxy_centre + + extra_galaxies_list.append(extra_galaxy) + +extra_galaxies = af.Collection(extra_galaxies_list) + +# Overall Model: + +model = af.Collection( + galaxies=af.Collection(galaxy=galaxy), extra_galaxies=extra_galaxies +) + +""" +The `info` attribute confirms the model includes the extra galaxy, with its fixed centre and its +free `effective_radius` and `sersic_index` parameters. +""" +print(model.info) + +""" +__Extra Galaxies Fit__ + +We fit this model with the same search set up as before. The model has only two more free parameters than the +noise-scaling fit, thanks to the fixed centre and linear light profile, so the fit remains fast. +""" +search = af.Nautilus( + path_prefix=Path("howtogalaxy", "chapter_4"), + name="tutorial_1_extra_galaxies_model", + unique_tag=dataset_name, + n_live=100, + n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. +) + +analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True) + +print( + "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_4/tutorial_1_extra_galaxies_model" + " folder for live output of the results, images and model." + " This Jupyter notebook cell with progress once search has completed - this could take some time!" +) + +result_extra_galaxies = search.fit(model=model, analysis=analysis) + +print("The search has finished run - you may now continue the notebook.") + +""" +Plotting the maximum log likelihood fit shows the extra galaxy's emission is now fitted and subtracted by its own +light profile, leaving the main galaxy's light cleanly fitted by the `Sersic` bulge. +""" +aplt.subplot_fit_imaging(fit=result_extra_galaxies.max_log_likelihood_fit) + +""" +The result's `info` shows the inferred `effective_radius` and `sersic_index` of the extra galaxy, alongside the +main galaxy's parameters, which are no longer at risk of being biased by the interloper's light. +""" +print(result_extra_galaxies.info) + +""" +__Which Approach When__ + +We have seen the two extremes of dealing with an extra galaxy. Choosing between them comes back to the question at +the start of this tutorial — does its light overlap the region we fit? — which in practice is settled by two +properties of the interloper: + +- **Distance from the main galaxy's emission**: An extra galaxy whose light is well separated from the main galaxy + (as in this tutorial) can be cleanly noise-scaled away. If its light blends into the main galaxy's emission, + noise scaling would also delete main-galaxy emission we need to fit — biasing exactly the outer isophotes that + constrain the `effective_radius` and `sersic_index` — and the light must be modeled instead. + +- **Brightness**: A faint interloper whose emission barely rises above the noise can often be ignored entirely, or + noise-scaled with no consequence. A bright one must be removed or modeled, or it will bias the fit. The brighter + the interloper, the further its light extends above the noise, and the more likely it blends with the main + galaxy. + +When in doubt, fit both approaches and compare the inferred models of the main galaxy: if its parameters shift +appreciably, the extra galaxy's light is leaking into the fit and explicit modeling is the safer choice. + +A middle ground also exists and is fully supported by the API: noise-scale the interloper's bright central regions +but also include it in the model, so its faint outer light under the main galaxy is still subtracted. The +`extra_galaxies` collection simply contains whatever galaxies you give it. + +Finally, the `SersicSph` profile we used for the extra galaxy can be swapped for any light model from the earlier +chapters. In particular, a Multi Gaussian Expansion basis (chapter 2) captures irregular interloper morphologies +with no extra non-linear parameters, and is the recommended choice once the number of extra galaxies grows beyond +a handful — see `autogalaxy_workspace/*/imaging/features/extra_galaxies/modeling.py` for this extension. + +__Wrap Up__ + +In this tutorial, we took the first step up in scale from the single galaxy of chapters 1 to 3, and learnt: + +1. Real galaxies live in crowded fields, and interloping extra galaxies are the first complication real data + throws at us. + +2. Whether an extra galaxy matters hinges on one question: does its **light** overlap the region of the image we + fit? If it does, an unmodeled interloper biases the main galaxy's inferred structure. + +3. Noise scaling removes an extra galaxy's light from the fit without adding model complexity, by zeroing its data + and inflating its noise, while keeping the pixels in the fit's geometry. + +4. The `extra_galaxies` modeling API includes extra galaxies in the model with their own light profiles, with + their centres fixed to the observed light centres to keep parameter space simple and well behaved. + +5. Which approach is appropriate depends on the interloper's distance from the main galaxy's emission and its + brightness, and the two approaches can be combined. + +Throughout, the extra galaxy was a nuisance: something to remove or account for, so that our analysis of the main +galaxy remained accurate. In the next tutorial we meet systems where that framing breaks down entirely, because a +second galaxy is not a nuisance but a co-equal subject of the analysis, blended with the first and of comparable +brightness. There, no single galaxy is "the" galaxy, and the model must treat them all on an equal footing. +""" diff --git a/scripts/simulators/extra_galaxy.py b/scripts/simulators/extra_galaxy.py new file mode 100644 index 0000000..e63704f --- /dev/null +++ b/scripts/simulators/extra_galaxy.py @@ -0,0 +1,238 @@ +""" +Simulator: Extra Galaxy +======================= + +This script simulates `Imaging` of a galaxy using light profiles where: + + - The main galaxy's bulge is an `Sersic`. + - There is one extra galaxy nearby, whose light is an `ExponentialSph`, located a few arc-seconds + from the main galaxy. + +This dataset is used in chapter 4 tutorial 1, which shows the two approaches to dealing with an +extra galaxy whose light contaminates the fit: noise scaling its emission out of the data, or +including it in the model via the extra galaxies API. + +__Contents__ + +- **Dataset Paths:** Set the output path for the simulated dataset. +- **Grid:** Create a 2D grid with adaptive over-sampling for simulation. +- **Galaxies:** Define the main galaxy and the nearby extra galaxy used for simulation. +- **Output:** Save the simulated dataset to FITS files. +- **Visualize:** Output subplot and image PNGs of the simulated dataset. +- **Mask Extra Galaxies:** Build and save `mask_extra_galaxies.fits` covering the extra galaxy's light. +- **Plane Output:** Save the Galaxies object as a JSON file. +- **Extra Galaxies Centres:** Save the extra galaxy's (y,x) centre as a JSON file. + +__Start Here Notebook__ + +If any code in this script is unclear, refer to the `simulators/simple.ipynb` notebook. +""" + +# from autogalaxy import setup_notebook; setup_notebook() + +from pathlib import Path + +import numpy as np + +import autogalaxy as ag +import autogalaxy.plot as aplt + +""" +__Dataset Paths__ + +The `dataset_type` describes the type of data being simulated and `dataset_name` gives it a descriptive name. +""" +dataset_type = "imaging" +dataset_name = "extra_galaxy" + +dataset_path = Path("dataset", dataset_type, dataset_name) + +""" +__Grid__ + +Simulate the image using a (y,x) grid with the adaptive over sampling scheme. + +The grid is larger than the 100 x 100 pixels used by other simulators in this folder, so that the extra galaxy, +which is offset from the main galaxy centre of (0.0", 0.0"), is comfortably contained in the image. + +The adaptive over sampling scheme is centred on both the main galaxy and the extra galaxy, ensuring the light of +both is over sampled accurately. +""" +grid = ag.Grid2D.uniform( + shape_native=(150, 150), + pixel_scales=0.1, +) + +extra_galaxy_centre = (1.0, 3.5) + +over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from( + grid=grid, + sub_size_list=[32, 8, 2], + radial_list=[0.3, 0.6], + centre_list=[(0.0, 0.0), extra_galaxy_centre], +) + +grid = grid.apply_over_sampling(over_sample_size=over_sample_size) + +""" +Simulate a simple Gaussian PSF for the image. +""" +psf = ag.Convolver.from_gaussian( + shape_native=(11, 11), sigma=0.1, pixel_scales=grid.pixel_scales +) + +""" +Create the simulator for the imaging data, which defines the exposure time, background sky, noise levels and psf. +""" +simulator = ag.SimulatorImaging( + exposure_time=300.0, + psf=psf, + background_sky_level=0.1, + add_poisson_noise_to_data=True, +) + +""" +__Galaxies__ + +Setup the main galaxy with a bulge (elliptical Sersic) for this simulation. +""" +galaxy = ag.Galaxy( + redshift=0.5, + bulge=ag.lp.Sersic( + centre=(0.0, 0.0), + ell_comps=ag.convert.ell_comps_from(axis_ratio=0.9, angle=45.0), + intensity=1.0, + effective_radius=0.8, + sersic_index=4.0, + ), +) + +""" +Setup the extra galaxy, a smaller galaxy whose light is a spherical Exponential profile, offset a few arc-seconds +from the main galaxy. + +Note that its redshift is the same as the main galaxy, which is not necessarily the case in real observations. +If it is at a different redshift, the tools for masking or modeling extra galaxies are equipped to handle this. +""" +extra_galaxy = ag.Galaxy( + redshift=0.5, + light=ag.lp.ExponentialSph( + centre=extra_galaxy_centre, intensity=2.0, effective_radius=0.5 + ), +) + +""" +Use these galaxies to generate the image for the simulated `Imaging` dataset. +""" +galaxies = ag.Galaxies(galaxies=[galaxy, extra_galaxy]) +aplt.plot_array(array=galaxies.image_2d_from(grid=grid), title="Image") + +""" +Pass the simulator galaxies, which creates the image which is simulated as an imaging dataset. +""" +dataset = simulator.via_galaxies_from(galaxies=galaxies, grid=grid) + +""" +Plot the simulated `Imaging` dataset before outputting it to fits. +""" +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Output__ + +Output the simulated dataset to the dataset path as .fits files. +""" +aplt.fits_imaging( + dataset=dataset, + data_path=dataset_path / "data.fits", + psf_path=dataset_path / "psf.fits", + noise_map_path=dataset_path / "noise_map.fits", + overwrite=True, +) + +""" +__Visualize__ + +Output a subplot of the simulated dataset, the image and the galaxies quantities to the dataset path as .png files. +""" +aplt.subplot_imaging_dataset( + dataset=dataset, output_path=dataset_path, output_format="png" +) +aplt.plot_array( + array=dataset.data, title="Data", output_path=dataset_path, output_format="png" +) + +aplt.subplot_galaxies( + galaxies=galaxies, grid=grid, output_path=dataset_path, output_format="png" +) + +""" +__Mask Extra Galaxies__ + +Build and output a `mask_extra_galaxies.fits` covering the extra galaxy's emission, so that the chapter 4 +tutorial which uses this dataset can load the mask directly without a separate data-preparation step. For real +data, a user would create this mask themselves by inspecting the image (the `autogalaxy_workspace`'s +`data_preparation` package includes a GUI for drawing it). + +The circle is sized to ~3x the extra galaxy's `effective_radius`, which comfortably covers the light extent of +the `ExponentialSph` profile used above. The geometry is derived from the same centre + radius defined for the +extra galaxy in this script, so it stays in sync with any future tweak to those values. + +`Mask2D.circular` honours the `PYAUTO_SMALL_DATASETS=1` env var, so the mask automatically shrinks alongside the +small-dataset image and never raises an out-of-bounds error. +""" +extra_galaxies_mask = np.zeros(dataset.shape_native, dtype=bool) + +for centre, radius in [ + (extra_galaxy_centre, 3.0 * 0.5), +]: + circle = ag.Mask2D.circular( + shape_native=dataset.shape_native, + pixel_scales=dataset.pixel_scales, + centre=centre, + radius=radius, + invert=True, # True inside the circle (i.e. masked region) + ) + extra_galaxies_mask = np.logical_or(extra_galaxies_mask, circle.native) + +mask_extra_galaxies = ag.Mask2D( + mask=extra_galaxies_mask, + pixel_scales=dataset.pixel_scales, +) + +aplt.fits_array( + array=mask_extra_galaxies, + file_path=dataset_path / "mask_extra_galaxies.fits", + overwrite=True, +) + +""" +__Plane Output__ + +Save the `Galaxies` in the dataset folder as a .json file, ensuring the true light profiles and galaxies +are safely stored and available to check how the dataset was simulated in the future. + +This can be loaded via the method `galaxies = ag.from_json()`. +""" +ag.output_to_json( + obj=galaxies, + file_path=Path(dataset_path, "galaxies.json"), +) + +""" +__Extra Galaxies Centres__ + +Save the (y,x) centre of the extra galaxy as a `Grid2DIrregular` JSON file. The chapter 4 tutorial loads this +file to fix the extra galaxy's light profile centre when composing the model. For real data, a user would mark +these centres themselves on the image (the `autogalaxy_workspace`'s `data_preparation` package shows how). +""" +extra_galaxies_centres = ag.Grid2DIrregular(values=[extra_galaxy_centre]) + +ag.output_to_json( + obj=extra_galaxies_centres, + file_path=Path(dataset_path, "extra_galaxies_centres.json"), +) + +""" +The dataset can be viewed in the folder `dataset/imaging/extra_galaxy`. +""" From 8e41c0b50b4fc68c073cd6932cbb0451bb5413f9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:38:20 +0000 Subject: [PATCH 08/15] docs: rehome search chaining and prior passing as chapter 2 tutorials 9-10 Retitled and reframed as the closing arc of the modeling chapter, motivated by tutorial 8's run-time discussion; output paths and search names move to chapter_2 with the tutorial 9 -> 10 result reload verified by identifier equality; references to the deleted x2-galaxies tutorial reroute to chapter 4 and the workspace chaining guide; lensing leaks in the copied prose fixed; tutorial 9's first model now actually applies the simplifications its prose promises (prior_count 4, was 6); stale chapter references in tutorials 3-6 and the chapter summary updated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- .../tutorial_10_prior_passing.py | 44 ++++++------ .../chapter_2_modeling/tutorial_11_summary | 4 +- .../tutorial_3_realism_and_complexity.py | 4 +- .../tutorial_4_dealing_with_failure.py | 2 +- .../tutorial_5_linear_profiles.py | 4 +- .../chapter_2_modeling/tutorial_6_masking.py | 4 +- .../tutorial_9_search_chaining.py | 71 +++++++++++-------- 7 files changed, 76 insertions(+), 57 deletions(-) diff --git a/scripts/chapter_2_modeling/tutorial_10_prior_passing.py b/scripts/chapter_2_modeling/tutorial_10_prior_passing.py index cf24a1b..1cf662f 100644 --- a/scripts/chapter_2_modeling/tutorial_10_prior_passing.py +++ b/scripts/chapter_2_modeling/tutorial_10_prior_passing.py @@ -1,6 +1,6 @@ """ -Tutorial 2: Prior Passing -========================= +Tutorial 10: Prior Passing +========================== In the previous tutorial, we used non-linear search chaining to break the model-fitting procedure down into two non-linear searches. This used an initial search to fit a simple model, whose results were used to tune and @@ -13,6 +13,7 @@ __Contents__ - **Initial Setup:** Load the dataset and apply a mask. +- **Dataset Auto-Simulation:** Simulate the dataset via its simulator script if it is not on your hard-disk. - **Model:** Compose the model for the first search. - **Search:** Run the first search. - **Result (Search 1):** Inspect the result of the first search. @@ -34,7 +35,7 @@ """ __Initial Setup__ -we'll use the same galaxying data as the previous tutorial, where: +we'll use the same galaxy data as the previous tutorial, where: - The galaxy's bulge is an `Sersic`. - The galaxy's disk is an `Exponential`. @@ -120,11 +121,11 @@ analysis_1 = ag.AnalysisImaging(dataset=dataset, use_jax=True) search_1 = af.Nautilus( - path_prefix=Path("howtogalaxy", "chapter_3"), - name="tutorial_1_search_chaining_1", + path_prefix=Path("howtogalaxy", "chapter_2"), + name="tutorial_9_search_chaining_1", unique_tag=dataset_name, n_live=100, - n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. + n_batch=50, # GPU batching and VRAM use explained in tutorial 2 of this chapter. ) result_1 = search_1.fit(model=model_1, analysis=analysis_1) @@ -197,15 +198,15 @@ analysis_2 = ag.AnalysisImaging(dataset=dataset, use_jax=True) search_2 = af.Nautilus( - path_prefix=Path("howtogalaxy", "chapter_3"), - name="tutorial_2_search_chaining_2", + path_prefix=Path("howtogalaxy", "chapter_2"), + name="tutorial_10_prior_passing_2", unique_tag=dataset_name, n_live=100, - n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. + n_batch=50, # GPU batching and VRAM use explained in tutorial 2 of this chapter. ) print( - "The non-linear search has begun running - checkout the workspace/output/5_chaining_searches" + "The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2" " folder for live output of the results, images and model." " This Jupyter notebook cell with progress once search has completed - this could take some time!" ) @@ -230,18 +231,19 @@ """ __Wrap Up__ -We will expand on the prior passing API in the following tutorials. The main thing to note is that we can pass -entire profiles or galaxies using prior passing, if their model does not change (which for the bulge and disk, was -not true). The API to pass a whole profile or galaxy is as follows: - +This tutorial has covered the core of the prior passing API; the chaining example in the `autogalaxy_workspace` +(`autogalaxy_workspace/scripts/guides/modeling/chaining.py`) expands on it further. The main thing to note is that +we can pass entire profiles or galaxies using prior passing, if their model does not change (which for the bulge +and disk above, was not true). The API to pass a whole profile or galaxy is as follows: + bulge = result_1.model.galaxies.galaxy.bulge galaxy = result_1.model.galaxies.galaxy - + We can also pass priors using an `instance` instead of a `model`. When an `instance` is used, the maximum likelihood parameter values are passed as fixed values that are therefore not fitted for by the non-linear search (reducing its -dimensionality). We will use this in the next tutorial to fit data with two galaxies, where fit one galaxy, fix it to -the best-fit model in a second search that fits the second galaxy, and then go on to fit both simultaneously in the -final search. +dimensionality). Chained fits often use this to fix one galaxy's light to the best-fit model of an early search +before fitting a second galaxy alongside it — chapter 4 of **HowToGalaxy**, which scales up to modeling multiple +galaxies, and the workspace's chaining example show this in action. Lets now think about how priors are passed. Checkout the `model.info` file of the second search of this tutorial. The parameters do not use the default priors we saw in search 1 (which are typically broad UniformPriors). Instead, @@ -251,13 +253,13 @@ - The sigma values are specified in the `width_modifier` field of the profile's entry in the `priors.yaml' config file (we will discuss why this is used in a moment). -Like the manual `GaussianPrior`'s that were used in tutorial 1, the prior passing API sets up the prior on each +Like the manual `GaussianPrior`'s that were used in the previous tutorial, the prior passing API sets up the prior on each parameter with a `GaussianPrior` centred on the high likelihood regions of parameter space! __Detailed Explanation Of Prior Passing__ To end, I provide a detailed overview of how prior passing works and illustrate tools that can be used to customize -its behaviour. It is up to you whether you want read this, or go ahead to the next tutorial! +its behaviour. It is up to you whether you want read this, or move on to the summary that concludes the chapter! Lets say I chain two parameters as follows: @@ -265,7 +267,7 @@ By invoking the `model` attribute, the prior is passed following 3 rules: - 1) The new parameter, in this case the einstein radius, uses a `GaussianPrior`.This is ideal, as the 1D pdf results + 1) The new parameter, in this case the effective radius, uses a `GaussianPrior`. This is ideal, as the 1D pdf results we compute at the end of a search are easily summarised as a Gaussian. 2) The mean of the `GaussianPrior` is the median PDF value of the parameter estimated in search 1. diff --git a/scripts/chapter_2_modeling/tutorial_11_summary b/scripts/chapter_2_modeling/tutorial_11_summary index e78d06c..f95c902 100755 --- a/scripts/chapter_2_modeling/tutorial_11_summary +++ b/scripts/chapter_2_modeling/tutorial_11_summary @@ -8,10 +8,12 @@ In this chapter, you learnt: 4) How one must carefully balance complexity and realism when fitting a model to a data-set. 5) The importance of factoring in run-speed when modeling a galaxy, and tricks to speed up the analysis. 6) About masking data in the analysis. +7) How to chain searches together, passing priors from one search to the next so that earlier fits of simpler + models guide later fits of more complex ones. At this point, you are ready to begin modeling galaxies. The 'modeling' folders in the autogalaxy_workspace contains a number of scripts that can be easily adopted to model galaxies using a variety of different approaches, models and non-linear searches. If you have your own galaxy data, I'd recommend you adapt these scripts to your data. If you don't have your own data, checkout the 'simulators' folder to simulate your own dataset! -Alternatively, you may wish to continue on to chapter 3 on search chaining. \ No newline at end of file +Alternatively, you may wish to continue on to chapter 3 on pixelized galaxy reconstructions. \ No newline at end of file diff --git a/scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py b/scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py index 6e33ba2..8c75590 100644 --- a/scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py +++ b/scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py @@ -225,8 +225,8 @@ and infers a local maxima, especially if we don't set it up with enough live points! At its core, modeling is all about learning how to get a non-linear search to find the global maxima region of -parameter space, even when the model is complex. This will be the main theme throughout the rest of this chapter -and is the main subject of chapter 3. +parameter space, even when the model is complex. This will be the main theme throughout the rest of this chapter, +culminating in the 'search chaining' technique that closes it. In the next exercise, we'll learn how to deal with failure and begin thinking about how we can ensure our non-linear search finds the global-maximum log likelihood solution. First, think about the following: diff --git a/scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py b/scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py index 2c9809e..db8cd32 100644 --- a/scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py +++ b/scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py @@ -421,7 +421,7 @@ So, we can now fit galaxies. And when it fails, we know how to get it to work. -In chapter 3 of **HowToGalaxy**, we will introduce a technique called 'non-linear search chaining', which performs a +At the end of this chapter, we will introduce a technique called 'non-linear search chaining', which performs a model fit by chaining together multiple searches back-to-back . This allows us to combine the 3 different approaches discussed and exploit the advantages of each, whilst not being hindered by their disadvantages. diff --git a/scripts/chapter_2_modeling/tutorial_5_linear_profiles.py b/scripts/chapter_2_modeling/tutorial_5_linear_profiles.py index cac2d39..545311f 100644 --- a/scripts/chapter_2_modeling/tutorial_5_linear_profiles.py +++ b/scripts/chapter_2_modeling/tutorial_5_linear_profiles.py @@ -522,7 +522,7 @@ lens modeling problems. If you have a desire to fit sources with even more complex morphologies we recommend you look at how to reconstruct -sources using pixelizations in the `modeling/features` section or chapter 4 of **HowToGalaxy**. +sources using pixelizations in the `modeling/features` section or chapter 3 of **HowToGalaxy**. __Wrap Up__ @@ -535,7 +535,7 @@ galaxies which may have multiple components with multiple centres (e.g. many star forming knots) which such a basis cannot catprue. -In chapter 4 of **HowToGalaxy** we introduce non-parametric pixelizations, which reconstruct the data in way +In chapter 3 of **HowToGalaxy** we introduce non-parametric pixelizations, which reconstruct the data in way that does not make assumptions like a centre and can thus reconstruct even more complex, asymmetric and irregular galaxy morphologies. """ diff --git a/scripts/chapter_2_modeling/tutorial_6_masking.py b/scripts/chapter_2_modeling/tutorial_6_masking.py index 209647c..988a34b 100644 --- a/scripts/chapter_2_modeling/tutorial_6_masking.py +++ b/scripts/chapter_2_modeling/tutorial_6_masking.py @@ -135,8 +135,8 @@ use as large of a mask as possible (whilst still achieving reasonable run-times). Aggressive masking will make **PyAutoGalaxy** run very fast, but could lead you to infer an incorrect model! -In chapter 3, where we introduce 'non-linear search chaining' we will see how we can use tighter masks in earlier -searches to achieve faster run times. +At the end of this chapter, where we introduce 'non-linear search chaining', we will see how chaining makes it +possible to use tighter masks in earlier searches to achieve faster run times. If your data includes the light of additional galaxies nearby you may much have no choice but to use a smaller circular mask, because it is important these objects do not interfere with the fit. diff --git a/scripts/chapter_2_modeling/tutorial_9_search_chaining.py b/scripts/chapter_2_modeling/tutorial_9_search_chaining.py index e70881d..aafb14c 100644 --- a/scripts/chapter_2_modeling/tutorial_9_search_chaining.py +++ b/scripts/chapter_2_modeling/tutorial_9_search_chaining.py @@ -1,14 +1,18 @@ """ -Tutorial 1: Search Chaining +Tutorial 9: Search Chaining =========================== -In chapter 2, we learnt how to perform modeling using a non-linear search. In all of the tutorials, we fitted the -data using just one non-linear search. In this chapter, we introduce a technique called 'non-linear search chaining', -which fits a model using a sequence of non-linear searches. The initial searches fit simpler models whose parameter -spaces can be more accurately and efficiently sampled. The results of this search are then passed to later searches -which fit models of gradually increasing complexity. +Throughout this chapter, we have fitted the data using just one non-linear search. The previous tutorial discussed +the run-time cost of doing this: with a single search, the dimensionality of the model, the priors on its parameters +and the search settings must all be juggled at once, leaving us little flexibility to trade them off against one +another. -Lets think back to tutorial 4 of chapter 2. We learnt there were three approaches one could take fitting a model +In this closing part of the chapter, we introduce a technique called 'non-linear search chaining', which fits a +model using a sequence of non-linear searches. The initial searches fit simpler models whose parameter spaces can +be more accurately and efficiently sampled. The results of this search are then passed to later searches which fit +models of gradually increasing complexity. + +Lets think back to tutorial 4 of this chapter. We learnt there were three approaches one could take fitting a model accurately if we found that a model fit failed. These were: 1) Tuning our priors to the galaxy we're fitting. @@ -21,23 +25,24 @@ In this tutorial, we are going to show how search chaining combines these 3 approaches such that we can fit complex and realistic models in a way that that can be generalized to many different galaxies. To do this, -we'll run 2 searches, and chain the model inferred in the first search to the priors of the second search`s lens +we'll run 2 searches, and chain the model inferred in the first search to the priors of the second search`s model. -Our first search will make the same bulge-disk alignment assumption we made in the previous tutorial. We saw that this -gives a reasonable model. However, we'll make a couple of extra simplifying assumptions, to really try and bring -our model complexity down and get the non-linear search running fast. +Our first search will make the same bulge-disk alignment assumption we made in tutorial 4 of this chapter. We saw +that this gives a reasonable model. However, we'll make a couple of extra simplifying assumptions, to really try and +bring our model complexity down and get the non-linear search running fast. The model we infer above will therefore be a lot less realistic. But it does not matter, because in the second search we are going to relax these assumptions and fit the more realistic model. The beauty is that, by running the first search, we can use its results to tune the priors of our second search. For example: 1) The first search should give us a pretty good idea of the galaxy's bulge and disk profiles, for example its - centre, intensity, effective radius. + centre, ellipticity and effective radius. __Contents__ - **Initial Setup:** Load the dataset and apply a mask. +- **Dataset Auto-Simulation:** Simulate the dataset via its simulator script if it is not on your hard-disk. - **Model:** Compose a simplified model for the first search with aligned bulge-disk assumptions. - **Search + Analysis:** Run the first search with the simplified model. - **Result:** Inspect the result of the first search. @@ -57,7 +62,7 @@ """ __Initial Setup__ -we'll use the same galaxy data as tutorial 4 of chapter 2, where: +we'll use the same galaxy data as tutorial 4 of this chapter, where: - The galaxy's bulge is an `Sersic`. - The galaxy's disk is an `Exponential`. @@ -132,6 +137,18 @@ disk.centre_0 = 0.0 disk.centre_1 = 0.0 +""" +Lets also use the same approach as tutorial 4 of this chapter and align the ellipticity of the bulge and disk, +removing another two parameters from the model. +""" +disk.ell_comps = bulge.ell_comps + +""" +We also know that the bulges of many galaxies are well described by a Sersic index of around 4 (a de Vaucouleurs +profile). Lets fix it to 4 this time. +""" +bulge.sersic_index = 4.0 + galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge, disk=disk) model_1 = af.Collection(galaxies=af.Collection(galaxy=galaxy)) @@ -147,21 +164,21 @@ Now lets create the search and analysis. """ search_1 = af.Nautilus( - path_prefix=Path("howtogalaxy", "chapter_3"), - name="tutorial_1_search_chaining_1", + path_prefix=Path("howtogalaxy", "chapter_2"), + name="tutorial_9_search_chaining_1", unique_tag=dataset_name, n_live=100, - n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. + n_batch=50, # GPU batching and VRAM use explained in tutorial 2 of this chapter. ) analysis_1 = ag.AnalysisImaging(dataset=dataset, use_jax=True) """ -Lets run the search, noting that our liberal approach to reducing the model complexity has reduced it to just -6 parameters. +Lets run the search, noting that our liberal approach to reducing the model complexity has reduced it to just +4 parameters. """ print( - "The non-linear search has begun running - checkout the workspace/output/5_chaining_searches" + "The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2" " folder for live output of the results, images and model." " This Jupyter notebook cell with progress once search has completed - this could take some time!" ) @@ -202,7 +219,7 @@ parameter was fixed in the previous search, its prior is based around the previous value. Don't worry about the sigma values for now, I've chosen values that I know will ensure reasonable sampling, but we'll cover this later. -__LENS BULGE PRIORS:__ +__BULGE PRIORS:__ """ bulge.centre.centre_0 = af.TruncatedGaussianPrior( mean=0.0, sigma=0.1, lower_limit=-np.inf, upper_limit=np.inf @@ -224,7 +241,7 @@ ) """ -__LENS DISK PRIORS:__ +__DISK PRIORS:__ """ disk.centre.centre_0 = af.TruncatedGaussianPrior( mean=0.0, sigma=0.1, lower_limit=-np.inf, upper_limit=np.inf @@ -261,20 +278,18 @@ Lets setup and run the search. As expected, it gives us the correct model. However, it does so significantly faster than we are used to! """ -batch_size = 50 # Explained chapter 2 tutorial 2 - search_2 = af.Nautilus( - path_prefix=Path("howtogalaxy", "chapter_3"), - name="tutorial_1_search_chaining_2", + path_prefix=Path("howtogalaxy", "chapter_2"), + name="tutorial_9_search_chaining_2", unique_tag=dataset_name, n_live=100, - n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. + n_batch=50, # GPU batching and VRAM use explained in tutorial 2 of this chapter. ) analysis_2 = ag.AnalysisImaging(dataset=dataset, use_jax=True) print( - "The non-linear search has begun running - checkout the workspace/output/5_chaining_searches" + "The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2" " folder for live output of the results, images and model." " This Jupyter notebook cell with progress once search has completed - this could take some time!" ) @@ -303,7 +318,7 @@ pretty much any galaxy and therefore get ourselves a decent model with which to tune search 2`s priors. You are probably thinking though that there is one huge, giant, glaring flaw in all of this that I've not mentioned. -Search 2 can`t be generalized to another lens, because its priors are tuned to the image we fitted. If we had a lot +Search 2 can`t be generalized to another galaxy, because its priors are tuned to the image we fitted. If we had a lot of galaxies, we`d have to write a new search for every single one. This isn't ideal, is it? Fortunately, we can pass priors in **PyAutoGalaxy** without specifying the specific values. The API for this technique, From c2d5556535d8166871aa42cd0b69ed03248682f1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:46:24 +0000 Subject: [PATCH 09/15] =?UTF-8?q?docs:=20chapter=203=20pixelizations=20?= =?UTF-8?q?=E2=80=94=20renumber,=20clean=20up,=20add=20the=20Bayesian=20fo?= =?UTF-8?q?rmalism=20lecture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New tutorial_5_bayesian_formalism collects the pixelized-reconstruction linear algebra for the galaxy context (mesh laid over the galaxy's own image — no ray tracing or borders, stated explicitly): mapping and blurred mapping matrices, data vector, curvature and regularization matrices, the solve and the five evidence terms, computed live. The other tutorials get chapter-3 framing, algebraic deferrals to tutorial 5, corrected model descriptions in the renamed tutorial 6 (model fit, path_prefix now chapter_3), a false 'star forming clumps' dataset claim removed, and README/introduction updates. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- scripts/chapter_3_pixelizations/README.md | 14 +- scripts/chapter_3_pixelizations/introduction | 52 +- .../tutorial_1_pixelizations.py | 12 +- .../tutorial_2_mappers.py | 23 +- .../tutorial_3_inversions.py | 40 +- .../tutorial_4_bayesian_regularization.py | 22 +- .../tutorial_5_bayesian_formalism.py | 618 ++++++++++++++++++ .../tutorial_6_model_fit.py | 72 +- 8 files changed, 743 insertions(+), 110 deletions(-) create mode 100644 scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py diff --git a/scripts/chapter_3_pixelizations/README.md b/scripts/chapter_3_pixelizations/README.md index 97c974d..ed3a902 100644 --- a/scripts/chapter_3_pixelizations/README.md +++ b/scripts/chapter_3_pixelizations/README.md @@ -1,15 +1,17 @@ -In chapter 4, we use **Pixelizations** to reconstruct complex source galaxies on pixelized grids. +In chapter 3, we use **Pixelizations** to reconstruct the light of complex galaxies on pixelized grids. **Colab** links to every tutorial are included. # Files -- [Tutorial 1: Pixelizations](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_1_pixelizations.ipynb) — Creating a pixel-grid in the source-plane. +- [Tutorial 1: Pixelizations](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb) — Creating a pixel-grid to reconstruct a galaxy's light. -- [Tutorial 2: Mappers](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_2_mappers.ipynb) — How a pixelization maps source-pixels to image-pixels. +- [Tutorial 2: Mappers](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb) — How a pixelization maps its pixels to image-pixels. -- [Tutorial 3: Inversions](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_3_inversions.ipynb) — Inverting the mappings to reconstruct the source's light. +- [Tutorial 3: Inversions](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb) — Inverting the mappings to reconstruct the galaxy's light. -- [Tutorial 4: Bayesian Regularization](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_4_bayesian_regularization.ipynb) — Smoothing the source within a Bayesian framework. +- [Tutorial 4: Bayesian Regularization](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb) — Smoothing the reconstruction within a Bayesian framework. -- [Tutorial 5: Model Fit](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_5_model_fit.ipynb) — An example modeling pipeline which uses an inversion. +- [Tutorial 5: Bayesian Formalism](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_5_bayesian_formalism.ipynb) — The linear algebra and Bayesian evidence equations behind pixelized galaxy reconstruction. + +- [Tutorial 6: Model Fit](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb) — An example modeling pipeline which uses an inversion. diff --git a/scripts/chapter_3_pixelizations/introduction b/scripts/chapter_3_pixelizations/introduction index d85c805..fc990be 100755 --- a/scripts/chapter_3_pixelizations/introduction +++ b/scripts/chapter_3_pixelizations/introduction @@ -1,27 +1,25 @@ -So, we've learnt how to build pipelines that model galaxies and customize them to our science case. - -We consider how using search chaining, we can fix complex galaxies made of multiple components or multiple -galaxies at the same time. - -However, we saw a pretty huge barrier when trying to fit very complex galaxies, that it requires an unwieldy number -of non-linear parameters. The non-linear parameter space would become very complex, and even a well crafted pipeline -may not fix the problem. - -In this chapter, we'll learn about pixelizations and inversions. These tools allow us to reconstruct components of a -galaxy using a pixel-grid. This makes no assumption about the galaxy's morphology, breaking the simplifying assumptions -inherent to analytic light profiles (e.g. symmetry). - -Remarkably, pixels grids use just a couple of non-linear parameters, meaning the 30+ non-linear parameters we required -to fit complex galaxy components before are going to be reduced to just 3 or 4! - -By combining these fits with parametric model, we are therefore able to fit certain structures (e.g. a bulge and disk) -whilst using these non-parametric pixelizations to fit other components that may otherwise degrade the fit. - -In particular, you'll learn how we: - -1) Pixelize a galaxy into a set of pixels that define mappings to image pixels. -2) Invert this pixelization to fit the galaxy and thus reconstruct its light. -3) Apply a smoothness prior on the galaxy reconstruction, called 'regularization', to ensure the solution is physicag. -4) Apply this prior in a Bayesian framework to objectively quantify the galaxy reconstruction's log likelihood. -6) Use alternative pixelizations that use Voronoi pixels adapted to the galaxy's mass model. -7) Use these features in PyAutoGalaxy pipelines. \ No newline at end of file +So, we've learnt how to model galaxies with light profiles, and how search chaining (introduced at the end of +chapter 2, tutorials 9-10) lets us fit galaxies made of multiple components in stages. + +However, we saw a pretty huge barrier when trying to fit very complex galaxies: they require an unwieldy number +of non-linear parameters. The non-linear parameter space becomes very complex, and even a well crafted chain of +searches may not fix the problem. + +In this chapter, we'll learn about pixelizations and inversions. These tools allow us to reconstruct components of a +galaxy using a pixel-grid. This makes no assumption about the galaxy's morphology, breaking the simplifying assumptions +inherent to analytic light profiles (e.g. symmetry). + +Remarkably, pixel grids use just a couple of non-linear parameters, meaning the 20+ non-linear parameters we required +to fit complex galaxy structures before are going to be reduced to just 2 or 3! + +By combining these fits with parametric models, we are therefore able to fit certain structures (e.g. a bulge and disk) +whilst using these non-parametric pixelizations to fit other components that may otherwise degrade the fit. + +In particular, you'll learn how we: + +1) Pixelize a galaxy's image into a set of pixels that define mappings to image pixels. +2) Invert this pixelization to fit the galaxy and thus reconstruct its light. +3) Apply a smoothness prior on the galaxy reconstruction, called 'regularization', to ensure the solution is physical. +4) Apply this prior in a Bayesian framework to objectively quantify the galaxy reconstruction's log likelihood. +5) Write down the linear algebra and Bayesian evidence equations that underpin all of the above. +6) Use these features in a PyAutoGalaxy modeling pipeline. diff --git a/scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py b/scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py index 343d43f..becc81e 100644 --- a/scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py +++ b/scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py @@ -1,5 +1,5 @@ """ -Tutorial 1: pixelizations +Tutorial 1: Pixelizations ========================= In the previous chapters, we used light profiles to model the light of a galaxy, where the light profile was an @@ -54,7 +54,7 @@ By itself, a pixelization does not tell us much. It has no grid of $(y,x)$ coordinates, no image, and no information about the galaxy we are fitting. -This information comes when we use the pixelization to create up a `Mapper`, which we perform below using the grid +This information comes when we use the pixelization to create a `Mapper`, which we perform below using the grid that we created above. """ interpolator = mesh.interpolator_from( @@ -64,7 +64,7 @@ mapper = ag.Mapper(interpolator=interpolator) """ -This `Mapper` is a `RectangularMapper` -- every `Mesh` and `Pixelization` generates it owns mapper. +This `Mapper` is a `RectangularMapper` -- every `Mesh` and `Pixelization` generates its own mapper. """ print(type(mapper)) @@ -93,7 +93,7 @@ plot_mapper( mapper=mapper, mesh_grid=mapper.source_plane_mesh_grid, - title="Recntagular Grid With Pixel Cenres", + title="Rectangular Grid With Pixel Centres", ) """ @@ -136,6 +136,6 @@ This was a relatively gentle overview of pixelizations, but one that was hopefully easy to follow. Think about the following questions before moving on to the next tutorial: - 1) The rectangular pixelization`s edges are aligned with the most exterior coordinates of the source-grid. This is - intentional, why do you think this is? + 1) The rectangular pixelization`s edges are aligned with the most exterior coordinates of the grid it was created + from. This is intentional, why do you think this is? """ diff --git a/scripts/chapter_3_pixelizations/tutorial_2_mappers.py b/scripts/chapter_3_pixelizations/tutorial_2_mappers.py index 1889c8a..3f45623 100644 --- a/scripts/chapter_3_pixelizations/tutorial_2_mappers.py +++ b/scripts/chapter_3_pixelizations/tutorial_2_mappers.py @@ -2,7 +2,7 @@ Tutorial 2: Mappers =================== -In the previous tutorial, we used a pixelization to create made a `Mapper`. However, it was not clear what a `Mapper` +In the previous tutorial, we used a pixelization to create a `Mapper`. However, it was not clear what a `Mapper` does, why it was called a mapper and whether it was mapping anything at all! Therefore, in this tutorial, we'll cover mappers in more detail. @@ -10,6 +10,7 @@ __Contents__ - **Initial Setup:** Load the dataset for illustration. +- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. - **Mappers:** Understand how mappers map image-plane pixels to pixelization pixels. - **Mask:** Apply a mask and see how it affects the mapper. - **Wrap Up:** Summary of mapper concepts. @@ -25,11 +26,10 @@ """ __Initial Setup__ -we'll use complex galaxy data, where: +we'll use galaxy data, where: - The galaxy's bulge is an `Sersic`. - The galaxy's disk is an `Exponential`. - - The galaxy's has four star forming clumps which are `Sersic` profiles. """ dataset_name = "simple" dataset_path = Path("dataset") / "imaging" / dataset_name @@ -66,7 +66,7 @@ """ __Mappers__ -We now setup a `Pixelization` and use it to create a `Mapper` via the plane`s source-plane grid, just like we did in +We now setup a `Pixelization` and use it to create a `Mapper` via the image's grid, just like we did in the previous tutorial. We will make its pixelization resolution half that of the grid above. @@ -84,7 +84,7 @@ mapper = ag.Mapper(interpolator=interpolator) """ -We now plot the `Mapper` alongside the image we used to generate the source-plane grid. +We now plot the `Mapper` alongside the image we used to generate the grid. Using the `Visuals2D` object we are also going to highlight specific grid coordinates certain colors, such that we can see how they map from the image grid to the pixelization grid. @@ -129,8 +129,7 @@ aplt.plot_array(array=dataset.data, title="Data") """ -We can now use the masked grid to create a new `Mapper` (using the same rectangular 25 x 25 pixelization -as before). +We can now use the masked grid to create a new `Mapper` (using the same rectangular pixelization as before). """ interpolator = mesh.interpolator_from( source_plane_data_grid=dataset.grids.pixelization, source_plane_mesh_grid=None @@ -143,8 +142,8 @@ aaplt.subplot_image_and_mapper(mapper=mapper, image=dataset.data) """ -First, We can see a red circle of dots in both the image and pixelization, showing where the edge of the mask -maps too in the pixelization. +First, we can see a red circle of dots in both the image and pixelization, showing where the edge of the mask +maps to in the pixelization. Now lets show that when we plot pixelization pixel indexes, they still appear in the same place in the image. """ @@ -167,7 +166,7 @@ In this tutorial, we learnt about mappers, and we used them to understand how the image and pixelization map to one another. Your exercises are: - 1) Think about how this could help us actually model galaxies. We have said we're going to reconstruct our galaxies - on the pixel-grid. So, how does knowing how each pixel maps to the image actually help us? If you`ve not got - any bright ideas, then worry not, that exactly what we're going to cover in the next tutorial. + 1) Think about how this could help us actually model galaxies. We have said we're going to reconstruct our galaxies + on the pixel-grid. So, how does knowing how each pixel maps to the image actually help us? If you`ve not got + any bright ideas, then worry not, that's exactly what we're going to cover in the next tutorial. """ diff --git a/scripts/chapter_3_pixelizations/tutorial_3_inversions.py b/scripts/chapter_3_pixelizations/tutorial_3_inversions.py index 55f8cd3..3e04ebe 100644 --- a/scripts/chapter_3_pixelizations/tutorial_3_inversions.py +++ b/scripts/chapter_3_pixelizations/tutorial_3_inversions.py @@ -7,12 +7,13 @@ - `Pixelization`'s: which place a pixel-grid over the image data. - `Mappers`'s: which describe how each pixelization pixel maps to one or more image pixels. -However, non of this has actually helped us fit galaxy data or reconstruct the galaxy. This is the subject +However, none of this has actually helped us fit galaxy data or reconstruct the galaxy. This is the subject of this tutorial, where the process of reconstructing the galaxy's light on the pixelization is called an `Inversion`. __Contents__ - **Initial Setup:** Load the dataset for illustration. +- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. - **Pixelization:** Create a pixelization and perform an inversion to reconstruct the galaxy. - **Positive Only Solver:** Ensure the reconstruction has only positive intensity values. - **Wrap Up:** Summary of inversion concepts. @@ -29,11 +30,10 @@ """ __Initial Setup__ -we'll use the same complex galaxy data as the previous tutorial, where: +we'll use the same galaxy data as the previous tutorial, where: - The galaxy's bulge is an `Sersic`. - The galaxy's disk is an `Exponential`. - - The galaxy's has four star forming clumps which are `Sersic` profiles. """ dataset_name = "simple" dataset_path = Path("dataset") / "imaging" / dataset_name @@ -124,16 +124,17 @@ aaplt.subplot_of_mapper(inversion=inversion, mapper_index=0) """ -There we have it, we have successfully reconstructed the galaxy using a rectangular pixel-grid. This has reconstructed -the complex blobs of light of the galaxy. +There we have it, we have successfully reconstructed the galaxy using a rectangular pixel-grid, capturing the +structure of its light without assuming an analytic form for it. -Pretty great, huh? If you ran the complex source pipeline in chapter 3, you'll remember that getting a model image -that looked this good simply *was not possible*. With an inversion, we can do this with ease and without having to +Pretty great, huh? If you fitted this galaxy via search chaining at the end of chapter 2 (tutorials 9-10), you'll +remember how much effort a good model image took. With an inversion, we can do this with ease and without having to perform model-fitting with 20+ parameters for the galaxy's light! -We will now briefly discuss how an inversion actually works, however the explanation I give in this tutorial will be -overly-simplified. To be good at modeling you do not need to understand the details of how an inversion works, you -simply need to be able to use an inversion to model a galaxy. +We will now briefly discuss how an inversion actually works, however the explanation given in this tutorial will be +overly-simplified. (The full linear algebra -- the mapping matrices, the linear solve and the equations behind +them -- is derived step-by-step in tutorial 5.) To be good at modeling you do not need to understand the details of +how an inversion works, you simply need to be able to use an inversion to model a galaxy. To begin, lets consider some random mappings between our mapper`s pixelization pixels and the image. """ @@ -196,17 +197,17 @@ """ __Positive Only Solver__ -All pixelized source reconstructions use a positive-only solver, meaning that every source-pixel is only allowed -to reconstruct positive flux values. This ensures that the source reconstruction is physical and that we don't -reconstruct negative flux values that don't exist in the real source galaxy (a common systematic solution in lens -analysis). +All pixelized galaxy reconstructions use a positive-only solver, meaning that every pixelization pixel is only +allowed to reconstruct positive flux values. This ensures that the reconstruction is physical and that we don't +reconstruct negative flux values that don't exist in the real galaxy (a common unphysical systematic in methods +without this constraint). It may be surprising to hear that this is a feature worth pointing out, but it turns out setting up the linear algebra to enforce positive reconstructions is difficult to make efficient. A lot of development time went into making this possible, where a bespoke fast non-negative linear solver was developed to achieve this. -Other methods in the literature often do not use a positive only solver, and therefore suffer from these -unphysical solutions, which can degrade the results of lens model in general. +Other methods in the literature often do not use a positive only solver, and therefore suffer from these +unphysical solutions, which can degrade the results of galaxy models in general. __Wrap Up__ @@ -221,7 +222,8 @@ __Detailed Explanation__ -If you are interested in a more detailed description of how inversions work, then checkout the file -`autogalaxy_workspace/*/imaging/features/pixelization/likelihood_function.ipynb` which gives a visual step-by-step -guide of the process alongside equations and references to literature on the subject. +The linear algebra sketched above -- setting up the mappings as a matrix and solving for the pixelization pixel +fluxes -- is derived in full in tutorial 5 of this chapter, where we build every matrix by hand and perform the +solve ourselves. The file `autogalaxy_workspace/*/imaging/features/pixelization/likelihood_function.ipynb` gives a +further visual step-by-step guide of the process alongside equations and references to literature on the subject. """ diff --git a/scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py b/scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py index dfbe672..90234a1 100644 --- a/scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py +++ b/scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py @@ -15,12 +15,14 @@ __Contents__ - **Initial Setup:** Load the dataset for illustration. +- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. - **Convenience Function:** A helper function for performing inversions. - **Pixelization:** Perform inversions with different regularization coefficients. - **Regularization:** Understand how regularization smooths the reconstruction. - **Bayesian Evidence:** Use the Bayesian evidence to objectively choose the regularization coefficient. - **Non-Linear and Linear:** Discussion of how regularization interacts with the non-linear search. - **Detailed Description:** In-depth explanation of how the Bayesian evidence penalizes overfitting. +- **Wrap Up:** Summary of Bayesian regularization concepts. """ # from autogalaxy import setup_notebook; setup_notebook() @@ -33,11 +35,10 @@ """ __Initial Setup__ -we'll use the same complex galaxy data as the previous tutorial, where: +we'll use the same galaxy data as the previous tutorial, where: - The galaxy's bulge is an `Sersic`. - The galaxy's disk is an `Exponential`. - - The galaxy's has four star forming clumps which are `Sersic` profiles. """ dataset_name = "simple" dataset_path = Path("dataset") / "imaging" / dataset_name @@ -125,9 +126,9 @@ def perform_fit_with_galaxy(dataset, galaxy): So, what is happening here? Why does reducing the `coefficient` do this to our reconstruction? First, we need to understand what regularization actually does! -When the inversion reconstructs the galaxy, it does not *only* compute the set of pixelization pixel fluxes that -best-fit the image. It also regularizes this solution, whereby it goes to every pixel on the rectangular grid -and computes the different between the reconstructed flux values of every pixel with its 4 neighboring pixels. +When the inversion reconstructs the galaxy, it does not *only* compute the set of pixelization pixel fluxes that +best-fit the image. It also regularizes this solution, whereby it goes to every pixel on the rectangular grid +and computes the difference between the reconstructed flux values of every pixel with its 4 neighboring pixels. If the difference in flux is large the solution is penalized, reducing its log likelihood. You can think of this as us applying a 'smoothness prior' on the reconstructed galaxy's light. @@ -158,7 +159,7 @@ def perform_fit_with_galaxy(dataset, galaxy): subplot_of_mapper(inversion=high_regularization_fit.inversion, mapper_index=0) """ -The figure above shows that we completely remove over-fitting. However, we now fit the image data less poorly, +The figure above shows that we completely remove over-fitting. However, we now fit the image data more poorly, due to the much higher level of smoothing. So, we now understand what regularization is and why it is necessary. There is one nagging question that remains, how @@ -191,9 +192,12 @@ def perform_fit_with_galaxy(dataset, galaxy): Bayesian evidence will decrease. The evidence penalizes solutions which are complex, which, in a Bayesian sense, are less probable (you may want to look up `Occam`s Razor`). -The Bayesian evidence therefore ensures we only invoke a more complex galaxy reconstruction when the data absolutely +The Bayesian evidence therefore ensures we only invoke a more complex galaxy reconstruction when the data absolutely necessitates it. +(This tutorial describes the Bayesian evidence in words; tutorial 5 writes it down as an equation, deriving each of +its terms via the linear algebra of the inversion.) + Lets take a look at the Bayesian evidence of the fits that we performed above, which is accessible from a `FitImaging` object via the `log_evidence` property: """ @@ -212,8 +216,8 @@ def perform_fit_with_galaxy(dataset, galaxy): Before we end, lets consider which aspects of an inversion are linear and which are non-linear. -The linear part of the inversion is the step that solves for the reconstruct pixelization pixel fluxes, including -accounting for the smoothing via regularizaton. We do not have to perform a non-linear search to determine the pixel +The linear part of the inversion is the step that solves for the reconstructed pixelization pixel fluxes, including +accounting for the smoothing via regularization. We do not have to perform a non-linear search to determine the pixel fluxes or compute the Bayesian evidence discussed above. However, determining the regularization `coefficient` that maximizes the Bayesian log evidence is a non-linear problem diff --git a/scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py b/scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py new file mode 100644 index 0000000..b2dd726 --- /dev/null +++ b/scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py @@ -0,0 +1,618 @@ +""" +Tutorial 5: Bayesian Formalism +============================== + +In tutorials 1 to 4, we built an intuition for how pixelized galaxy reconstruction works: pixelizations place a +pixel-grid over the galaxy's image, mappers pair pixelization pixels with image-pixels, inversions solve for the +pixel fluxes that best fit the data, and regularization smooths the solution within a Bayesian framework. + +This tutorial collects the linear algebra behind all of that in one place. The hands-on tutorials built intuition, +and now we write down the equations. We will construct, step-by-step and in code, every matrix and vector the +inversion uses, solve for the galaxy reconstruction ourselves and compute the Bayesian evidence by hand, comparing +our answer at the end to the `FitImaging` object which performs this calculation internally. + +None of this is required to *use* **PyAutoGalaxy** for galaxy modeling. However, if you publish results which use a +pixelization, this is the calculation your paper's likelihood function section will describe, and understanding it +removes any sense that the galaxy reconstruction is a "black box". + +The formalism follows Warren & Dye 2003 (https://arxiv.org/abs/astro-ph/0302587), hereafter WD03, with the data +vector notation of Nightingale & Dye 2015 (https://arxiv.org/abs/1412.7436), hereafter N15. These papers derived +the method for reconstructing gravitationally lensed sources, but the linear algebra is identical for a galaxy's +own light -- the only lensing-specific step (ray tracing image-pixel coordinates to a source-plane) is simply +absent, with the mesh laid directly over the galaxy's image. The Bayesian evidence was derived by Suyu et al. 2006 +(https://arxiv.org/abs/astro-ph/0601493) and translated to **PyAutoGalaxy** notation in Dye et al. 2008 +(https://arxiv.org/abs/0804.4002). + +__Contents__ + +- **Initial Setup:** Load the dataset the previous tutorials fitted. +- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. +- **Mask:** Mask the data so the likelihood is evaluated only where the galaxy's light is. +- **Over Sampling:** Disable over sampling so the algebra stays simple. +- **Mesh Shape:** Fix the rectangular mesh's shape and note why edge pixels are zeroed. +- **Galaxy:** Create the galaxy whose pixelization pairs the mesh with `Constant` regularization. +- **Image Grid:** The masked image-pixel coordinates the mesh is laid over -- no ray tracing occurs. +- **Mesh Pixel Centres:** Overlay the rectangular mesh over the masked image grid. +- **Interpolation:** Pair every image-pixel with mesh pixels via bilinear interpolation. +- **Mapper:** Package the interpolation into a `Mapper` describing all image-to-mesh mappings. +- **Mapping Matrix:** Express the mappings as the 2D matrix $f$. +- **Blurred Mapping Matrix:** Convolve every column of $f$ with the imaging PSF. +- **Data Vector:** Compute the data vector $D$ from the blurred mapping matrix, data and noise-map. +- **Curvature Matrix:** Compute the curvature matrix $F$. +- **Unregularized Solve:** Solve $s = F^{-1} D$ and see the over-fitted mess this produces. +- **Regularization Matrix:** Compute the regularization matrix $H$ encoding the smoothness prior. +- **Galaxy Reconstruction:** Solve the regularized system $s = [F + H]^{-1} D$. +- **Image Reconstruction:** Map the reconstruction back to image resolution via the blurred mapping matrix. +- **Likelihood Function:** The five terms which combine into the log evidence. +- **Chi Squared:** The goodness-of-fit of the reconstructed image to the data. +- **Regularization Term:** The penalty $s^{T} H s$ applied by the smoothness prior. +- **Complexity Terms:** The log determinant terms which penalize complex galaxy reconstructions. +- **Noise Normalization Term:** The Gaussian noise normalization. +- **Log Evidence:** Combine all five terms into the log evidence. +- **Fit:** Compare our by-hand log evidence to the `FitImaging` object's internal calculation. +- **Wrap Up:** Summary and next steps. +""" + +# from autogalaxy import setup_notebook; setup_notebook() + +import matplotlib.pyplot as plt +import numpy as np +from pathlib import Path + +import autogalaxy as ag +import autogalaxy.plot as aplt +from autoarray.inversion.plot.mapper_plots import plot_mapper + +""" +__Initial Setup__ + +we'll use the same galaxy data as the previous tutorials, where: + + - The galaxy's bulge is an `Sersic`. + - The galaxy's disk is an `Exponential`. + +For simplicity, the model in this tutorial is a pixelization only, with no light profiles: the mesh absorbs the +bulge and disk together. In WD03's notation this means the light profile model image $b_{j}$ is zero everywhere. If +light profiles are included in the galaxy model, their PSF-convolved image is computed first and subtracted from +the data before the steps below -- nothing else about the formalism changes. +""" +dataset_name = "simple" +dataset_path = Path("dataset") / "imaging" / dataset_name + +""" +__Dataset Auto-Simulation__ + +If the dataset does not already exist on your system, it will be created by running the corresponding +simulator script. This ensures that all example scripts can be run without manually simulating data first. +""" +if ag.util.dataset.should_simulate(str(dataset_path)): + import subprocess + import sys + + subprocess.run( + [sys.executable, "scripts/simulators/simple.py"], + check=True, + ) + +dataset = ag.Imaging.from_fits( + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + psf_path=dataset_path / "psf.fits", + pixel_scales=0.1, +) + +aplt.subplot_imaging_dataset(dataset=dataset) + +""" +__Mask__ + +The likelihood is only evaluated within a mask, which we define as the same 2.0" circle used in the previous +tutorials, containing all of the galaxy's light. +""" +mask = ag.Mask2D.circular( + shape_native=dataset.shape_native, + pixel_scales=dataset.pixel_scales, + radius=2.0, +) + +masked_dataset = dataset.apply_mask(mask=mask) + +aplt.subplot_imaging_dataset(dataset=masked_dataset) + +""" +__Over Sampling__ + +Over sampling splits every image pixel into a sub-grid of sub-pixels, each of which is paired with mesh pixels +individually. It improves accuracy, but it also multiplies the number of rows in every matrix below by the number +of sub-pixels. + +To keep the algebra as easy to follow as possible, we disable over sampling by setting both sub-grid sizes to 1, so +each image pixel is represented by the single coordinate at its centre. +""" +masked_dataset = masked_dataset.apply_over_sampling( + over_sample_size_lp=1, + over_sample_size_pixelization=1, +) + +""" +__Mesh Shape__ + +The `mesh_shape` defines the number of pixels in the rectangular mesh used to reconstruct the galaxy, set below +to 20 x 20 = 400 mesh pixels. + +We use the `RectangularUniform` mesh, where all rectangular mesh pixels have the same size, rather than the +`RectangularAdaptDensity` mesh used in the previous tutorials. The uniform mesh keeps the geometry simple, and every +equation below applies unchanged to the adaptive meshes -- only the mesh pixel centres move. + +By default, mesh pixels at the edge of the mesh are forced to solutions of zero flux by the linear algebra solver. +This prevents unphysical solutions where the mesh edge lights up to fit residuals, and does not change any of the +formalism below. +""" +mesh_pixels_yx = 20 +mesh_shape = (mesh_pixels_yx, mesh_pixels_yx) + +""" +__Galaxy__ + +We create the galaxy whose `Pixelization` pairs the `RectangularUniform` mesh with `Constant` regularization (whose +role appears later, when we reach the matrix $H$). + +The galaxy has no light profiles -- the mesh alone will reconstruct its bulge and disk. +""" +pixelization = ag.Pixelization( + mesh=ag.mesh.RectangularUniform(shape=mesh_shape), + regularization=ag.reg.Constant(coefficient=1.0), +) + +galaxy = ag.Galaxy(redshift=0.5, pixelization=pixelization) + +""" +__Image Grid__ + +In the lensing version of this formalism (see **PyAutoLens**), every image-pixel coordinate is first ray-traced to +a source-plane, and the mesh is laid over the traced coordinates. For a galaxy there is no lensing, so this step +simply does not exist: the mesh is laid directly over the galaxy's own image, and the grid the inversion uses is +the grid of masked image-pixel centres itself. + +This grid is `masked_dataset.grids.pixelization`, with one coordinate at the centre of every masked image pixel +(because we disabled over sampling above). + +(The `Mapper` attributes below carry `source_plane_` prefixes because **PyAutoGalaxy** and **PyAutoLens** share one +inversion implementation. For a galaxy, the "source plane" is just the image plane the galaxy lives in.) +""" +aplt.plot_grid(grid=masked_dataset.grids.pixelization, title="Masked Image Grid") + +""" +__Mesh Pixel Centres__ + +To reconstruct the galaxy on a `RectangularUniform` mesh, we need the centres of its rectangular mesh pixels. + +We compute these by overlaying a uniform rectangular grid over the masked image grid, sized so the mesh fully +contains the masked region without wasted edge pixels beyond it. +""" +from autoarray.inversion.mesh.mesh.rectangular_adapt_density import overlay_grid_from + +mesh_grid = overlay_grid_from( + shape_native=mesh_shape, grid=masked_dataset.grids.pixelization +) + +""" +__Interpolation__ + +We now combine the two grids to create an `Interpolator`, which describes how every image-pixel coordinate maps to +the rectangular mesh pixels. + +For a rectangular mesh the scheme is bilinear interpolation: every image pixel maps to the rectangular pixel it +lands in *and* its three nearest neighbours, with weights set by how close the coordinate is to each pixel centre. +Interpolation is what lets the mesh reconstruct smooth galaxy morphologies. We can print the mappings and weights +of the first image pixel to confirm it pairs with four mesh pixels. +""" +interpolator = pixelization.mesh.interpolator_from( + source_plane_data_grid=masked_dataset.grids.pixelization, + source_plane_mesh_grid=mesh_grid, +) + +print(interpolator.mappings[0]) +print(interpolator.weights[0]) + +""" +__Mapper__ + +The interpolator creates a `Mapper`, the object tutorials 1 and 2 introduced. It packages up the mapping between +every image pixel and every rectangular mesh pixel. + +The key attribute is `pix_indexes_for_sub_slim_index`, mapping every image-pixel index (`sub_slim_index`) to the +mesh-pixel indexes (`pix_indexes`) it interpolates onto, alongside the number of mappings per image pixel and +their interpolation weights. +""" +mapper = ag.Mapper(interpolator=interpolator) + +pix_indexes_for_sub_slim_index = mapper.pix_indexes_for_sub_slim_index + +print(pix_indexes_for_sub_slim_index[0:9]) +print(mapper.pix_sizes_for_sub_slim_index[0:9]) +print(mapper.pix_weights_for_sub_slim_index[0:9]) + +""" +__Mapping Matrix__ + +The `mapping_matrix` expresses these image-pixel to mesh-pixel mappings as a single 2D matrix, with +dimensions `(total_image_pixels, total_mesh_pixels)`. + +Each column is the "image" of one mesh pixel: entries are the interpolation weights for image pixels which map to +that mesh pixel and zero everywhere else. +""" +mapping_matrix = ag.util.mapper.mapping_matrix_from( + pix_indexes_for_sub_slim_index=pix_indexes_for_sub_slim_index, + pix_size_for_sub_slim_index=mapper.pix_sizes_for_sub_slim_index, + pix_weights_for_sub_slim_index=mapper.pix_weights_for_sub_slim_index, + pixels=mapper.pixels, + total_mask_pixels=mapper.source_plane_data_grid.mask.pixels_in_mask, + slim_index_for_sub_slim_index=mapper.slim_index_for_sub_slim_index, + sub_fraction=mapper.over_sampler.sub_fraction, +) + +plt.imshow(mapping_matrix, aspect=(mapping_matrix.shape[1] / mapping_matrix.shape[0])) +plt.show() +plt.close() + +""" +Because each column is an image of zeros and interpolation weights, we can plot any column as a 2D image showing +all image pixels its mesh pixel maps to. For a mesh pixel near the mesh centre, these image pixels sit at the +centre of the galaxy, where its bulge is brightest. +""" +central_mesh_pixel = (mesh_shape[0] // 2) * mesh_shape[1] + mesh_shape[1] // 2 + +array_2d = ag.Array2D( + values=mapping_matrix[:, central_mesh_pixel], mask=masked_dataset.mask +) + +aplt.plot_array(array=array_2d, title="Image of Central Mesh Pixel") + +""" +__Blurred Mapping Matrix__ + +The imaging data is blurred by the telescope's PSF, so the model must be too. Since each column of the mapping +matrix is an image, we simply convolve each column with the PSF via 2D convolution. + +This produces the `blurred_mapping_matrix`, with the same dimensions `(total_image_pixels, total_mesh_pixels)`. + +In WD03 this matrix is denoted $f_{ij}$, where $i$ runs over all $I$ mesh pixels and $j$ over all $J$ image +pixels. For example: + + - $f_{0, 2} = 0.3$ indicates that image-pixel $2$ maps to mesh-pixel $0$ with a weight of $0.3$ after PSF + convolution. + - $f_{4, 8} = 0$ indicates that image-pixel $8$ does not map to mesh-pixel $4$, even after PSF convolution. + +(The indexing of the code's `mapping_matrix` is transposed relative to WD03's $f$: image pixels are the first index +in the code, but the second index in the equations.) + +PSF blurring has an important consequence: it correlates neighbouring columns, so the images of nearby mesh pixels +now overlap far more than the interpolation alone produced. +""" +blurred_mapping_matrix = masked_dataset.psf.convolved_mapping_matrix_from( + mapping_matrix=mapping_matrix, mask=masked_dataset.mask +) + +plt.imshow( + blurred_mapping_matrix, + aspect=(blurred_mapping_matrix.shape[1] / blurred_mapping_matrix.shape[0]), +) +plt.colorbar() +plt.show() +plt.close() + +""" +__Data Vector (D)__ + +We now pose the reconstruction as a linear inversion, converting the blurred mapping matrix, data and noise-map +into two objects: the data vector $D$ and the curvature matrix $F$. + +The data vector has dimensions `(total_mesh_pixels,)` and is given by (WD03 / N15): + + $\vec{D}_{i} = \sum_{j=1}^{J} f_{ij} (d_{j} - b_{j}) / \sigma_{j}^2 \, \, .$ + +Where: + + - $d_{j}$ are the image-pixel data values. + - $b_{j}$ are the model images of any light profiles in the galaxy model (zero here, because our model is a + pixelization only). + - $\sigma_{j}^2$ are the statistical uncertainties of each image pixel. + +Each entry of $D$ is therefore the noise-weighted overlap between one mesh pixel's blurred image and the data: +it measures how much evidence the data provides for flux in that mesh pixel, with the PSF fully accounted for. +""" +data_vector = ag.util.inversion_imaging.data_vector_via_blurred_mapping_matrix_from( + blurred_mapping_matrix=blurred_mapping_matrix, + image=np.array(masked_dataset.data), + noise_map=np.array(masked_dataset.noise_map), +) + +plt.imshow( + data_vector.reshape(data_vector.shape[0], 1), aspect=10.0 / data_vector.shape[0] +) +plt.colorbar() +plt.show() +plt.close() + +""" +__Curvature Matrix (F)__ + +The curvature matrix has dimensions `(total_mesh_pixels, total_mesh_pixels)` and is given by (WD03): + + ${F}_{ik} = \sum_{j=1}^{J} f_{ij} f_{kj} / \sigma_{j}^2 \, \, .$ + +Every entry of $F$ is the noise-weighted overlap between the blurred images of two mesh pixels: $F_{ik}$ sums the +product of columns $i$ and $k$ of $f$ over all image pixels. For $F_{ik}$ to be non-zero, mesh pixels $i$ and $k$ +must share at least one image pixel, which happens for neighbouring pixels via interpolation and for pixels +further apart via PSF blurring. + +$F$ describes how degenerate pairs of mesh pixels are with one another: two mesh pixels whose blurred images +overlap heavily can trade flux between themselves whilst fitting the data almost equally well. +""" +curvature_matrix = ag.util.inversion.curvature_matrix_via_mapping_matrix_from( + mapping_matrix=blurred_mapping_matrix, noise_map=masked_dataset.noise_map +) + +plt.imshow(curvature_matrix) +plt.colorbar() +plt.show() +plt.close() + +""" +__Unregularized Solve__ + +The inversion seeks the mesh-pixel fluxes $s$ (a vector with one entry per mesh pixel) that minimize the +chi-squared: + + $\chi^2 = \sum_{j=1}^{J} \bigg[ \frac{(\sum_{i=1}^{I} s_{i} f_{ij}) + b_{j} - d_{j}}{\sigma_{j}} \bigg]^2$ + +Setting the derivative of $\chi^2$ with respect to each $s_{i}$ to zero gives the linear system whose solution is +(equation 5 of WD03): + + $s = F^{-1} D$ + +We can solve this directly with NumPy. (Without regularization the curvature matrix is often singular, so the loop +below adds a tiny value to its diagonal to avoid a `LinAlgError` -- it is a numerical crutch, not part of the +formalism.) +""" +for i in range(curvature_matrix.shape[0]): + curvature_matrix[i, i] += 1e-8 + +reconstruction = np.linalg.solve(curvature_matrix, data_vector) + +plot_mapper(mapper=mapper, solution_vector=reconstruction) + +""" +The reconstructed mesh-pixel fluxes are a noisy, unsmooth mess -- exactly the over-fitting we saw in tutorial 4 +when we lowered the regularization coefficient towards zero. The linear inversion is fitting the noise in the +data, because this system of equations is ill-posed: we need a smoothness prior. + +__Regularization Matrix (H)__ + +Regularization adds a linear regularization term $G_{L}$ to the merit function we minimize (equation 11 of WD03): + + $G = \chi^2 + \lambda \, G_{L}$ + +where $\lambda$ is the `regularization_coefficient` controlling the degree of smoothing. The `Constant` scheme uses +gradient regularization (equation 14 of WD03): + + $G_{L} = \sum_{i}^{I} \sum_{n=1}^{N} [s_{i} - s_{i, n}]^2$ + +In words: for every mesh pixel, compare its flux with each of its $N$ neighbours $n$, and penalize solutions where +the differences are large. This is precisely the "smoothness prior" of tutorial 4, now written as an equation. + +To fold this into the linear algebra we define the regularization matrix $H$, with +dimensions `(total_mesh_pixels, total_mesh_pixels)` (equation 13 of WD03): + + $H_{ik} = \frac{1}{2} \frac{\partial^{2} G_{L}}{\partial s_{i} \partial s_{k}}$ + +$H$ has the coefficient $\lambda$ folded into it. Its non-zero off-diagonal entries mark pairs of mesh pixels +which are neighbours and therefore regularized with one another; most entries are zero because most mesh pixels +are not neighbours. +""" +regularization_matrix = ag.util.regularization.constant_regularization_matrix_from( + coefficient=galaxy.pixelization.regularization.coefficient, + neighbors=mapper.neighbors, + neighbors_sizes=mapper.neighbors.sizes, +) + +plt.imshow(regularization_matrix) +plt.colorbar() +plt.show() +plt.close() + +""" +__Galaxy Reconstruction (s)__ + +$H$ enters the linear system as follows (equation 12 of WD03): + + $s = [F + H]^{-1} D$ + +We add the two matrices and solve again. The diagonal jitter used above is no longer needed, because $H$ makes the +system well-posed. +""" +curvature_reg_matrix = np.add(curvature_matrix, regularization_matrix) + +reconstruction = np.linalg.solve(curvature_reg_matrix, data_vector) + +plot_mapper(mapper=mapper, solution_vector=reconstruction) + +""" +The reconstructed fluxes are now smooth and physical: regularization has suppressed the noisy solution and the +reconstruction actually looks like the galaxy's bulge and disk, without over-fitting the noise. + +__Image Reconstruction__ + +Using the reconstructed mesh-pixel fluxes, we map the reconstruction back to image resolution via the blurred +mapping matrix (so the reconstructed image includes PSF blurring) to produce the model image of the galaxy. +""" +mapped_reconstructed_data = ( + ag.util.inversion.mapped_reconstructed_data_via_mapping_matrix_from( + mapping_matrix=blurred_mapping_matrix, reconstruction=reconstruction + ) +) + +mapped_reconstructed_data = ag.Array2D( + values=mapped_reconstructed_data, mask=masked_dataset.mask +) + +aplt.plot_array(array=mapped_reconstructed_data, title="Reconstructed Image") + +""" +__Likelihood Function__ + +We now quantify the goodness-of-fit of the galaxy reconstruction, computing the quantity tutorial 4 called the +Bayesian evidence. The log evidence consists of five terms: + + $-2 \, \mathrm{ln} \, \epsilon = \chi^2 + s^{T} H s + \mathrm{ln} \, [ \mathrm{det} (F + H) ] - \mathrm{ln} \, [ \mathrm{det} (H) ] + \sum_{j=1}^{J} \mathrm{ln} \, [2 \pi (\sigma_{j})^2 ] \, .$ + +This expression was first derived by Suyu et al. 2006 (https://arxiv.org/abs/astro-ph/0601493), equation (19), and +is given in **PyAutoGalaxy** notation by Dye et al. 2008 (https://arxiv.org/abs/0804.4002), equation (5). + +We now compute each term in turn. + +__Chi Squared__ + +The first term is the $\chi^2$ statistic from the merit function above, computed as: + + - `model_data` = the reconstructed image of the galaxy (plus any light profile model images, zero here). + - `residual_map` = (`data` - `model_data`) + - `normalized_residual_map` = (`data` - `model_data`) / `noise_map` + - `chi_squared_map` = (`normalized_residual_map`) ** 2.0 + - `chi_squared` = sum(`chi_squared_map`) + +High chi-squared values indicate image pixels the reconstruction fits poorly, lowering the likelihood. +""" +model_image = mapped_reconstructed_data + +residual_map = masked_dataset.data - model_image +normalized_residual_map = residual_map / masked_dataset.noise_map +chi_squared_map = normalized_residual_map**2.0 + +chi_squared = np.sum(chi_squared_map) + +print(chi_squared) + +""" +__Regularization Term__ + +The second term, $s^{T} H s$, is the $\lambda \, G_{L}$ regularization penalty evaluated at the solution: the +summed difference in flux between all neighbouring mesh pixels, weighted by the regularization coefficient (which +is already folded into $H$). + +Less smooth solutions have larger values of this term and therefore lower likelihoods. +""" +regularization_term = np.matmul( + reconstruction.T, np.matmul(regularization_matrix, reconstruction) +) + +print(regularization_term) + +""" +__Complexity Terms__ + +Up to this point, nothing has justified our choice of `regularization_coefficient=1.0`. We cannot choose it using +the two terms above, because increasing the coefficient smooths the solution more, which *both* worsens the +chi-squared *and* (for a fixed solution) raises the regularization penalty. Optimizing those two terms alone would +drive the coefficient to zero and put us right back at the over-fitted mess. + +The two log determinant terms, $\mathrm{ln} \, [ \mathrm{det} (F + H) ]$ and $- \mathrm{ln} \, [ \mathrm{det} (H) ]$, +fix this. Together they measure how *complex* the galaxy reconstruction is -- roughly, how many effective degrees +of freedom the mesh uses after regularization correlates its pixels -- and penalize more complex solutions. +Lowering the regularization coefficient frees the mesh to use more of its flexibility, increasing this complexity +penalty. + +These terms therefore counteract the chi-squared and regularization terms, so the highest evidence goes to +solutions which fit the data well with the *simplest* galaxy reconstruction. This is the Occam's razor behaviour +that tutorial 4 demonstrated empirically. +""" +log_curvature_reg_matrix_term = np.linalg.slogdet(curvature_reg_matrix)[1] +log_regularization_matrix_term = np.linalg.slogdet(regularization_matrix)[1] + +print(log_curvature_reg_matrix_term) +print(log_regularization_matrix_term) + +""" +__Noise Normalization Term__ + +The likelihood function assumes the imaging data consists of independent Gaussian noise in every image pixel, and +the final term is the normalization of those Gaussians: the sum of the log of every noise-map value squared. + +Because the noise-map is fixed, this term is constant throughout galaxy modeling and has no impact on the model +we infer -- it simply normalizes the likelihood. +""" +noise_normalization = float(np.sum(np.log(2 * np.pi * masked_dataset.noise_map**2.0))) + +print(noise_normalization) + +""" +__Log Evidence__ + +We can now combine the five terms into the log evidence of the galaxy reconstruction. +""" +log_evidence = float( + -0.5 + * ( + chi_squared + + regularization_term + + log_curvature_reg_matrix_term + - log_regularization_matrix_term + + noise_normalization + ) +) + +print(log_evidence) + +""" +__Fit__ + +Everything above is what the `FitImaging` object does internally when it fits a galaxy with a pixelization. We can +see this by performing the fit and comparing its `log_evidence` to ours. + +The two values do not agree exactly, because the real fit improves on our simplified solve in two ways mentioned +along the way: it uses the positive-only solver (tutorial 3), which forbids the negative mesh-pixel fluxes our +unconstrained `np.linalg.solve` permits, and it zeroes the pixels at the edge of the mesh. Our unconstrained solve +exploits that extra (unphysical) freedom to push its chi-squared lower than the real solver allows, which is why +our by-hand log evidence comes out somewhat higher. Neither constraint changes the formalism -- the same $f$, $D$, +$F$ and $H$ feed a solver with extra conditions on $s$. +""" +galaxies = ag.Galaxies(galaxies=[galaxy]) + +fit = ag.FitImaging(dataset=masked_dataset, galaxies=galaxies) + +print(fit.log_evidence) + +aplt.subplot_fit_imaging(fit=fit) + +""" +__Wrap Up__ + +We have walked through the complete linear algebra of a pixelized galaxy reconstruction: + + - The `mapping_matrix` and PSF-blurred mapping matrix $f$, whose columns are the blurred images of each + mesh pixel. + + - The data vector $D$ and curvature matrix $F$, the noise-weighted overlaps of those images with the data and with + each other. + + - The regularization matrix $H$, which encodes the smoothness prior, and the linear solve $s = [F + H]^{-1} D$ for + the galaxy reconstruction. + + - The five terms of the Suyu et al. 2006 log evidence -- chi-squared, the regularization penalty, the two log + determinant complexity terms and the noise normalization -- and their Bayesian interpretation as an Occam's + razor which favours the simplest galaxy reconstruction the data allows. + +During galaxy modeling, this whole calculation is one likelihood evaluation: the non-linear search varies the +model's parameters (for example the mesh resolution, the regularization coefficient, or the light profiles fitted +alongside the mesh), and each sample triggers the full solve and evidence computation above. + +Two simplifications are worth remembering: real fits use over sampling (each image pixel contributes several +sub-pixel rows to $f$) and **PyAutoGalaxy** uses a positive-only solver for $s$ rather than the unconstrained +`np.linalg.solve` used here (see tutorial 3). The workspace +guide `autogalaxy_workspace/*/imaging/features/pixelization/likelihood_function.ipynb` repeats this walk-through +with additional visualization of every step, including how light profiles combine with the mesh. + +In the next tutorial, we return to hands-on territory and use pixelizations in an actual model-fit, combining light +profiles and an inversion via search chaining. +""" diff --git a/scripts/chapter_3_pixelizations/tutorial_6_model_fit.py b/scripts/chapter_3_pixelizations/tutorial_6_model_fit.py index a7fe3df..de01cec 100644 --- a/scripts/chapter_3_pixelizations/tutorial_6_model_fit.py +++ b/scripts/chapter_3_pixelizations/tutorial_6_model_fit.py @@ -1,27 +1,28 @@ """ -Tutorial 5: Model-Fit +Tutorial 6: Model Fit ===================== In the previous tutorials we used an inversion to reconstruct a complex galaxy. However, from the perspective of -a scientific analysis, it is not clear how useful this was. When I fit a galaxy with light profiles, I learn about +a scientific analysis, it is not clear how useful this was. When we fit a galaxy with light profiles, we learn about its brightness (`intensity`), size (`effective_radius`), compactness (`sersic_index`), etc. -What did I learn about the galaxy I reconstructed? Not a lot, perhaps. +What did we learn about the galaxy we reconstructed? Not a lot, perhaps. -Inversions are most useful when combined with light profiles. For the complex galaxy we used throughout this tutorial, -we can fit it with light profiles to quantify the properties of its `bulge` and `disk` components, whilst -simultaneously fitting the clumps with the inversion so as to ensure they do not impact the fit. +Inversions are most useful when combined with light profiles. For the complex galaxy we have used throughout this +chapter, we can fit it with light profiles to quantify the properties of its `bulge` and `disk` components, whilst +simultaneously fitting any residual structure with the inversion so as to ensure it does not impact the fit. -To illustrate modeling using an inversion this tutorial therefore revisits the complex galaxy model-fit that we -performed in tutorial 4 of chapter 3. This time, as you have probably guessed, we will fit part of the complex galaxy -using an inversion. +To illustrate modeling using an inversion, this tutorial therefore revisits the complex galaxy model-fit that we +performed via search chaining at the end of chapter 2 (tutorials 9-10). This time, as you have probably guessed, we +will fit part of the galaxy using an inversion. We will use search chaining to do this, first fitting the main galaxy components with light profiles, thereby initializing the bulge and disk components. In the later searches we will switch to an `Inversion`. __Contents__ -- **Initial Setup:** Load the complex galaxy dataset and apply a mask. +- **Initial Setup:** Load the galaxy dataset and apply a mask. +- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. - **Model + Search + Analysis + Model-Fit (Search 1):** Fit light profiles to the main galaxy components. - **Mesh Shape:** Discussion of how mesh shape affects the inversion. - **Model + Search + Analysis + Model-Fit (Search 2):** Fit with a pixelization for residual structure. @@ -39,11 +40,10 @@ """ __Initial Setup__ -we'll use complex galaxy data, where: +we'll use the same galaxy data as the previous tutorials, where: - The galaxy's bulge is an `Sersic`. - The galaxy's disk is an `Exponential`. - - The galaxy's has four star forming clumps which are `Sersic` profiles. """ dataset_name = "simple" dataset_path = Path("dataset") / "imaging" / dataset_name @@ -113,7 +113,7 @@ model_1 = af.Collection(galaxies=af.Collection(galaxy=galaxy)) search_1 = af.Nautilus( - path_prefix=Path("howtogalaxy", "chapter_4"), + path_prefix=Path("howtogalaxy", "chapter_3"), name="search[1]", unique_tag=dataset_name, n_live=100, @@ -127,11 +127,11 @@ """ __Mesh Shape__ -The `mesh_shape` parameter defines number of pixels used by the rectangular mesh to reconstruct the source, -set below to 28 x 28. +The `mesh_shape` parameter defines the number of pixels used by the rectangular mesh to reconstruct the galaxy, +set below to 28 x 28. The `mesh_shape` must be fixed before modeling and cannot be a free parameter of the model, because JAX uses the -mesh shape to define static shaped arrays which use the mesh to reconstruct the source. For a rectangular +mesh shape to define static shaped arrays which use the mesh to reconstruct the galaxy. For a rectangular mesh, the same number of pixels must be used in the y and x directions. """ mesh_pixels_yx = 28 @@ -143,17 +143,18 @@ We use the results of search 1 to create the model fitted in search 2, where: - The galaxy's bulge is an `Sersic` [0 parameters: parameters fixed from search 1]. - + - The galaxy's disk is an `Exponential` [0 parameters: parameters fixed from search 1]. - - The galaxy's clumps are reconstructed `RectangularAdaptDensity` mesh with resolution as free parameters [2 parameters]. + - Residual galaxy structure is reconstructed using a `RectangularAdaptDensity` mesh with a fixed 28 x 28 + shape [0 parameters]. - - This pixelization is regularized using a `Constant` scheme [1 parameter]. + - This pixelization is regularized using a `GaussianKernel` scheme [2 parameters]. -The number of free parameters and therefore the dimensionality of non-linear parameter space is N=3. +The number of free parameters and therefore the dimensionality of non-linear parameter space is N=2. -This search allows us to very efficiently set up the resolution of the mesh and regularization coefficient -of the regularization scheme, before using these models to refit the galaxy mass model. +This search allows us to very efficiently set up the regularization scheme's parameters, before refitting the +galaxy's bulge and disk alongside the inversion in search 3. """ pixelization = af.Model( ag.Pixelization, @@ -172,7 +173,7 @@ model_2 = af.Collection(galaxies=af.Collection(galaxy=galaxy)) search_2 = af.Nautilus( - path_prefix=Path("howtogalaxy", "chapter_4"), + path_prefix=Path("howtogalaxy", "chapter_3"), name="search[2]", unique_tag=dataset_name, n_live=50, @@ -193,16 +194,17 @@ We use the results of searches 1 and 2 to create the model fitted in search 3, where: - The galaxy's bulge is an `Sersic` [7 parameters: priors initialized from search 1]. - - - The galaxy's disk is an `Exponential` [6 parameters: priors initialized from search 1]. - - The galaxy's light uses a `RectangularAdaptDensity` mesh[parameters fixed to results of search 2]. + - The galaxy's disk is an `Sersic` [7 parameters: priors initialized from search 1]. - - This pixelization is regularized using a `Constant` scheme [parameters fixed to results of search 2]. + - Residual galaxy structure uses a `RectangularAdaptDensity` mesh with a fixed 28 x 28 shape [0 parameters]. -The number of free parameters and therefore the dimensionality of non-linear parameter space is N=13. + - This pixelization is regularized using a `GaussianKernel` scheme [2 parameters]. -This search allows us to refit the bulge and disk components with an inversion that takes care of the clumps. +The number of free parameters and therefore the dimensionality of non-linear parameter space is N=16. + +This search allows us to refit the bulge and disk components with an inversion that takes care of any residual +structure the light profiles do not capture. """ bulge = af.Model(ag.lp.Sersic) bulge.ell_comps = result_1.model.galaxies.galaxy.bulge.ell_comps @@ -228,7 +230,7 @@ model_3 = af.Collection(galaxies=af.Collection(galaxy=galaxy)) search_3 = af.Nautilus( - path_prefix=Path("howtogalaxy", "chapter_4"), + path_prefix=Path("howtogalaxy", "chapter_3"), name="search[3]", unique_tag=dataset_name, n_live=100, @@ -242,5 +244,13 @@ """ __Wrap Up__ -And with that, we now have a pipeline to model galaxies using an inversion! +And with that, we now have a pipeline to model galaxies using an inversion! + +This concludes the pixelizations chapter. You now have every tool this chapter set out to teach: pixelizations and +mappers (tutorials 1-2), inversions (tutorial 3), Bayesian regularization and the evidence (tutorials 4-5), and a +search-chained pipeline which combines light profiles with an inversion (this tutorial). + +In chapter 4, we scale up from a single galaxy to the busier fields real imaging surveys deliver: datasets with +extra galaxies near the main galaxy, blended multi-galaxy systems modeled simultaneously, and cluster fields with +many member galaxies. """ From 05a6e93462bfd87353dfc433c34074f9561f04aa Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:47:44 +0000 Subject: [PATCH 10/15] docs: update chapter READMEs and top-level overviews for the new structure Chapter 2's index gains tutorials 9-10; a chapter 4 index is added; the top-level scripts/notebooks READMEs replace the old chaining/pixelization chapter overviews with the real chapter 3 (pixelizations + Bayesian formalism) and chapter 4 (scaling up galaxies) maps; notebooks copies synced. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- notebooks/README.md | 35 ++++++++++--------- notebooks/chapter_2_modeling/README.md | 7 +++- notebooks/chapter_3_pixelizations/README.md | 14 ++++---- .../chapter_4_scaling_up_galaxies/README.md | 12 +++++++ scripts/README.md | 35 ++++++++++--------- scripts/chapter_2_modeling/README.md | 7 +++- .../chapter_4_scaling_up_galaxies/README.md | 12 +++++++ 7 files changed, 80 insertions(+), 42 deletions(-) create mode 100644 notebooks/chapter_4_scaling_up_galaxies/README.md create mode 100644 scripts/chapter_4_scaling_up_galaxies/README.md diff --git a/notebooks/README.md b/notebooks/README.md index dc57797..c694177 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -14,7 +14,7 @@ Welcome to **HowToGalaxy** - The **PyAutoGalaxy** tutorial! # HOW TO TACKLE HowToGalaxy -The **HowToGalaxy** lecture series current sits at 4 chapters, and each will take a day or so to go through +The **HowToGalaxy** lecture series currently sits at 4 chapters, and each will take a day or so to go through properly. You probably want to be modeling galaxies faster than that! Furthermore, the concepts in the later chapters are pretty challenging, and familiarity and modeling is desirable before you tackle them. @@ -85,31 +85,32 @@ fit CCD imaging data of a galaxy with a model. At the end, you'll understand: 2. How to fit a model to galaxy CCD imaging via a non-linear search. 3. The trade-off between realism and complexity when choosing a model. 4. Why an incorrect model may be inferred and how to prevent this from happening. -5. The challenges that are involved in inferred a robust model in a computationally reasonable run-time. +5. The challenges that are involved in inferring a robust model in a computationally reasonable run-time. +6. How to chain non-linear searches together to build automated modeling pipelines with prior passing. **Once completed, you'll be ready to model your own galaxies with PyAutoGalaxy!** # OVERVIEW OF CHAPTER 3 (Intermediate) -**Automated Modeling with non-linear search chaining** +**Using an inversion to perform a pixelized morphology reconstruction** -In chapter 3, we'll learn how to chain multiple non-linear searches together to build automated modeling pipelines -which can: - -1. Break-down the fitting of a model using multiple non-linear searches and prior passing. -2. Use a custom pipeline to fit data containing multiple galaxy where each galaxy is fitted one at a time. -3. Fit the global structure of a galaxy, followed by faint morphological features like a bar. - -# OVERVIEW OF CHAPTER 4 (Intermediate) - -**Using an inverison to perform a pixelized morphology reconstructions** - -In chapter 4, we'll learn how to reconstruct morphology features of a galaxy using a pixel-grid, ensuring that we can -fit an accurate model to sources with complex and irregular morphologies. You'll learn how to: +In chapter 3, we'll learn how to reconstruct the morphological features of a galaxy using a pixel-grid, ensuring +that we can fit an accurate model to galaxies with complex and irregular morphologies. You'll learn how to: 1. Pixelize a galaxy reconstruction into pixels. 2. Perform a linear inversion using this pixelization to reconstruct the galaxy's light. 3. Apply a smoothness prior on the galaxy reconstruction, called regularization. 4. Apply smoothing within a Bayesian framework to objectively quantify the reconstruction's complexity. -5. Use alternative pixelizations, for example a Voronoi mesh. +5. Write down the linear algebra and Bayesian evidence equations that underpin the whole framework. 6. Use these features to fit a model via non-linear searches. + +# OVERVIEW OF CHAPTER 4 (Advanced) + +**Scaling Up Galaxies** + +In chapter 4, we'll scale galaxy modeling up beyond a single galaxy, learning how to: + +1. Handle extra galaxies near the one being fitted, by scaling their light out of the fit or modeling them explicitly. +2. Model two or more blended galaxies simultaneously and understand the degeneracies this creates. +3. Model cluster fields, where the brightest cluster galaxy is fitted richly and the member population is + composed from a catalogue. diff --git a/notebooks/chapter_2_modeling/README.md b/notebooks/chapter_2_modeling/README.md index 1526e61..75440ca 100644 --- a/notebooks/chapter_2_modeling/README.md +++ b/notebooks/chapter_2_modeling/README.md @@ -1,4 +1,5 @@ -In chapter 2, we'll take you through how to model galaxies using a non-linear search. +In chapter 2, we'll take you through how to model galaxies using a non-linear search, ending with how +searches are chained together into automated modeling pipelines. **Colab** links to every tutorial are included. @@ -19,3 +20,7 @@ In chapter 2, we'll take you through how to model galaxies using a non-linear se - [Tutorial 7: Results](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_2_modeling/tutorial_7_results.ipynb) — Overview of the results available after successfully fitting a model. - [Tutorial 8: Need for Speed](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_2_modeling/tutorial_8_need_for_speed.ipynb) — How to fit complex models whilst balancing efficiency and run-time. + +- [Tutorial 9: Search Chaining](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb) — Breaking the modeling procedure into a chained sequence of model-fits. + +- [Tutorial 10: Prior Passing](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb) — How the results of earlier searches are passed to later searches. diff --git a/notebooks/chapter_3_pixelizations/README.md b/notebooks/chapter_3_pixelizations/README.md index 97c974d..ed3a902 100644 --- a/notebooks/chapter_3_pixelizations/README.md +++ b/notebooks/chapter_3_pixelizations/README.md @@ -1,15 +1,17 @@ -In chapter 4, we use **Pixelizations** to reconstruct complex source galaxies on pixelized grids. +In chapter 3, we use **Pixelizations** to reconstruct the light of complex galaxies on pixelized grids. **Colab** links to every tutorial are included. # Files -- [Tutorial 1: Pixelizations](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_1_pixelizations.ipynb) — Creating a pixel-grid in the source-plane. +- [Tutorial 1: Pixelizations](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb) — Creating a pixel-grid to reconstruct a galaxy's light. -- [Tutorial 2: Mappers](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_2_mappers.ipynb) — How a pixelization maps source-pixels to image-pixels. +- [Tutorial 2: Mappers](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb) — How a pixelization maps its pixels to image-pixels. -- [Tutorial 3: Inversions](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_3_inversions.ipynb) — Inverting the mappings to reconstruct the source's light. +- [Tutorial 3: Inversions](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb) — Inverting the mappings to reconstruct the galaxy's light. -- [Tutorial 4: Bayesian Regularization](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_4_bayesian_regularization.ipynb) — Smoothing the source within a Bayesian framework. +- [Tutorial 4: Bayesian Regularization](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb) — Smoothing the reconstruction within a Bayesian framework. -- [Tutorial 5: Model Fit](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_pixelizations/tutorial_5_model_fit.ipynb) — An example modeling pipeline which uses an inversion. +- [Tutorial 5: Bayesian Formalism](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_5_bayesian_formalism.ipynb) — The linear algebra and Bayesian evidence equations behind pixelized galaxy reconstruction. + +- [Tutorial 6: Model Fit](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb) — An example modeling pipeline which uses an inversion. diff --git a/notebooks/chapter_4_scaling_up_galaxies/README.md b/notebooks/chapter_4_scaling_up_galaxies/README.md new file mode 100644 index 0000000..075fe28 --- /dev/null +++ b/notebooks/chapter_4_scaling_up_galaxies/README.md @@ -0,0 +1,12 @@ +In chapter 4, we scale galaxy modeling up beyond a single galaxy: extra galaxies, blended multi-galaxy +systems, and cluster fields. + +**Colab** links to every tutorial are included. + +# Files + +- [Tutorial 1: Extra Galaxies](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.ipynb) — Handling nearby galaxies by scaling their light out of the fit or modeling them explicitly. + +- [Tutorial 2: Multi Galaxy](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.ipynb) — Modeling two or more blended galaxies simultaneously and the degeneracies this creates. + +- [Tutorial 3: Cluster](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_scaling_up_galaxies/tutorial_3_cluster.ipynb) — Modeling a cluster field's BCG and catalogue-driven member population. diff --git a/scripts/README.md b/scripts/README.md index dc57797..c694177 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -14,7 +14,7 @@ Welcome to **HowToGalaxy** - The **PyAutoGalaxy** tutorial! # HOW TO TACKLE HowToGalaxy -The **HowToGalaxy** lecture series current sits at 4 chapters, and each will take a day or so to go through +The **HowToGalaxy** lecture series currently sits at 4 chapters, and each will take a day or so to go through properly. You probably want to be modeling galaxies faster than that! Furthermore, the concepts in the later chapters are pretty challenging, and familiarity and modeling is desirable before you tackle them. @@ -85,31 +85,32 @@ fit CCD imaging data of a galaxy with a model. At the end, you'll understand: 2. How to fit a model to galaxy CCD imaging via a non-linear search. 3. The trade-off between realism and complexity when choosing a model. 4. Why an incorrect model may be inferred and how to prevent this from happening. -5. The challenges that are involved in inferred a robust model in a computationally reasonable run-time. +5. The challenges that are involved in inferring a robust model in a computationally reasonable run-time. +6. How to chain non-linear searches together to build automated modeling pipelines with prior passing. **Once completed, you'll be ready to model your own galaxies with PyAutoGalaxy!** # OVERVIEW OF CHAPTER 3 (Intermediate) -**Automated Modeling with non-linear search chaining** +**Using an inversion to perform a pixelized morphology reconstruction** -In chapter 3, we'll learn how to chain multiple non-linear searches together to build automated modeling pipelines -which can: - -1. Break-down the fitting of a model using multiple non-linear searches and prior passing. -2. Use a custom pipeline to fit data containing multiple galaxy where each galaxy is fitted one at a time. -3. Fit the global structure of a galaxy, followed by faint morphological features like a bar. - -# OVERVIEW OF CHAPTER 4 (Intermediate) - -**Using an inverison to perform a pixelized morphology reconstructions** - -In chapter 4, we'll learn how to reconstruct morphology features of a galaxy using a pixel-grid, ensuring that we can -fit an accurate model to sources with complex and irregular morphologies. You'll learn how to: +In chapter 3, we'll learn how to reconstruct the morphological features of a galaxy using a pixel-grid, ensuring +that we can fit an accurate model to galaxies with complex and irregular morphologies. You'll learn how to: 1. Pixelize a galaxy reconstruction into pixels. 2. Perform a linear inversion using this pixelization to reconstruct the galaxy's light. 3. Apply a smoothness prior on the galaxy reconstruction, called regularization. 4. Apply smoothing within a Bayesian framework to objectively quantify the reconstruction's complexity. -5. Use alternative pixelizations, for example a Voronoi mesh. +5. Write down the linear algebra and Bayesian evidence equations that underpin the whole framework. 6. Use these features to fit a model via non-linear searches. + +# OVERVIEW OF CHAPTER 4 (Advanced) + +**Scaling Up Galaxies** + +In chapter 4, we'll scale galaxy modeling up beyond a single galaxy, learning how to: + +1. Handle extra galaxies near the one being fitted, by scaling their light out of the fit or modeling them explicitly. +2. Model two or more blended galaxies simultaneously and understand the degeneracies this creates. +3. Model cluster fields, where the brightest cluster galaxy is fitted richly and the member population is + composed from a catalogue. diff --git a/scripts/chapter_2_modeling/README.md b/scripts/chapter_2_modeling/README.md index 1526e61..75440ca 100644 --- a/scripts/chapter_2_modeling/README.md +++ b/scripts/chapter_2_modeling/README.md @@ -1,4 +1,5 @@ -In chapter 2, we'll take you through how to model galaxies using a non-linear search. +In chapter 2, we'll take you through how to model galaxies using a non-linear search, ending with how +searches are chained together into automated modeling pipelines. **Colab** links to every tutorial are included. @@ -19,3 +20,7 @@ In chapter 2, we'll take you through how to model galaxies using a non-linear se - [Tutorial 7: Results](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_2_modeling/tutorial_7_results.ipynb) — Overview of the results available after successfully fitting a model. - [Tutorial 8: Need for Speed](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_2_modeling/tutorial_8_need_for_speed.ipynb) — How to fit complex models whilst balancing efficiency and run-time. + +- [Tutorial 9: Search Chaining](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb) — Breaking the modeling procedure into a chained sequence of model-fits. + +- [Tutorial 10: Prior Passing](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb) — How the results of earlier searches are passed to later searches. diff --git a/scripts/chapter_4_scaling_up_galaxies/README.md b/scripts/chapter_4_scaling_up_galaxies/README.md new file mode 100644 index 0000000..075fe28 --- /dev/null +++ b/scripts/chapter_4_scaling_up_galaxies/README.md @@ -0,0 +1,12 @@ +In chapter 4, we scale galaxy modeling up beyond a single galaxy: extra galaxies, blended multi-galaxy +systems, and cluster fields. + +**Colab** links to every tutorial are included. + +# Files + +- [Tutorial 1: Extra Galaxies](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.ipynb) — Handling nearby galaxies by scaling their light out of the fit or modeling them explicitly. + +- [Tutorial 2: Multi Galaxy](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.ipynb) — Modeling two or more blended galaxies simultaneously and the degeneracies this creates. + +- [Tutorial 3: Cluster](https://colab.research.google.com/github/PyAutoLabs/HowToGalaxy/blob/2026.8.4.1/notebooks/chapter_4_scaling_up_galaxies/tutorial_3_cluster.ipynb) — Modeling a cluster field's BCG and catalogue-driven member population. From 4e0f539c04fc1b87bb07f159df57890673978850 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:48:21 +0000 Subject: [PATCH 11/15] docs: update llms.txt learning path to the new chapter map Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- llms.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llms.txt b/llms.txt index 0dcbc88..536f0ee 100644 --- a/llms.txt +++ b/llms.txt @@ -26,8 +26,8 @@ real analysis?" below). - **Chapter 1 — Introduction** → [scripts/chapter_1_introduction/](scripts/chapter_1_introduction/): Galaxies from first principles: grids & galaxies, light profiles, simulating data, and fitting data. The conceptual foundation. - **Chapter 2 — Modeling** → [scripts/chapter_2_modeling/](scripts/chapter_2_modeling/): Fitting a galaxy model to data with a non-linear search — parameter spaces, priors, dealing with failure, linear light profiles, masking, and reading results. -- **Chapter 3 — Search Chaining** → [scripts/chapter_3_search_chaining/](scripts/chapter_3_search_chaining/): Breaking a hard fit into a sequence of searches (prior passing) and modeling multi-galaxy images. -- **Chapter 4 — Pixelizations** → [scripts/chapter_4_pixelizations/](scripts/chapter_4_pixelizations/): Reconstructing irregular galaxy structure on a pixel-grid instead of with light profiles — mappers, inversions, Bayesian regularization. +- **Chapter 3 — Pixelizations** → [scripts/chapter_3_pixelizations/](scripts/chapter_3_pixelizations/): Reconstructing irregular galaxy structure on a pixel-grid instead of with light profiles — mappers, inversions, Bayesian regularization, the Bayesian formalism. +- **Chapter 4 — Scaling Up Galaxies** → [scripts/chapter_4_scaling_up_galaxies/](scripts/chapter_4_scaling_up_galaxies/): Beyond a single galaxy — extra galaxies, blended multi-galaxy systems, and cluster fields. ## I want to understand… @@ -37,8 +37,8 @@ real analysis?" below). - **How does a non-linear search find a model?** → [scripts/chapter_2_modeling/tutorial_1_non_linear_search.py](scripts/chapter_2_modeling/tutorial_1_non_linear_search.py): Parameter space, priors, MLE / MCMC / nested sampling. - **My fit found a wrong (local-maxima) solution — what do I do?** → [scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py](scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py): Prior tuning, reducing complexity, and looking harder. - **What are linear light profiles / MGE?** → [scripts/chapter_2_modeling/tutorial_5_linear_profiles.py](scripts/chapter_2_modeling/tutorial_5_linear_profiles.py): Solving intensities linearly; the Multi-Gaussian Expansion basis. -- **How do I model an image with two galaxies?** → [scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py](scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py): Search chaining to handle multi-galaxy degeneracies. -- **How do I reconstruct irregular structure on a pixel grid?** → [scripts/chapter_4_pixelizations/tutorial_1_pixelizations.py](scripts/chapter_4_pixelizations/tutorial_1_pixelizations.py) through [tutorial_4_bayesian_regularization.py](scripts/chapter_4_pixelizations/tutorial_4_bayesian_regularization.py). +- **How do I model an image with two galaxies?** → [scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py](scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py): Joint fitting of blended galaxies and their degeneracies. +- **How do I reconstruct irregular structure on a pixel grid?** → [scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py](scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py) through [tutorial_4_bayesian_regularization.py](scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py). ## Ready for real analysis? From d3304b8428068c8a8f686d10f6f75967203aff6e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:59:19 +0000 Subject: [PATCH 12/15] =?UTF-8?q?docs:=20chapters=203-4=20review=20fixes?= =?UTF-8?q?=20=E2=80=94=20cluster=20geometry,=20claims=20and=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review: the cluster member that sat outside the tutorial's own mask moves inside it (simulator + prose aligned, dataset re-verified end-to-end); per-member parameter arithmetic corrected (5 per free SersicSph); inversions tutorial reordered so the file ends on its wrap up; contents/section bijections restored across chapters 3-4; the formalism lecture's contents mirror its section markers; narrative back-references added where tutorial 2 re-taught tutorial 1's fixed-centres rationale; stale workspace boilerplate and wrong-repo dataset paths cleaned from the simulators. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- .../tutorial_3_inversions.py | 22 +++++++++---------- .../tutorial_5_bayesian_formalism.py | 8 +++---- .../tutorial_1_extra_galaxies.py | 5 +++-- .../tutorial_2_multi_galaxy.py | 6 +++-- .../tutorial_3_cluster.py | 19 +++++++++------- scripts/simulators/cluster.py | 2 +- scripts/simulators/sersic.py | 2 +- scripts/simulators/sersic_x2.py | 7 +----- scripts/simulators/simple.py | 10 ++++----- 9 files changed, 41 insertions(+), 40 deletions(-) diff --git a/scripts/chapter_3_pixelizations/tutorial_3_inversions.py b/scripts/chapter_3_pixelizations/tutorial_3_inversions.py index 3e04ebe..0bfb324 100644 --- a/scripts/chapter_3_pixelizations/tutorial_3_inversions.py +++ b/scripts/chapter_3_pixelizations/tutorial_3_inversions.py @@ -16,8 +16,8 @@ - **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. - **Pixelization:** Create a pixelization and perform an inversion to reconstruct the galaxy. - **Positive Only Solver:** Ensure the reconstruction has only positive intensity values. +- **Detailed Explanation:** Where the linear algebra behind inversions is derived in full. - **Wrap Up:** Summary of inversion concepts. -- **Detailed Explanation:** In-depth explanation of the linear algebra behind inversions. """ # from autogalaxy import setup_notebook; setup_notebook() @@ -209,21 +209,21 @@ Other methods in the literature often do not use a positive only solver, and therefore suffer from these unphysical solutions, which can degrade the results of galaxy models in general. +__Detailed Explanation__ + +The linear algebra sketched above -- setting up the mappings as a matrix and solving for the pixelization pixel +fluxes -- is derived in full in tutorial 5 of this chapter, where we build every matrix by hand and perform the +solve ourselves. The file `autogalaxy_workspace/*/imaging/features/pixelization/likelihood_function.ipynb` gives a +further visual step-by-step guide of the process alongside equations and references to literature on the subject. + __Wrap Up__ And, we're done, here are a few questions to get you thinking about inversions: - 1) The inversion provides the maximum log likelihood solution to the observed image. Is there a problem with seeking - the highest likelihood solution? Is there a risk that we're going to fit other things in the image than just the + 1) The inversion provides the maximum log likelihood solution to the observed image. Is there a problem with seeking + the highest likelihood solution? Is there a risk that we're going to fit other things in the image than just the galaxy? What happens if you reduce the `coefficient` of the regularization object above to zero? - 2) The exterior pixels in the rectangular pixel-grid have no image-pixels in them. However, they are still given a + 2) The exterior pixels in the rectangular pixel-grid have no image-pixels in them. However, they are still given a reconstructed flux. Given these pixels do not map to the data, where is this value coming from? - -__Detailed Explanation__ - -The linear algebra sketched above -- setting up the mappings as a matrix and solving for the pixelization pixel -fluxes -- is derived in full in tutorial 5 of this chapter, where we build every matrix by hand and perform the -solve ourselves. The file `autogalaxy_workspace/*/imaging/features/pixelization/likelihood_function.ipynb` gives a -further visual step-by-step guide of the process alongside equations and references to literature on the subject. """ diff --git a/scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py b/scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py index b2dd726..005819c 100644 --- a/scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py +++ b/scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py @@ -37,11 +37,11 @@ - **Mapper:** Package the interpolation into a `Mapper` describing all image-to-mesh mappings. - **Mapping Matrix:** Express the mappings as the 2D matrix $f$. - **Blurred Mapping Matrix:** Convolve every column of $f$ with the imaging PSF. -- **Data Vector:** Compute the data vector $D$ from the blurred mapping matrix, data and noise-map. -- **Curvature Matrix:** Compute the curvature matrix $F$. +- **Data Vector (D):** Compute the data vector $D$ from the blurred mapping matrix, data and noise-map. +- **Curvature Matrix (F):** Compute the curvature matrix $F$. - **Unregularized Solve:** Solve $s = F^{-1} D$ and see the over-fitted mess this produces. -- **Regularization Matrix:** Compute the regularization matrix $H$ encoding the smoothness prior. -- **Galaxy Reconstruction:** Solve the regularized system $s = [F + H]^{-1} D$. +- **Regularization Matrix (H):** Compute the regularization matrix $H$ encoding the smoothness prior. +- **Galaxy Reconstruction (s):** Solve the regularized system $s = [F + H]^{-1} D$. - **Image Reconstruction:** Map the reconstruction back to image resolution via the blurred mapping matrix. - **Likelihood Function:** The five terms which combine into the log evidence. - **Chi Squared:** The goodness-of-fit of the reconstructed image to the data. diff --git a/scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py b/scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py index 1548559..5707215 100644 --- a/scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py +++ b/scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py @@ -201,8 +201,9 @@ """ __Noise Scaling Fit__ -We now fit this noise-scaled dataset with a model that does **not** include the extra galaxy. The model is the -same one we used in chapter 2's linear profiles tutorial: a linear `Sersic` bulge. +We now fit this noise-scaled dataset with a model that does **not** include the extra galaxy. The model is a +single linear `Sersic` bulge, using the linear light profiles introduced in chapter 2's linear profiles +tutorial. """ bulge = af.Model(ag.lp_linear.Sersic) diff --git a/scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py b/scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py index ace7e27..2666b94 100644 --- a/scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py +++ b/scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py @@ -31,7 +31,8 @@ __Contents__ -- **Initial Setup:** Load imaging of two blended galaxies, simulating it first if it is not on disk. +- **Initial Setup:** Load imaging of two blended galaxies. +- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. - **Mask:** Define a mask which encloses the light of both galaxies. - **Over Sampling:** Centre the adaptive over sampling grid on every galaxy, not just one. - **Why Not Fit Them Separately?:** Why masking out one galaxy and fitting the other biases both measurements. @@ -235,7 +236,8 @@ __Fixing the Centres__ Before fitting, we make one change that is standard practice for blended systems: we **fix each galaxy's -centre to its observed light centre**, removing 2 free parameters per galaxy. +centre to its observed light centre**, removing 2 free parameters per galaxy. The previous tutorial fixed the +interloper's centre for the same reason; in a blend the case is even stronger. Why? With a single galaxy, the data constrains the centre extremely well: it is simply the peak of the light, and there is only one plausible culprit for every photon. With blended light this breaks down. Each pixel's diff --git a/scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py b/scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py index dbef76f..1a3b018 100644 --- a/scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py +++ b/scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py @@ -41,8 +41,9 @@ The previous tutorials composed models the way we have all chapter: one `af.Model(ag.Galaxy)` per galaxy, each with its own free light profile. A blend of two or three galaxies is fine. A cluster is not: -- **Parameters**: ten members with a free spherical Sersic each is 40 free parameters before we even touch - the BCG; a hundred members is 400. No non-linear search can sample such a space reliably. +- **Parameters**: ten members with a free spherical Sersic each (5 parameters per galaxy) is 50 free + parameters before we even touch the BCG; a hundred members is 500. No non-linear search can sample such a + space reliably. - **Information**: the faint members do not contain enough signal to constrain four free parameters each — most of those dimensions would be unconstrained noise. @@ -60,7 +61,9 @@ __Contents__ -- **Dataset:** Load the simulated cluster field (1 BCG + 10 members), auto-simulating it if absent. +- **The Scaling Problem:** Why one-free-model-per-galaxy cannot scale to a cluster's member population. +- **Dataset:** Load the simulated cluster field (1 BCG + 10 members). +- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist. - **Member Catalogue:** Load the member centres and luminosities from `scaling_galaxies.csv`. - **Masking:** Mask the wide cluster field and over-sample every galaxy's centre. - **Model:** Compose the two-tier cluster model — free BCG MGE + catalogue-driven member tier. @@ -120,7 +123,7 @@ the centre, with the fainter members scattered across the frame. In real data the diffuse intra-cluster light would fill the space between them. -Note the field of view compared to earlier chapters — the members sit up to ~8" from the centre, so the +Note the field of view compared to earlier chapters — the members sit up to ~10" from the centre, so the image is far larger than the ~6" cutouts we fitted before. """ aplt.subplot_imaging_dataset(dataset=dataset) @@ -214,8 +217,8 @@ intensity_i = intensity_scale * luminosity_i The single `intensity_scale` prior is defined once, outside the loop, and every member's intensity is an -arithmetic expression of it — the prior-linking arithmetic we have used throughout the lectures, now doing -population-scale work. The whole 10-member tier therefore contributes ONE free parameter to the non-linear +arithmetic expression of it — an extension of the prior linking we used in earlier chapters (pairing two +parameters with `=`, e.g. `bulge.centre = disk.centre`), now doing population-scale work. The whole 10-member tier therefore contributes ONE free parameter to the non-linear search, and would still contribute one with 200 members. Physically, `intensity_scale` is the conversion between the catalogue's luminosity units and the image's @@ -372,8 +375,8 @@ - **One galaxy** (chapters 1-3): light profiles, non-linear searches, linear profiles and the MGE, and pixelized reconstructions — the core toolkit, applied to a single galaxy at the centre of its image. -- **Extra galaxies**: nearby galaxies whose light contaminates the target's, included in the model so the - target's photometry is unbiased. +- **Extra galaxies**: nearby galaxies whose light contaminates the target's, noise-scaled out of the fit or + included in the model so the target's photometry is unbiased. - **Multi-galaxy blends**: systems of overlapping galaxies modeled simultaneously, each with its own free light model. diff --git a/scripts/simulators/cluster.py b/scripts/simulators/cluster.py index 688d641..39bc94b 100644 --- a/scripts/simulators/cluster.py +++ b/scripts/simulators/cluster.py @@ -70,7 +70,7 @@ (-6.5, -8.0), (-2.5, 6.5), (7.0, -2.0), - (-8.0, 8.5), + (-7.0, 7.5), (2.0, -8.5), (-4.0, -3.5), ] diff --git a/scripts/simulators/sersic.py b/scripts/simulators/sersic.py index ae392b1..95b2c45 100644 --- a/scripts/simulators/sersic.py +++ b/scripts/simulators/sersic.py @@ -147,5 +147,5 @@ ) """ -The dataset can be viewed in the folder `autogalaxy_workspace/dataset/imaging/simple__sersic`. +The dataset can be viewed in the folder `dataset/imaging/simple__sersic`. """ diff --git a/scripts/simulators/sersic_x2.py b/scripts/simulators/sersic_x2.py index 21eab79..38bb9e2 100644 --- a/scripts/simulators/sersic_x2.py +++ b/scripts/simulators/sersic_x2.py @@ -18,11 +18,6 @@ - **Visualize:** Output subplot and image PNGs of the simulated dataset. - **Plane Output:** Save the Galaxies object as a JSON file. -__Advanced__ - -This is an advanced simulator script, meaning that detailed explanations of certain code are omitted. Refer to -simulators not in the `advanced` folder for more detailed comments. - __Start Here Notebook__ If any code in this script is unclear, refer to the `simulators/simple.ipynb` notebook. @@ -166,5 +161,5 @@ ) """ -The dataset can be viewed in the folder `autogalaxy_workspace/dataset/imaging/sersic_x2`. +The dataset can be viewed in the folder `dataset/imaging/sersic_x2`. """ diff --git a/scripts/simulators/simple.py b/scripts/simulators/simple.py index 8c96b87..4a1e12a 100644 --- a/scripts/simulators/simple.py +++ b/scripts/simulators/simple.py @@ -29,9 +29,9 @@ The `dataset_type` describes the type of data being simulated and `dataset_name` gives it a descriptive name. They define the folder the dataset is output to on your hard-disk: - - The image will be output to `/autogalaxy_workspace/dataset/dataset_type/dataset_name/image.fits`. - - The noise-map will be output to `/autogalaxy_workspace/dataset/dataset_type/dataset_name/noise_map.fits`. - - The psf will be output to `/autogalaxy_workspace/dataset/dataset_type/dataset_name/psf.fits`. + - The image will be output to `/dataset/dataset_type/dataset_name/image.fits`. + - The noise-map will be output to `/dataset/dataset_type/dataset_name/noise_map.fits`. + - The psf will be output to `/dataset/dataset_type/dataset_name/psf.fits`. """ dataset_type = "imaging" dataset_name = "simple" @@ -39,7 +39,7 @@ """ The path where the dataset will be output. -In this example, this is: `/autogalaxy_workspace/dataset/imaging/simple` +In this example, this is: `/dataset/imaging/simple` """ dataset_path = Path("dataset", dataset_type, dataset_name) @@ -185,7 +185,7 @@ ) """ -The dataset can be viewed in the folder `autogalaxy_workspace/dataset/imaging/simple`. +The dataset can be viewed in the folder `dataset/imaging/simple`. __JAX Variant__ From 732d88d1373e5560b3defd0ffec7fecc63d64e93 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 15:07:38 +0000 Subject: [PATCH 13/15] =?UTF-8?q?docs:=20chapters=201-2=20review=20fixes?= =?UTF-8?q?=20=E2=80=94=20titles,=20lensing=20leaks,=20stack=20claims?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review of the mirrored chapters: leftover HowToLens titles fixed (summary and results tutorials); a results-tutorial model component literally named mass with a light profile becomes bulge; lensing vocabulary scrubbed from prose across both chapters; false stack claims corrected against the installed configs (Nautilus n_live default, prior widths and limits, plot-function signatures, removed references to deleted MatPlot2D/Visuals2D objects); Contents lists match bodies; parameter counts match composed models; ~45 typos fixed; chapter_optional path_prefixes and dataset descriptions corrected; the chapter 2 summary file converts to LF and points at chapter 4. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- scripts/chapter_1_introduction/README.md | 2 +- .../tutorial_0_visualization.py | 39 +++++------ .../tutorial_1_grids_and_galaxies.py | 44 ++++++------- .../chapter_1_introduction/tutorial_2_data.py | 4 +- .../tutorial_3_fitting.py | 23 +++---- .../tutorial_5_summary.py | 34 +++++----- .../tutorial_10_prior_passing.py | 35 +++++----- .../chapter_2_modeling/tutorial_11_summary | 39 +++++------ .../tutorial_1_non_linear_search.py | 34 +++++----- .../tutorial_2_practicalities.py | 44 ++++++------- .../tutorial_3_realism_and_complexity.py | 24 +++---- .../tutorial_4_dealing_with_failure.py | 43 ++++++------ .../tutorial_5_linear_profiles.py | 65 +++++++++---------- .../chapter_2_modeling/tutorial_6_masking.py | 22 +++---- .../chapter_2_modeling/tutorial_7_results.py | 18 ++--- .../tutorial_9_search_chaining.py | 10 +-- scripts/chapter_optional/tutorial_searches.py | 40 ++++++------ 17 files changed, 249 insertions(+), 271 deletions(-) diff --git a/scripts/chapter_1_introduction/README.md b/scripts/chapter_1_introduction/README.md index 8869dad..debdb91 100644 --- a/scripts/chapter_1_introduction/README.md +++ b/scripts/chapter_1_introduction/README.md @@ -1,4 +1,4 @@ -In chapter 1, we introduce you to strong gravitational lensing and the core **PyAutoGalaxy** API. +In chapter 1, we introduce you to galaxies, the light they emit and the core **PyAutoGalaxy** API. **Colab** links to every tutorial are included. diff --git a/scripts/chapter_1_introduction/tutorial_0_visualization.py b/scripts/chapter_1_introduction/tutorial_0_visualization.py index 24a6c3f..9245e5a 100644 --- a/scripts/chapter_1_introduction/tutorial_0_visualization.py +++ b/scripts/chapter_1_introduction/tutorial_0_visualization.py @@ -78,12 +78,12 @@ Does the figure display correctly on your computer screen? -If not, you can customize common matplotlib options by passing them directly to `plot_array`: +If not, you can customize common plot options by passing them directly to `plot_array`: - `title=`: Set the figure title. - - `figsize=`: Control the figure size as a `(width, height)` tuple. - `colormap=`: Set the matplotlib colormap name (e.g. `"jet"`, `"gray"`). - - `xlabel=`, `ylabel=`: Override the default axis labels. + - `use_log10=`: Plot the image on a log10 scale, revealing faint features. + - `vmin=`, `vmax=`: Override the minimum and maximum values of the colormap. """ aplt.plot_array( array=dataset.data, @@ -91,32 +91,30 @@ ) """ -Many matplotlib options can be customized, but for now we're only concerned with making sure figures display clear in +Many plot options can be customized, but for now we're only concerned with making sure figures display clearly in your Jupyter Notebooks. Nevertheless, a comprehensive API reference guide of all available plot arguments can be found in the `autogalaxy_workspace/*/guides/plot` package. You should check this out once you are more familiar with **PyAutoGalaxy**. -Ideally, we would not specify a `figsize` every time we plot an image. Fortunately, default values can be fully +Ideally, we would not specify these options every time we plot an image. Fortunately, default values can be fully customized via the config files. Checkout the `general.yaml` file in `autogalaxy_workspace/config/visualize`. -All default matplotlib values are here. There are a lot of entries, so lets focus on whats important for displaying -figures: +All default visualization values are here. There are a lot of entries, so lets focus on whats important for +displaying figures: - - mat_wrap.yaml -> Figure -> figure: -> figsize - - mat_wrap.yaml -> YLabel -> figure: -> fontsize - - mat_wrap.yaml -> XLabel -> figure: -> fontsize - - mat_wrap.yaml -> TickParams -> figure: -> labelsize - - mat_wrap.yaml -> YTicks -> figure: -> labelsize - - mat_wrap.yaml -> XTicks -> figure: -> labelsize + - general.yaml -> general -> backend: the matplotlib backend used for visualization. + - general.yaml -> colormap: the default colormap of 2D plots. + - general.yaml -> subplot_shape_to_figsize_factor: controls the size of subplot figures. + - general.yaml -> colorbar: the font sizes of colorbar labels. Don't worry about all the other files or options listed for now, as they'll make a lot more sense once you are familiar with **PyAutoGalaxy**. If you had to change any of the above settings to get the figures to display clearly, you should update their values in the corresponding config files above (you will need to reset your Jupyter notebook server for these changes to -take effect, so make sure you have the right values using the `figsize` argument in the cell above beforehand!). +take effect, so make sure you have the right values using the arguments in the cell above beforehand!). __Subplots__ @@ -130,11 +128,12 @@ """ __Visuals__ -Visuals can be added to any figure by passing them as keyword arguments directly to `plot_array`. +Visuals can be added to any figure by passing them as keyword arguments directly to `plot_array`, for +example `grid=`, `positions=` and `lines=`. The `autogalaxy_workspace/*/guides/plot` examples illustrate every +overlay argument. -For example, we can plot a mask on the image above by passing `mask=mask`. - -The `visuals` example illustrates every overlay argument, for example `mask=`, `grid=`, `positions=`, `lines=`, etc. +Some visuals appear automatically. For example, if we apply a mask to the dataset, figures of it zoom into the +unmasked region and only show the unmasked data — as shown by the plot below, which applies an annular mask. """ mask = ag.Mask2D.circular_annular( shape_native=dataset.shape_native, @@ -143,7 +142,9 @@ outer_radius=3.0, ) -aplt.plot_array(array=dataset.data, title="Data") +dataset = dataset.apply_mask(mask=mask) + +aplt.plot_array(array=dataset.data, title="Data With Annular Mask") """ __Wrap Up__ diff --git a/scripts/chapter_1_introduction/tutorial_1_grids_and_galaxies.py b/scripts/chapter_1_introduction/tutorial_1_grids_and_galaxies.py index 2451b64..5a81a3c 100644 --- a/scripts/chapter_1_introduction/tutorial_1_grids_and_galaxies.py +++ b/scripts/chapter_1_introduction/tutorial_1_grids_and_galaxies.py @@ -6,7 +6,7 @@ ellipticals, spirals and irregulars. He produced a diagram of these galaxies, called the Hubble Tuning Fork, which is shown below and still discussed by astronomers in the modern day: -![HubbleTuning](https://github.com/PyAutoLabs/autogalaxy_workspace/blob/main/scripts/chapter_1_introduction/HubbleTuningFork.jpg) +![HubbleTuning](https://github.com/PyAutoLabs/HowToGalaxy/blob/main/scripts/chapter_1_introduction/HubbleTuningFork.jpg) To make his diagram, Hubble looked at images of each galaxy in his sample, and subjectively judged by eye how to classify it. Today, Astronomers use computer software, statistical algorithms and image processing techniques to @@ -34,12 +34,12 @@ - **Grids:** Create a uniform grid of (y,x) coordinates and show how it can be used to measure the light of a galaxy. - **Geometry:** How to shift and rotate a grid, and convert it to elliptical coordinates. - **Light Profiles:** Using light profiles, analytic functions that describe how a galaxy's light is distributed. -- **Galaxies:** Creating galaxies containing light profiles and computing the image of a galaxy. - **One Dimension Projection:** Create projected 2D radial grids for 1D profile calculations. +- **Galaxies:** Creating galaxies containing light profiles and computing the image of a galaxy. - **Unit Conversion:** Converting angular distances to physical distances using cosmology. - **Wrap Up:** Summary of the key concepts covered in this tutorial. -The imports below are required to run the howtogalaxy tutorials in a Jupiter notebook. They also import the +The imports below are required to run the howtogalaxy tutorials in a Jupyter notebook. They also import the `autogalaxy` package and the `autogalaxy.plot` module which are used throughout the tutorials. """ @@ -130,7 +130,7 @@ We can shift the grid to a new center, (y_c, x_c), by subtracting this center from each coordinate. """ -centre = (0.3, 0.5) # Shifting the grid to be centered at y=1.0", x=2.0". +centre = (0.3, 0.5) # Shifting the grid to be centered at y=0.3", x=0.5". grid_shifted = grid grid_shifted[:, 0] = grid_shifted[:, 0] - centre[0] # Shift in y-direction. @@ -351,10 +351,10 @@ """ __One Dimension Projection__ -We often want to calculative 1D quantities of a light profile, for example to plot how its light changes as +We often want to calculate 1D quantities of a light profile, for example to plot how its light changes as a function of radius. -To do this, we must still input a 2D grid into the `image_2d_from` method, therefore we create a project 2D +To do this, we must still input a 2D grid into the `image_2d_from` method, therefore we create a projected 2D radial grid as follows which has shape [Number_of_1d_coordinates, 2] and where all [:,0] entries are the same. A simple example of such a grid is as follows with 4 1D coordinates is: @@ -372,14 +372,10 @@ As in this example, we often already have a 2D grid we are using to calculate images of a light profile and it would be convenient to simply create `grid_2d_projected` from that. -For example, we may want the project grid which traces it major axis in uniform radial steps. +For example, we may want the projected grid which traces the light profile's major axis in uniform radial steps. -This is easily computed using the `grid_2d_radial_project_from` function and passing the `centre` and `angle` -of a light profile we can make it align with the light profile itself. - -Note how in this example the two galaxy bulges are not rotationally aligned but we aligned the projected -grid with the first galaxy. The centres are aligned, but if they were not that would cause similar -issues. +This is easily computed using the `grid_2d_radial_projected_from` function: by passing the `centre` and `angle` +of a light profile we can make the projected grid align with the light profile itself. """ grid_2d_projected = grid.grid_2d_radial_projected_from( centre=sersic_light_profile.centre, angle=sersic_light_profile.angle() @@ -388,9 +384,9 @@ image_1d = sersic_light_profile.image_2d_from(grid=grid_2d_projected) """ -We can now plot the 1D radial profile of the light profile. This profile shows how the intensity of the light -changes as a function of distance from the profile's center. This is a more informative way to visualize the light p -rofile's distribution. +We can now plot the 1D radial profile of the light profile. This profile shows how the intensity of the light +changes as a function of distance from the profile's center. This is a more informative way to visualize the light +profile's distribution. When we plot 1D quantities, we do not use built-in plotting functions as in 2D, but instead use standard matplotlib functionality. @@ -411,7 +407,7 @@ Since galaxy light distributions often cover a wide range of values, they are typically better visualized on a log10 scale. This approach helps highlight details in the faint outskirts of a light profile. -The `MatPlot2D` object has a `use_log10` option that applies this transformation automatically. Below, you can see +The plotting functions have a `use_log10` option that applies this transformation automatically. Below, you can see that the image plotted in log10 space reveals more details. """ aplt.plot_array( @@ -506,8 +502,8 @@ and disk profiles in this example share the same `centre`, meaning that plotting them together on the same 1D plot shows how they vary relative to one another. -If the `centre` of the profiles were different, when you make the 1D plot you would need to decide if you should the -profiles offset from one another or plot them both from zero. +If the `centre` of the profiles were different, when you make the 1D plot you would need to decide if you should plot +the profiles offset from one another or plot them both from zero. """ grid_2d_projected = grid.grid_2d_radial_projected_from( centre=galaxy.bulge.centre, angle=galaxy.bulge.angle() @@ -594,12 +590,10 @@ """ This `kpc_per_arcsec` can be used as a conversion factor between arcseconds and kiloparsecs when plotting images of -galaxies. - -We compute this value and plot the image in converted units of kiloparsecs. +galaxies, for example by converting plot tick labels to kiloparsecs. -This passes the plotting modules `Units` object a `ticks_convert_factor` and manually specified the new units of the -plot ticks. +The `autogalaxy_workspace/*/guides/units` examples show how to perform these conversions when plotting; below we +simply plot the image in its default units of arc-seconds. """ aplt.plot_array(array=galaxy.image_2d_from(grid=grid), title="Image") @@ -647,7 +641,7 @@ on a higher resolution grid than the image data to ensure the calculation is accurate. For a new user, the details of over-sampling are not important, therefore just be aware that all calculations use an -adaptive over sampling scheme which high accuracy across all use cases. +adaptive over sampling scheme which ensures high accuracy across all use cases. Once you are more experienced, you should read up on over-sampling in more detail via the `autogalaxy_workspace/*/guides/advanced/over_sampling.ipynb` notebook. diff --git a/scripts/chapter_1_introduction/tutorial_2_data.py b/scripts/chapter_1_introduction/tutorial_2_data.py index 68fe088..060b332 100644 --- a/scripts/chapter_1_introduction/tutorial_2_data.py +++ b/scripts/chapter_1_introduction/tutorial_2_data.py @@ -169,8 +169,8 @@ We also need to consider the units of our image data. Let’s assume that the galaxy image is measured in units of electrons per second ($e^- s^{-1}$), which is standard for CCD imaging data. To simulate the number of electrons -actually detected in each pixel, we multiply the image by the observation’s exposure time. This conversion changes t -he units to the total number of electrons collected per pixel over the entire exposure time. +actually detected in each pixel, we multiply the image by the observation’s exposure time. This conversion changes +the units to the total number of electrons collected per pixel over the entire exposure time. Once the image is converted, we add Poisson noise, simulating the randomness in the photon-to-electron conversion process. After adding the noise, we convert the image back to units of electrons per second for analysis, as diff --git a/scripts/chapter_1_introduction/tutorial_3_fitting.py b/scripts/chapter_1_introduction/tutorial_3_fitting.py index d73b7f7..66e496b 100644 --- a/scripts/chapter_1_introduction/tutorial_3_fitting.py +++ b/scripts/chapter_1_introduction/tutorial_3_fitting.py @@ -126,11 +126,8 @@ print(mask[48:53, 48:53]) # Central pixels are `False` and therefore unmasked. """ -We can visualize the mask over the galaxy image using an `Imaging`, which helps us adjust the mask as needed. -This is useful to ensure that the mask appropriately covers the galaxy's light and does not exclude important regions. - -To overlay objects like a mask onto a figure, we use the `Visuals2D` object. This tool allows us to add custom -visuals to any plot, providing flexibility in creating tailored visual representations. +We can visualize the mask over the galaxy image, which helps us adjust the mask as needed. This is useful to ensure +that the mask appropriately covers the galaxy's light and does not exclude important regions. """ aplt.plot_array(array=dataset.data, title="Imaging Data With Mask") @@ -141,8 +138,8 @@ dataset = dataset.apply_mask(mask=mask) """ -When we plot the masked imaging data again, the mask is now automatically included in the plot, even though we did -not explicitly pass it using the `Visuals2D` object. The plot also zooms into the unmasked area, showing only the +When we plot the masked imaging data again, the mask is now automatically included in the plot, even though we did +not explicitly pass it to the plot function. The plot also zooms into the unmasked area, showing only the region where we will focus our analysis. This is particularly helpful when working with large images, as it centers the view on the regions where the galaxy's signal is detected. """ @@ -175,7 +172,7 @@ print(dataset.data.native.shape) print( dataset.data.slim.shape -) # This should be lower than the total number of pixels, e.g., 100 x 100 = 10,000 +) # This should be lower than the total number of pixels, e.g., 101 x 101 = 10,201 """ The `mask` object also has a `pixels_in_mask` attribute, which gives the number of unmasked pixels. This should @@ -232,8 +229,8 @@ Now that our data is masked, we are ready to proceed with the fitting process. -Fitting the data is done using the `Galaxy` and `Galaxies objects that we introduced in tutorial 2. We will start by -setting up a `Galaxies`` object, using the same galaxy configuration that we previously used to simulate the +Fitting the data is done using the `Galaxy` and `Galaxies` objects that we introduced in tutorial 1. We will start by +setting up a `Galaxies` object, using the same galaxy configuration that we previously used to simulate the imaging data. This setup will give us what is known as a 'perfect' fit, as the simulated and fitted models are identical. """ galaxy = ag.Galaxy( @@ -301,7 +298,7 @@ aplt.plot_array(array=fit.model_data, title="Model Image") """ -The `residual_map` is the different between the observed image and model image, showing where in the image the fit is +The `residual_map` is the difference between the observed image and model image, showing where in the image the fit is good (e.g. low residuals) and where it is bad (e.g. high residuals). The expression for the residual map is simply: @@ -322,7 +319,7 @@ """ Are these residuals indicative of a good fit to the data? Without considering the noise in the data, it's difficult -to ascertain. That is, its hard to ascenrtain if a residual value is large or small because this depends on the +to ascertain. That is, its hard to ascertain if a residual value is large or small because this depends on the amount of noise in that pixel. The `normalized_residual_map` divides the residual-map by the noise-map, giving the residual in units of the noise. @@ -476,7 +473,7 @@ These metrics are standard ways to quantify the quality of a model fit. They are applicable not only to 1D data but also to more complex data structures like 2D images, 3D data cubes, or any other multidimensional datasets. -__Incorrect Fit___ +__Incorrect Fit__ In the previous section, we successfully created and fitted a galaxy model to the image data, resulting in an excellent fit. The residual map and chi-squared map showed no significant discrepancies, indicating that the diff --git a/scripts/chapter_1_introduction/tutorial_5_summary.py b/scripts/chapter_1_introduction/tutorial_5_summary.py index 3c51e73..2ce406c 100644 --- a/scripts/chapter_1_introduction/tutorial_5_summary.py +++ b/scripts/chapter_1_introduction/tutorial_5_summary.py @@ -1,5 +1,5 @@ """ -Tutorial 9: Summary +Tutorial 5: Summary =================== In this chapter, we have learnt that: @@ -7,10 +7,10 @@ 1) **PyAutoGalaxy** uses Cartesian `Grid2D`'s of $(y,x)$ coordinates to evaluate galaxy luminous emission. 2) These grids are combined with light profiles to compute images and other quantities. 3) Profiles are grouped together to make galaxies. - 4) Collections of galaxies (at the same redshift) can be made.. + 4) Collections of galaxies (at the same redshift) can be made. 5) The Universe's cosmology can be input into this `Galaxies` to convert its units to kiloparsecs. 6) The galaxies's image can be used to simulate galaxy `Imaging` like it was observed with a real telescope. - 7) This data can be fitted, so to as quantify how well a model galaxy system represents the observed image. + 7) This data can be fitted, so as to quantify how well a model galaxy system represents the observed image. In this summary, we'll go over all the different Python objects introduced throughout this chapter and consider how they come together as one. @@ -73,7 +73,7 @@ """ __Object Composition__ -Lets now consider how all of the objects we've covered throughout this chapter (`LightProfile`'s, `MassProfile`'s, +Lets now consider how all of the objects we've covered throughout this chapter (`LightProfile`'s, `Galaxy`'s, `Galaxies`'s) come together. The `Galaxies` contain the `Galaxy`'s which contains the `Profile`'s: @@ -102,8 +102,8 @@ Understanding how these objects decompose into the different components of a galaxy is important for general **PyAutoGalaxy** use. -As the galaxy systems that we analyse become more complex, it is useful to know how to decompose their light -profiles, galaxies and galaxies to extract different pieces of information about the galaxy. +As the galaxy systems that we analyse become more complex, it is useful to know how to decompose their light +profiles and galaxies to extract different pieces of information about the galaxy. For example, we made our galaxy above with two light profiles, a `bulge` and `disk`. We can plot the image of each component individually, now that we know how to break-up the different components of the galaxies. @@ -115,11 +115,11 @@ """ __Visualization__ -Furthermore, using the `MatPLot2D` and `Visuals2D` objects we can visualize any aspect we're interested -in and fully customize the figure. +Furthermore, using the arguments of the `aplt` plotting functions we can visualize any aspect we're interested +in and fully customize the figure. -Before beginning chapter 2 of **HowToGalaxy**, you should checkout the package `autogalaxy_workspace/plot`. -This provides a full API reference of every plotting option in **PyAutoGalaxy**, allowing you to create your own +Before beginning chapter 2 of **HowToGalaxy**, you should checkout the package `autogalaxy_workspace/*/guides/plot`. +This provides a full API reference of every plotting option in **PyAutoGalaxy**, allowing you to create your own fully customized figures of galaxies with minimal effort! """ aplt.plot_array(array=galaxies[0].bulge.image_2d_from(grid=grid), title="Bulge Image") @@ -130,15 +130,15 @@ __Code Design__ To end, I want to quickly talk about the **PyAutoGalaxy** code-design and structure, which was really the main topic of -this tutoriag. +this tutorial. -Throughout this chapter, we never talk about anything like it was code. We didn`t refer to 'variables', 'parameters`' -'functions' or 'dictionaries', did we? Instead, we talked about 'galaxies'. We discussed +Throughout this chapter, we never talk about anything like it was code. We didn't refer to 'variables', 'parameters', +'functions' or 'dictionaries', did we? Instead, we talked about 'galaxies'. We discussed the objects that we, as scientists, think about when we consider a galaxy system. -Software that abstracts the underlying code in this way follows an `object-oriented design`, and it is our hope +Software that abstracts the underlying code in this way follows an `object-oriented design`, and it is our hope with **PyAutoGalaxy** that we've made its interface (often called the API for short) very intuitive, whether you were -previous familiar with galaxy morphology or a complete newcomer! +previously familiar with galaxy morphology or a complete newcomer! __Source Code__ @@ -161,13 +161,13 @@ __Wrap Up__ -You`ve learn a lot in this chapter, but what you have not learnt is how to 'model' a real galaxy. +You've learnt a lot in this chapter, but what you have not learnt is how to 'model' a real galaxy. In the real world, we have no idea what the 'correct' combination of light profiles are that will give a good fit to a galaxy. Modeling is the process of finding the model which provides a good fit and it is the topic of chapter 2 of **HowToGalaxy**. -Finally, if you enjoyed doing the **HowToGalaxy** tutorials please git us a star on the **PyAutoGalaxy** GitHub +Finally, if you enjoyed doing the **HowToGalaxy** tutorials please give us a star on the **PyAutoGalaxy** GitHub repository: https://github.com/PyAutoLabs/PyAutoGalaxy diff --git a/scripts/chapter_2_modeling/tutorial_10_prior_passing.py b/scripts/chapter_2_modeling/tutorial_10_prior_passing.py index 1cf662f..ff634b4 100644 --- a/scripts/chapter_2_modeling/tutorial_10_prior_passing.py +++ b/scripts/chapter_2_modeling/tutorial_10_prior_passing.py @@ -6,7 +6,7 @@ non-linear searches. This used an initial search to fit a simple model, whose results were used to tune and initialize the priors of a more complex model that was fitted by the second search. -However, the results were passed between searches were passed manually. I explicitly wrote out every result as a prior +However, the results were passed between searches manually. I explicitly wrote out every result as a prior containing the values inferred in the first search. **PyAutoGalaxy** has an API for passing priors in a more generalized way, which is the topic of this tutorial. @@ -37,7 +37,7 @@ we'll use the same galaxy data as the previous tutorial, where: - - The galaxy's bulge is an `Sersic`. + - The galaxy's bulge is a `Sersic`. - The galaxy's disk is an `Exponential`. All the usual steps for setting up a model fit (masking, analysis, etc.) are included below. @@ -144,7 +144,7 @@ write out the inferred parameter values of each component. The details of how prior passing is performed will be expanded upon at the end of the tutorial. -We start with the bulge, which in the previous search was an `Sersic` with its centre fixed to (0.0, 0.0) +We start with the bulge, which in the previous search was a `Sersic` with its centre fixed to (0.0, 0.0) and its `sersic_index` fixed to 4.0. The API for passing priors is shown below and there are two things worth noting: 1) We pass the priors using the `model` attribute of the result. This informs **PyAutoGalaxy** to pass the result as a @@ -165,7 +165,7 @@ ) """ -For the disk, we are passing the result of an `Exponential` to an `Sersic`. +For the disk, we are passing the result of an `Exponential` to a `Sersic`. We do not pass the `ell_comps` because this would pair them to the `bulge`, as was performed in the first model-fit. @@ -208,7 +208,7 @@ print( "The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_2 = search_2.fit(model=model_2, analysis=analysis_2) @@ -250,8 +250,8 @@ they use GaussianPrior`s where: - The mean values are the median PDF results of every parameter in search 1. - - The sigma values are specified in the `width_modifier` field of the profile's entry in the `priors.yaml' config - file (we will discuss why this is used in a moment). + - The sigma values are specified in the `width_modifier` field of the profile's entry in the `config/priors` + config files (we will discuss why this is used in a moment). Like the manual `GaussianPrior`'s that were used in the previous tutorial, the prior passing API sets up the prior on each parameter with a `GaussianPrior` centred on the high likelihood regions of parameter space! @@ -276,8 +276,8 @@ parameter space that correspond to highest log likelihood solutions in the previous search. Our priors therefore correspond to the `correct` regions of parameter space. - 3) The sigma of the Gaussian uses the value specified for the profile in the `config/priors/*.yaml` config file's - `width_modifer` field (check these files out now). + 3) The sigma of the Gaussian uses the value specified for the profile in the `config/priors/*.yaml` config file's + `width_modifier` field (check these files out now). The idea here is simple. We want a value of sigma that gives a `GaussianPrior` wide enough to search a broad region of parameter space, so that the model can change if a better solution is nearby. However, we want it @@ -285,7 +285,7 @@ into an incorrect solution! The `width_modifier` values in the priors config file have been chosen based on our experience as being a good -balance broadly sampling parameter space but not being so narrow important solutions are missed. +balance of broadly sampling parameter space but not being so narrow that important solutions are missed. There are two ways a value is specified using the priors/width file: @@ -306,12 +306,9 @@ However, there are parameters where using an absolute value does not make sense. Intensity is a good example of this. The intensity of an image depends on its units, S/N, galaxy brightness, etc. There is no single absolute value that -one can use to generically chain the intensity of any two proflies. Thus, it makes more sense to chain them using +one can use to generically chain the intensity of any two profiles. Thus, it makes more sense to chain them using the relative value from a previous search. -We can customize how priors are passed from the results of a search and non-linear search by editing the - `prior_passer` settings in the `general.yaml` config file. - __EXAMPLE__ Lets go through an example using a real parameter. Lets say in search 1 we fit the galaxy's light with an @@ -323,12 +320,12 @@ The prior on the galaxy's bulge sersic index in search 2 would thus be a `GaussianPrior` with mean=4.0. -The value of the Sersic index `width_modifier` in the priors config file sets sigma. The prior config file specifies -that we use an "Absolute" value of 0.8 to chain this prior. Thus, the `GaussianPrior` in search 2 would have a -mean=4.0 and sigma=0.8. +The value of the Sersic index `width_modifier` in the priors config file sets sigma. The prior config file specifies +that we use an "Absolute" value of 1.5 to chain this prior. Thus, the `GaussianPrior` in search 2 would have a +mean=4.0 and sigma=1.5. -If the prior config file had specified that we use an relative value of 0.8, the GaussianPrior in search 2 would have a -mean=4.0 and sigma = 4.0 * 0.8 = 3.2. +If the prior config file had instead specified that we use a relative value of 0.8, the GaussianPrior in search 2 would +have a mean=4.0 and sigma = 4.0 * 0.8 = 3.2. And with that, we're done. Chaining priors is a bit of an art form, but one that works really well. """ diff --git a/scripts/chapter_2_modeling/tutorial_11_summary b/scripts/chapter_2_modeling/tutorial_11_summary index f95c902..bcbaf7d 100755 --- a/scripts/chapter_2_modeling/tutorial_11_summary +++ b/scripts/chapter_2_modeling/tutorial_11_summary @@ -1,19 +1,20 @@ -Congratulations, you've completed the second chapter of the **PyAutoGalaxy** tutorials! - -In this chapter, you learnt: - -1) What a non-linear search is. -2) How to use a non-linear search to sample the parameter space of a model. -3) How the priors we give parameters specify our search of this parameter space. -4) How one must carefully balance complexity and realism when fitting a model to a data-set. -5) The importance of factoring in run-speed when modeling a galaxy, and tricks to speed up the analysis. -6) About masking data in the analysis. -7) How to chain searches together, passing priors from one search to the next so that earlier fits of simpler - models guide later fits of more complex ones. - -At this point, you are ready to begin modeling galaxies. The 'modeling' folders in the -autogalaxy_workspace contains a number of scripts that can be easily adopted to model galaxies using a variety of -different approaches, models and non-linear searches. If you have your own galaxy data, I'd recommend you adapt these -scripts to your data. If you don't have your own data, checkout the 'simulators' folder to simulate your own dataset! - -Alternatively, you may wish to continue on to chapter 3 on pixelized galaxy reconstructions. \ No newline at end of file +Congratulations, you've completed the second chapter of the **PyAutoGalaxy** tutorials! + +In this chapter, you learnt: + +1) What a non-linear search is. +2) How to use a non-linear search to sample the parameter space of a model. +3) How the priors we give parameters specify our search of this parameter space. +4) How one must carefully balance complexity and realism when fitting a model to a data-set. +5) The importance of factoring in run-speed when modeling a galaxy, and tricks to speed up the analysis. +6) About masking data in the analysis. +7) How to chain searches together, passing priors from one search to the next so that earlier fits of simpler + models guide later fits of more complex ones. + +At this point, you are ready to begin modeling galaxies. The 'modeling' folders in the +autogalaxy_workspace contains a number of scripts that can be easily adopted to model galaxies using a variety of +different approaches, models and non-linear searches. If you have your own galaxy data, I'd recommend you adapt these +scripts to your data. If you don't have your own data, checkout the 'simulators' folder to simulate your own dataset! + +Alternatively, you may wish to continue on to chapter 3 on pixelized galaxy reconstructions, or chapter 4 on +scaling up to systems with multiple galaxies. \ No newline at end of file diff --git a/scripts/chapter_2_modeling/tutorial_1_non_linear_search.py b/scripts/chapter_2_modeling/tutorial_1_non_linear_search.py index 0d2aad8..bb861c0 100644 --- a/scripts/chapter_2_modeling/tutorial_1_non_linear_search.py +++ b/scripts/chapter_2_modeling/tutorial_1_non_linear_search.py @@ -30,7 +30,7 @@ __Overview__ -In this tutorial, we will use a non-linear search to fit a single Serisc light profile to simulated imaging of a +In this tutorial, we will use a non-linear search to fit a single Sersic light profile to simulated imaging of a galaxy. We will: - Introduce concept like a "parameter space", "likelihood surface" and "priors", and relate them to how a non-linear @@ -40,7 +40,7 @@ model instance to the data. - Fit datasets with different non-linear searches, including a maximum likelihood estimator (MLE), - Markok Chain Monte Carlo (MCMC) and nested sampling. + Markov Chain Monte Carlo (MCMC) and nested sampling. __Contents__ @@ -83,8 +83,8 @@ This defines a parameter space in three dimensions, representing the relationships between \(x\), \(y\), \(z\), and the output \(f(x, y, z)\). -This concept of parameter space is closely related to how we approach model-fitting. For instance, in chapter 1, w -e created instances of a `Galaxy` object with +This concept of parameter space is closely related to how we approach model-fitting. For instance, in chapter 1, +we created instances of a `Galaxy` object with parameters like \( (\text{`centre_0`}, \text{`centre_1`}, \text{`ell_comps_0`}, \text{`ell_comps_1`}, \text{`intensity`}, \text{`effective_radius`}, \text{`sersic_index`}) \). These parameters were used to fit data and compute a log likelihood. @@ -101,7 +101,7 @@ This parameter space is "non-linear," meaning the relationship between the model parameters and the log likelihood is not a simple linear one. Because of this non-linearity, we cannot predict the log likelihood from a given set of model -parameters without actually performing a fit to the data, as we did in tutorial 1. +parameters without actually performing a fit to the data, as we did in the fitting tutorial of chapter 1. __Non-Linear Search__ @@ -149,7 +149,7 @@ __Deeper Background__ -**The descriptions of how searches work in this example are simplfied and phoenomenological and do not give a full +**The descriptions of how searches work in this example are simplified and phenomenological and do not give a full description of how they work at a deep statistical level. The goal is to provide you with an intuition for how to use them and when different searches are appropriate for different problems. Later tutorials will provide a more formal description of how these searches work.** @@ -175,7 +175,7 @@ [PyAutoFit](https://github.com/PyAutoLabs/PyAutoFit), an open-source project that allows complex model fitting techniques to be straightforwardly integrated into scientific modeling software. -**PyAutoFit** is actually a spin-off project of **PyAutoGalaxy**. whereby we found that the statistic techniques and +**PyAutoFit** is actually a spin-off project of **PyAutoGalaxy**, whereby we found that the statistical techniques and methods we applied to model galaxies could be used in a more general setting to many different scientific problems. Check it out if you are interested in developing your own software to perform advanced model-fitting! @@ -361,7 +361,7 @@ 2. **Markov Chain Monte Carlo (MCMC)** using the `Emcee` non-linear search. 3. **Nested Sampling** using the `Nautilus` non-linear search. -In this example, non-linear search results are stored in memory rather and not written to hard disk because the fits +In this example, non-linear search results are stored in memory rather than written to hard disk because the fits are fast and can therefore be easily regenerated. The next tutorial will perform fits which write results to the hard-disk and discuss the outputs that are generated. @@ -394,7 +394,7 @@ print( """ The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + This Jupyter notebook cell will progress once the search has completed - this could take a few minutes! """ ) @@ -409,7 +409,7 @@ The `info` attribute shows the result in a readable format. -[Above, we discussed that the `info_whitespace_length` parameter in the config files could b changed to make +[Above, we discussed that the `info_whitespace_length` parameter in the config files could be changed to make the `model.info` attribute display optimally on your computer. This attribute also controls the whitespace of the `result.info` attribute.] """ @@ -456,7 +456,7 @@ print( """ The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + This Jupyter notebook cell will progress once the search has completed - this could take a few minutes! """ ) @@ -481,7 +481,7 @@ However, MLE has its limitations. As seen above, it can get "stuck" in a local maximum, particularly if the starting point is poorly chosen. In complex model-fitting problems, providing a suitable starting point can be -challenging. While MLE performed well in the example with just three parameters, it struggles with models that have +challenging. While MLE performed well in the example with just six parameters, it struggles with models that have many parameters, as the complexity of the likelihood surface makes simply moving "up" the gradient less effective. The MLE also does not provide any information on the errors on the parameters, which is a significant limitation. @@ -519,7 +519,7 @@ print( """ The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + This Jupyter notebook cell will progress once the search has completed - this could take a few minutes! """ ) @@ -540,7 +540,7 @@ A major advantage of MCMC is that it provides estimates of parameter uncertainties by "mapping out" the likelihood surface, unlike MLE, which only finds the maximum likelihood solution. These error estimates are accessible in -the `result.info` string and through the `result.samples` object, which is explained fully in tutorial 5. +the `result.info` string and through the `result.samples` object, which is explained fully in tutorial 7. While a good starting point wasn't necessary for this simple model, it becomes essential for efficiently mapping the likelihood surface in more complex models with many parameters. The code below shows an MCMC fit using a good starting @@ -563,14 +563,14 @@ search = af.Emcee( nwalkers=20, # The number of walkers we'll use to sample parameter space. - nsteps=300, # The number of steps each walker takes, after which 10 * 200 = 2000 steps the non-linear search ends. + nsteps=300, # The number of steps each walker takes, after which 20 * 300 = 6000 steps the non-linear search ends. initializer=initializer, ) print( """ The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + This Jupyter notebook cell will progress once the search has completed - this could take a few minutes! """ ) @@ -628,7 +628,7 @@ print( """ The non-linear search has begun running. - This Jupyter notebook cell with progress once the search has completed - this could take a few minutes! + This Jupyter notebook cell will progress once the search has completed - this could take a few minutes! """ ) diff --git a/scripts/chapter_2_modeling/tutorial_2_practicalities.py b/scripts/chapter_2_modeling/tutorial_2_practicalities.py index aec66ec..c99ab94 100644 --- a/scripts/chapter_2_modeling/tutorial_2_practicalities.py +++ b/scripts/chapter_2_modeling/tutorial_2_practicalities.py @@ -22,15 +22,15 @@ **PyAutoFit:** The parent package of PyAutoGalaxy, which handles practicalities of model-fitting. **Initial Setup:** Load the dataset we'll fit a model to using a non-linear search. **Mask:** Apply a mask to the dataset. + **Over Sampling:** Apply adaptive over sampling to the dataset. **Model:** Introduce the model we will fit to the data. **Search:** Setup the non-linear search, Nautilus, used to fit the model to the data. **Search Settings:** Discuss the settings of the non-linear search, including the number of live points. - **Number Of Cores:** Discuss how to use multiple cores to fit models faster in parallel. - **Parallel Script:** Running the model-fit in parallel if a bug occurs in a Jupiter notebook. **Iterations Per Update:** How often the non-linear search outputs the current results to hard-disk. **Analysis:** Create the Analysis object which contains the `log_likelihood_function` that the non-linear search calls. + **Run Times:** Estimate the run-time of the model-fit before initiating it. **Model-Fit:** Fit the model to the data. - **Result:** Print the results of the model-fit to the terminal. + **Result Info:** Print the results of the model-fit to the terminal. **Output Folder:** Inspect the output folder where results are stored. **Unique Identifier:** Discussion of the unique identifier of the model-fit which names the folder in the output directory. **Output Folder Contents:** What is output to the output folder (model results, visualization, etc.). @@ -164,8 +164,8 @@ more thoroughly, increasing accuracy but also runtime. Fewer live points reduce run-time but may make the search less reliable, possibly getting stuck in local maxima. -The ideal number of live points depends on model complexity. More parameters generally require more live points, but -the default of 200 is sufficient for most galaxy models. Lower values can still yield reliable results, particularly +The ideal number of live points depends on model complexity. More parameters generally require more live points, but +the default of 3000 is more than sufficient for most galaxy models. Lower values can still yield reliable results, particularly for simpler models. For this example (6 parameters), we reduce the live points to 80 to speed up runtime without compromising accuracy. @@ -185,8 +185,8 @@ Depending on how long it takes for the model to be fitted to the data (see discussion about run times below), this can take up a large fraction of the run-time of the non-linear search. -For this fit, the fit is very fast, thus we set a high value of `iterations_per_quick_update=10000` to ensure these updates -so not slow down the overall speed of the model-fit. +For this fit, the fit is very fast, thus we set a high value of `iterations_per_quick_update=2500` to ensure these updates +do not slow down the overall speed of the model-fit. **If the iteration per update is too low, the model-fit may be significantly slowed down by the time it takes to output results and visualization frequently to hard-disk. If your fit is consistent displaying a log saying that it @@ -197,7 +197,7 @@ name="tutorial_2_practicalities", unique_tag=dataset_name, n_live=80, - n_batch=50, # GPU lens model fits are batched and run simultaneously, see VRAM section below. + n_batch=50, # GPU model fits are batched and run simultaneously, see the run times section below. iterations_per_quick_update=2500, ) @@ -221,15 +221,15 @@ - **The log likelihood evaluation time:** the time it takes for a single `instance` of the model to be fitted to the dataset such that a log likelihood is returned. - - **The number of iterations (e.g. log likelihood evaluations) performed by the non-linear search:** more complex lens + - **The number of iterations (e.g. log likelihood evaluations) performed by the non-linear search:** more complex models require more iterations to converge to a solution (and as discussed above, settings like the number of live points also control this). For this analysis, the log likelihood evaluation time is ~0.05 seconds, which is extremely fast for model fitting. -The more advanced fitting techniques discussed at the end of chapter 1 (e.g. shapelets, multi Gaussian expansions, -pixelizations) have longer log likelihood evaluation times. However, on GPU, they can be so fast they may not produce -significantly longer overall run-time feasible. +The more advanced fitting techniques discussed later in this chapter (e.g. multi Gaussian expansions, shapelets) and +the pixelizations of chapter 3 have longer log likelihood evaluation times. However, on GPU, they can be so fast that +they may not produce significantly longer overall run-times. To estimate the expected overall run time of the model-fit we multiply the log likelihood evaluation time by an estimate of the number of iterations the non-linear search will perform, which is around 50000 to 100000 for this model. @@ -264,7 +264,7 @@ print( "The non-linear search has begun running - checkout the autogalaxy_workspace/output/" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result = search.fit(model=model, analysis=analysis) @@ -276,7 +276,7 @@ A concise readable summary of the results is given by printing its `info` attribute. -[Above, we discussed that the `info_whitespace_length` parameter in the config files could b changed to make +[Above, we discussed that the `info_whitespace_length` parameter in the config files could be changed to make the `model.info` attribute display optimally on your computer. This attribute also controls the whitespace of the `result.info` attribute.] """ @@ -308,9 +308,9 @@ __Output Folder Contents__ -Now this is running you should checkout the `autogalaxy_workspace/output` folder. This is where the results of the -search are written to hard-disk (in the `start_here` folder), where all outputs are human readable (e.g. as .json, -.csv or text files). +Now this is running you should checkout the `autogalaxy_workspace/output` folder. This is where the results of the +search are written to hard-disk (in the `tutorial_2_practicalities` folder), where all outputs are human readable +(e.g. as .json, .csv or text files). As the fit progresses, results are written to the `output` folder on the fly using the highest likelihood model found by the non-linear search so far. This means you can inspect the results of the model-fit as it runs, without having to @@ -353,15 +353,15 @@ The PDF shows the 1D and 2D probabilities estimated for every parameter after the model-fit. The two dimensional figures can show the degeneracies between different parameters, for example how increasing the intensity $I$ of the -source galaxy and decreasing its effective radius $R_{Eff}$ lead to similar likelihoods and probabilities. +galaxy and decreasing its effective radius $R_{Eff}$ lead to similar likelihoods and probabilities. This PDF will be discussed more in the next tutorial. -The plot is labeled with short hand parameter names (e.g. `sersic_index` is mapped to the short hand -parameter `n`). These mappings ate specified in the `config/notation.yaml` file and can be customized by users. +The plot is labeled with short hand parameter names (e.g. `sersic_index` is mapped to the short hand +parameter `n`). These mappings are specified in the `config/notation.yaml` file and can be customized by users. -The superscripts of labels correspond to the name each component was given in the model (e.g. for the `Isothermal` -mass its name `mass` defined when making the `Model` above is used). +The superscripts of labels correspond to the name each component was given in the model (e.g. for the `Sersic` +bulge its name `bulge` defined when making the `Model` above is used). __Other Practicalities__ diff --git a/scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py b/scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py index 8c75590..45640fb 100644 --- a/scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py +++ b/scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py @@ -7,7 +7,7 @@ In this tutorial, we'll use a more realistic model, which consists of the following light profiles: - - An `Sersic` light profile for the galaxy's bulge [7 parameters]. + - A `Sersic` light profile for the galaxy's bulge [7 parameters]. - An `Exponential` light profile for the galaxy's disk [6 parameters] This model has 13 free parameters, meaning that the parameter space and likelihood function it defines has a @@ -36,9 +36,9 @@ """ __Initial Setup__ -we'll use new galaxying data, where: +we'll use new imaging data of a galaxy, where: - - The galaxy's bulge is an `Sersic`. + - The galaxy's bulge is a `Sersic`. - The galaxy's disk is an `Exponential`. """ dataset_name = "simple" @@ -124,7 +124,7 @@ print( "The non-linear search has begun running - checkout the autogalaxy_workspace/output/howtogalaxy/chapter_2/tutorial_3_realism_and_complexity" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result = search.fit(model=model, analysis=analysis) @@ -179,9 +179,9 @@ ) print( - "The non-linear search has begun running - checkout the autogalaxy_workspace/output/3_realism_and_complexity" + "The non-linear search has begun running - checkout the autogalaxy_workspace/output/howtogalaxy/chapter_2/tutorial_3_realism_and_complexity__local_maxima" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_local_maxima = search.fit(model=model, analysis=analysis) @@ -197,14 +197,14 @@ print(result_local_maxima.info) """ -Lats look at the fit to the `Imaging` data, which is clearly worse than our original fit above. +Lets look at the fit to the `Imaging` data, which is clearly worse than our original fit above. """ aplt.subplot_fit_imaging(fit=result_local_maxima.max_log_likelihood_fit) """ Finally, just to be sure we hit a local maxima, lets compare the maximum log likelihood values of the two results -The local maxima value is significantly lower, confirming that our non-linear search simply failed to locate lens +The local maxima value is significantly lower, confirming that our non-linear search simply failed to locate galaxy models which fit the data better when it searched parameter space. """ print("Likelihood of Global Model:") @@ -220,7 +220,7 @@ inferring a local maxima is still very real, especially as we make our model more complex. Lets think about *complexity*. As we make our model more realistic, we also made it more complex. For this -tutorial, our non-linear parameter space went from 7 dimensions to 13. This means there was a much larger *volume* of +tutorial, our non-linear parameter space went from 6 dimensions to 13. This means there was a much larger *volume* of parameter space to search. As this volume grows, there becomes a higher chance that our non-linear search gets lost and infers a local maxima, especially if we don't set it up with enough live points! @@ -237,7 +237,7 @@ 2) The non-linear search failed because parameter space was too complex. Could we make it less complex, whilst still keeping our model fairly realistic? - 3) The galaxy in this example had only 7 non-linear parameters. Real galaxies may have multiple components (e.g. a - disk, bulge, bar, star-forming knot) and there may even be more than 1 galaxy! Do you think there is any hope of - us navigating a parameter space if the galaxies contributes 30+ parameters? + 3) The galaxy in this example had only 13 non-linear parameters. Real galaxies may have multiple components (e.g. a + disk, bulge, bar, star-forming knot) and there may even be more than 1 galaxy! Do you think there is any hope of + us navigating a parameter space if the galaxies contribute 30+ parameters? """ diff --git a/scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py b/scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py index db8cd32..4080920 100644 --- a/scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py +++ b/scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py @@ -39,7 +39,7 @@ we'll use the same galaxy data as the previous tutorial, where: - - The galaxy's bulge is an `Sersic`. + - The galaxy's bulge is a `Sersic`. - The galaxy's disk is an `Exponential`. """ dataset_name = "simple" @@ -131,10 +131,10 @@ disk = af.Model(ag.lp.Exponential) """ -By default, the prior on the $(y,x)$ coordinates of a `LightProfile` is a GaussianPrior with +By default, the prior on the $(y,x)$ coordinates of a `LightProfile` is a GaussianPrior with `mean=0.0` and `sigma=0.3`. However, visual inspection of our galaxy image tells us that its centre (based on the -galaxy's luminous emission) is at x = 0.0" and y = 0.0", so lets reduce the `sigma` value on this prior so the -non-linear search looks over a very narrow range of `centre` values in parameter space. +galaxy's luminous emission) is at x = 0.0" and y = 0.0", so lets replace this prior with a narrow `UniformPrior`, so +the non-linear search looks over a very narrow range of `centre` values in parameter space. """ bulge.centre_0 = af.UniformPrior(lower_limit=-0.05, upper_limit=0.05) bulge.centre_1 = af.UniformPrior(lower_limit=-0.05, upper_limit=0.05) @@ -142,8 +142,8 @@ disk.centre_1 = af.UniformPrior(lower_limit=-0.05, upper_limit=0.05) """ -By default, the elliptical components of the of our galaxy's elliptical `LightProfile` are `TruncatedGaussianPrior`'s -with `mean=0.0` and `sigma=0.5`. Note that the solution `ell_comps=(0.0, 0.0)` corresponds to a spherical system +By default, the elliptical components of our galaxy's elliptical `LightProfile` are `TruncatedGaussianPrior`'s +with `mean=0.0` and `sigma=0.3`. Note that the solution `ell_comps=(0.0, 0.0)` corresponds to a spherical system and that all physical solutions (e.g. with axis-ratios running from 0.0 -> 1.0 and position angles 0.0 -> 180.0 degrees) are encapsulated for solutions where each component runs from -1.0 -> 1.0). @@ -193,7 +193,7 @@ The `sersic_index` defines how concentrated the light profile is. In galaxy structure studies, values of Sersic index around 1.0 indicate a disk galaxy (which is the value the `Exponential` uses). -Higher values of 3 or 4 indicate an elliptical galaxy. **PyAutoGalaxy** assumes a `UniformPrior` between 0.8 and 8.0 +Higher values of 3 or 4 indicate an elliptical galaxy. **PyAutoGalaxy** assumes a `UniformPrior` between 0.8 and 5.0 by default on this parameter, as a user could model galaxies where the galaxy is of any morphology. @@ -205,12 +205,7 @@ """ We now compose the overall model, where the galaxy model uses the `Model` components above which had their -priors customizes. - -In this exercise, I'm not going to change any priors on the galaxy. Whilst modeling experts can look at a -galaxy and often tell you roughly where the galaxy is located, it is something of art -form. Furthermore, the source's morphology can be pretty complex, making it difficult to come up with a good source -prior! +priors customized. """ galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge, disk=disk) @@ -238,7 +233,7 @@ print( "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_4_custom_priors" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_custom_priors = search.fit(model=model, analysis=analysis) @@ -272,8 +267,8 @@ Disadvantages: - If we specified a prior incorrectly the non-linear search will infer an incorrect solution. - - The priors for the search were tailored to the specific galaxy we fitted. If we are fitting multiple galaxies, - we would have customize the priors for every single fit, for large samples of galaxies this would take a lot of time! + - The priors for the search were tailored to the specific galaxy we fitted. If we are fitting multiple galaxies, + we would have to customize the priors for every single fit, for large samples of galaxies this would take a lot of time! __Approach 2: Reducing Complexity__ @@ -325,7 +320,7 @@ print( "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_4_reducing_complexity" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_bulge_disk_align = search.fit(model=model, analysis=analysis) @@ -354,8 +349,8 @@ Advantages: - - By reducing parameter space`s complexity we again had a higher chance of inferring the global maximum log - likelihood and the time required by the search to do this is reducing. + - By reducing parameter space`s complexity we again had a higher chance of inferring the global maximum log + likelihood and the time required by the search to do this is reduced. - Unlike tuned priors, the search was not specific to one galaxy and we could run it on many galaxy images. Disadvantages: @@ -365,7 +360,7 @@ __Approach 3: Look Harder__ In approaches 1 and 2 we extended our non-linear search an olive branch and helped it find the highest log likelihood -regions of parameter space. In approach 3 ,we're going to tell it to just `look harder`. +regions of parameter space. In approach 3, we're going to tell it to just `look harder`. Every non-linear search has settings which govern how thoroughly it searches parameter space, with the number of live points that was passed to `Nautilus` an example of such a setting. The more thoroughly the search looks, the more likely @@ -374,8 +369,8 @@ We create a more thorough `nautilus` search, that uses `n_live=200`. What these settings are actually changing is discussed in the optional tutorial `HowToGalaxy/chapter_optional/tutorial_searches.ipynb`. -Due to the long run times of this search, we comment it output below so it does not run. Feel free to undo these -comments so the script runs faster. +Due to the long run times of this search, we comment out the fit below so it does not run. Feel free to undo these +comments if you wish to perform the search yourself. """ galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic, disk=ag.lp.Exponential) @@ -392,7 +387,7 @@ print( "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_4_look_harder" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) # result_look_harder = search.fit(model=model, analysis=analysis) @@ -416,7 +411,7 @@ Disadvantage: - - Its potentially expensive. Very expensive. For very complex models, the run times can hours, days, weeks or, dare + - Its potentially expensive. Very expensive. For very complex models, the run times can be hours, days, weeks or, dare I say it, months! So, we can now fit galaxies. And when it fails, we know how to get it to work. diff --git a/scripts/chapter_2_modeling/tutorial_5_linear_profiles.py b/scripts/chapter_2_modeling/tutorial_5_linear_profiles.py index 545311f..cffe2b9 100644 --- a/scripts/chapter_2_modeling/tutorial_5_linear_profiles.py +++ b/scripts/chapter_2_modeling/tutorial_5_linear_profiles.py @@ -53,7 +53,7 @@ we'll use the same galaxy data as the previous tutorial, where: - - The galaxy's bulge is an `Sersic`. + - The galaxy's bulge is a `Sersic`. - The galaxy's disk is an `Exponential`. """ dataset_name = "simple" @@ -169,10 +169,10 @@ For linear light profiles, the log likelihood evaluation increases to around ~0.05 seconds per likelihood evaluation. This is still fast, but it does mean that the fit may take around five times longer to run. -However, because two free parameters have been removed from the model (the `intensity` of the lens bulge and -source bulge), the total number of likelihood evaluations will reduce. Furthermore, the simpler parameter space -likely means that the fit will take less than 10000 per free parameter to converge. This is aided further -by the reduction in `n_live` to 100. +However, because two free parameters have been removed from the model (the `intensity` of the bulge and +disk), the total number of likelihood evaluations will reduce. Furthermore, the simpler parameter space +likely means that the fit will take less than 10000 likelihood evaluations per free parameter to converge. This is +aided further by the reduction in `n_live` to 100. Fits using standard light profiles and linear light profiles therefore take roughly the same time to run. However, the simpler parameter space of linear light profiles means that the model-fit is more reliable, less susceptible to @@ -183,7 +183,7 @@ print( "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_5_linear_light_profile" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_linear_light_profile = search.fit(model=model, analysis=analysis) @@ -242,11 +242,9 @@ - All share the same centre and elliptical components. - The `sigma` size of the Gaussians increases in log10 increments. - -Because `log10(1.0) = 0.0` the first Gaussian `sigma` value is therefore 0.0001, whereas because `log10(10) = 1.0` -the size of the final Gaussian is 1.0. -The equation below has therefore been chosen to provide intuition on the scale of the Gaussians. +The `sigma` values are spaced in log10 increments from a tenth of the pixel scale (0.01") to the mask +radius (3.0"), ensuring the Gaussians span all scales the data constrains. """ total_gaussians = 30 @@ -299,11 +297,11 @@ fit = ag.FitImaging(dataset=dataset, galaxies=galaxies) """ -By plotting the fit, we see that the `Basis` does a reasonable job at capturing the appearance of the lens galaxy. +By plotting the fit, we see that the `Basis` does a reasonable job at capturing the appearance of the galaxy. There are imperfections, but this is because we did not fit the model via a non-linear search in order to determine the optimal values of the Gaussians in the basis. In particular, the Gaussians above were all spherical, when the -lens galaxy is elliptical. +galaxy is elliptical. We rectify this below, where we use a non-linear search to determine the optimal values of the Gaussians! """ @@ -321,7 +319,7 @@ - The centres and elliptical components of each family of Gaussians are all linked together. - The `sigma` size of the Gaussians increases in log10 increments. -The number of free parameters and therefore the dimensionality of the MGe is just N=4. +The number of free parameters and therefore the dimensionality of the MGE is just N=4. """ total_gaussians = 30 @@ -430,7 +428,7 @@ print(model.info) """ -We now fit the model, with just `n_live=50` given the simiplicity of parameter space. +We now fit the model, with just `n_live=50` given the simplicity of parameter space. """ search = af.Nautilus( path_prefix=Path("howtogalaxy", "chapter_2"), @@ -443,7 +441,7 @@ print( "The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_5_basis" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_basis = search.fit(model=model, analysis=analysis) @@ -460,8 +458,8 @@ Visualizing the fit shows that we successfully fit the data to the noise level. Note that the result objects `max_log_likelihood_galaxies` and `max_log_likelihood_fit` automatically convert -all linear light profiles to ordinary light profiles, including every single one of the 20 Gaussians fitted -above. +all linear light profiles to ordinary light profiles, including every single one of the 40 Gaussians fitted +above. This means we can use them directly to perform the visualization below. """ @@ -477,11 +475,11 @@ __Multi Gaussian Expansion Benefits__ Symmetric light profiles (e.g. elliptical Sersics) may leave significant residuals, because they fail to capture -irregular and asymmetric morphological of galaxies (e.g. isophotal twists, an ellipticity which varies radially). +irregular and asymmetric morphological features of galaxies (e.g. isophotal twists, an ellipticity which varies radially). An MGE fully captures these features and can therefore much better represent the emission of complex galaxies. The MGE model can be composed in a way that has fewer non-linear parameters than an elliptical Sersic. In this example, -a groups of Gaussians is used to represent the `bulge` of the galaxy, which in total correspond to just N=4 non-linear +a group of Gaussians is used to represent the `bulge` of the galaxy, which in total correspond to just N=4 non-linear parameters (a `bulge` and `disk` comprising two linear Sersics has N=10 parameters). The MGE model parameterization is also composed such that neither the `intensity` parameters or any of the @@ -494,7 +492,7 @@ __Positive Only Solver__ -Many codes which use linear algebra typically rely on a linear algabra solver which allows for positive and negative +Many codes which use linear algebra typically rely on a linear algebra solver which allows for positive and negative values of the solution (e.g. `np.linalg.solve`), because they are computationally fast. This is problematic, as it means that negative surface brightnesses values can be computed to represent a galaxy's @@ -510,19 +508,19 @@ In addition to the Gaussians used in this example, there is another basis function implemented in PyAutoGalaxy that is commonly used to represent the light of galaxies, called a `Shapelet`. -Shapelets are basis functions with analytic properties that are appropriate for capturing the exponential / disk-like -features of a galaxy. They do so over a wide range of scales, and can often represent features in source galaxies +Shapelets are basis functions with analytic properties that are appropriate for capturing the exponential / disk-like +features of a galaxy. They do so over a wide range of scales, and can often represent features in galaxies that a single Sersic function or MGE cannot. An example using shapelets is given at `autogalaxy_workspace/scripts/imaging/features/shapelets/modeling.py`. - -Feel free to experiment with using shapelets as the galaxy by yourself. However they incur higher computational -overheads than the MGE and include a free parameter which governs the size of the basis functions and therefore source, -slowing down convergence of the non-linear search. We have found that MGEs perform better than shapelets in most -lens modeling problems. -If you have a desire to fit sources with even more complex morphologies we recommend you look at how to reconstruct -sources using pixelizations in the `modeling/features` section or chapter 3 of **HowToGalaxy**. +Feel free to experiment with using shapelets to model the galaxy yourself. However they incur higher computational +overheads than the MGE and include a free parameter which governs the size of the basis functions and therefore galaxy, +slowing down convergence of the non-linear search. We have found that MGEs perform better than shapelets in most +galaxy modeling problems. + +If you have a desire to fit galaxies with even more complex morphologies we recommend you look at how to reconstruct +galaxies using pixelizations in the `modeling/features` section or chapter 3 of **HowToGalaxy**. __Wrap Up__ @@ -531,14 +529,11 @@ and robust. Perhaps the biggest downside to basis functions is that they are only as good as the features they can capture -in the data. For example, a baiss of Gaussians still assumes that they have a well defined centre, but there are -galaxies which may have multiple components with multiple centres (e.g. many star forming knots) which such a -basis cannot catprue. +in the data. For example, a basis of Gaussians still assumes that they have a well defined centre, but there are +galaxies which may have multiple components with multiple centres (e.g. many star forming knots) which such a +basis cannot capture. In chapter 3 of **HowToGalaxy** we introduce non-parametric pixelizations, which reconstruct the data in way that does not make assumptions like a centre and can thus reconstruct even more complex, asymmetric and irregular galaxy morphologies. """ -basis = ag.lp_basis.Basis( - profile_list=gaussian_list, regularization=ag.reg.Constant(coefficient=1.0) -) diff --git a/scripts/chapter_2_modeling/tutorial_6_masking.py b/scripts/chapter_2_modeling/tutorial_6_masking.py index 988a34b..e7b742a 100644 --- a/scripts/chapter_2_modeling/tutorial_6_masking.py +++ b/scripts/chapter_2_modeling/tutorial_6_masking.py @@ -2,7 +2,7 @@ Tutorial 6: Masking =================== -We have learnt everything we need to know about non-linear searches to model a galaxy and infer a good lens +We have learnt everything we need to know about non-linear searches to model a galaxy and infer a good model solution. Now, lets consider masking in more detail, something we have not given much consideration previously. We'll also learn a neat trick to improve the speed and accuracy of a non-linear search. @@ -27,7 +27,7 @@ we'll use the same galaxy data as tutorials 1 & 2, where: - - The galaxy's `LightProfile` is an `Sersic`. + - The galaxy's `LightProfile` is a `Sersic`. """ dataset_name = "simple__sersic" dataset_path = Path("dataset") / "imaging" / dataset_name @@ -89,10 +89,10 @@ """ __Model + Search + Analysis__ -Lets fit the data using this mask, by creating the search as per usuag. Note that the `imaging` data with this mask +Lets fit the data using this mask, by creating the search as per usual. Note that the `imaging` data with this mask applied is passed into the `AnalysisImaging` object, ensuring that this is the mask the model-fit uses. """ -galaxy = af.Model(ag.Galaxy, redshift=1.0, bulge=ag.lp.Sersic) +galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic) model = af.Collection(galaxies=af.Collection(galaxy=galaxy)) search = af.Nautilus( @@ -110,7 +110,7 @@ """ __Discussion__ -So, we can choose the mask we use in a model-fit. We know that we want the mask remove as little of the galaxy's light, +So, we can choose the mask we use in a model-fit. We know that we want the mask to remove as little of the galaxy's light, but is this the 'right' mask? What is the 'right' mask? Maybe we want a bigger mask? a smaller mask? When it comes to choosing a mask, we are essentially balancing two things: computational run-time and accuracy. When we @@ -121,7 +121,7 @@ If run-time was not a consideration we would always choose a bigger mask, for two reasons: 1) The galaxy may have very faint emission that when you choose the mask you simply do not notice. Overly aggressive - masking runs the risk of us inadvertantly masking out some of the galaxy's light, which would otherwise better + masking runs the risk of us inadvertently masking out some of the galaxy's light, which would otherwise better constrain the model! 2) When the data is fitted with a model image, the fit is performed only within the masked region. For certain galaxies @@ -138,16 +138,16 @@ At the end of this chapter, where we introduce 'non-linear search chaining', we will see how chaining makes it possible to use tighter masks in earlier searches to achieve faster run times. -If your data includes the light of additional galaxies nearby you may much have no choice but to use a smaller -circular mask, because it is important these objects do not interfere with the fit. +If your data includes the light of additional galaxies nearby you may have no choice but to use a smaller +circular mask, because it is important these objects do not interfere with the fit. -In fact, you can drawcustom masks that remove their light entirely. You may now wish to checkout -the `autogalaxy_workspace/*/imaging/data_preparation` package. This includes tools for creating custom masks and +In fact, you can draw custom masks that remove their light entirely. You may now wish to checkout +the `autogalaxy_workspace/*/imaging/data_preparation` package. This includes tools for creating custom masks and marking the positions on a galaxy (via a GUI) so you can use them in a model-fit. __Wrap Up__ -There are is one thing you should bare in mind in terms of masking: +There is one thing you should bear in mind in terms of masking: 1) Customizing the mask for the analysis of one galaxy gets the analysis running fast and can provide accurate non-linear sampling. However, for a large sample of galaxies, this high level of customization may take a lot of time. diff --git a/scripts/chapter_2_modeling/tutorial_7_results.py b/scripts/chapter_2_modeling/tutorial_7_results.py index 5fa803f..85201f4 100644 --- a/scripts/chapter_2_modeling/tutorial_7_results.py +++ b/scripts/chapter_2_modeling/tutorial_7_results.py @@ -1,9 +1,9 @@ """ -Tutorial 5: Results +Tutorial 7: Results =================== In the previous tutorials, each search returned a `Result` object, which we used to plot the maximum log likelihood -fit each model-fit. In this tutorial, we'll take a look at the result object in a little more detail. +fit of each model-fit. In this tutorial, we'll take a look at the result object in a little more detail. __Contents__ @@ -25,7 +25,7 @@ """ __Initial Setup__ -Lets use the model-fit performed in tutorial 1 to get a `Result` object. +Lets repeat the simple Sersic model-fit performed earlier in this chapter to get a `Result` object. """ dataset_name = "simple__sersic" dataset_path = Path("dataset") / "imaging" / dataset_name @@ -69,12 +69,12 @@ dataset = dataset.apply_over_sampling(over_sample_size_lp=over_sample_size) model = af.Collection( - galaxies=af.Collection(galaxy=af.Model(ag.Galaxy, redshift=0.5, mass=ag.lp.Sersic)) + galaxies=af.Collection(galaxy=af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic)) ) search = af.Nautilus( path_prefix=Path("howtogalaxy", "chapter_2"), - name="tutorial_1_non_linear_search", + name="tutorial_7_results", unique_tag=dataset_name, n_live=80, n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2. @@ -105,7 +105,7 @@ For example, the `Samples` object contains all of the non-linear search samples, including the parameters of every successful model evaluation and their log likelihood values. These are used for computing information -about the model-fit, such as the most probably parameter estimates and the error inferred for every parameter. +about the model-fit, such as the most probable parameter estimates and the error inferred for every parameter. """ print(result.samples) print("Parameters of 1st Sample:") @@ -122,7 +122,7 @@ `autogalaxy_workspace/*/results` -The results API for CCD imaging data is the same as for other data types (e.g. interferometer, point-soures). This +The results API for CCD imaging data is the same as for other data types (e.g. interferometer). This package can therefore be used to learn the API and then translate to other data types. __Database__ @@ -150,7 +150,7 @@ - Use the `Samples` to produce many different results from the fit, including error estimates on parameters and plots of the probability density function of parameters in 1D and 2D. - - Visualize results, for example the fit to a lens dataset. + - Visualize results, for example the fit to a galaxy dataset. __Wrap Up__ @@ -158,6 +158,6 @@ Even if you are only modeling a small sample of galaxies, if you anticipate using **PyAutoGalaxy** for the long-term I strongly recommend you begin using the database to inspect and analyse your result. -This is because it makes it simple to perform all analyse in a Jupyter notebook, which is the most flexible and +This is because it makes it simple to perform all analysis in a Jupyter notebook, which is the most flexible and versatile way to check results and make figures. """ diff --git a/scripts/chapter_2_modeling/tutorial_9_search_chaining.py b/scripts/chapter_2_modeling/tutorial_9_search_chaining.py index aafb14c..c1cd413 100644 --- a/scripts/chapter_2_modeling/tutorial_9_search_chaining.py +++ b/scripts/chapter_2_modeling/tutorial_9_search_chaining.py @@ -24,7 +24,7 @@ much on searching parameter space for longer, we could end up with search`s that take days, weeks or months to run. In this tutorial, we are going to show how search chaining combines these 3 approaches such that we can fit -complex and realistic models in a way that that can be generalized to many different galaxies. To do this, +complex and realistic models in a way that can be generalized to many different galaxies. To do this, we'll run 2 searches, and chain the model inferred in the first search to the priors of the second search`s model. @@ -64,7 +64,7 @@ we'll use the same galaxy data as tutorial 4 of this chapter, where: - - The galaxy's bulge is an `Sersic`. + - The galaxy's bulge is a `Sersic`. - The galaxy's disk is an `Exponential`. """ dataset_name = "simple" @@ -180,7 +180,7 @@ print( "The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_1 = search_1.fit(model=model_1, analysis=analysis_1) @@ -209,7 +209,7 @@ possibility that there might be a better solution nearby. In contrast, `UniformPrior`'s put hard limits on what values a parameter can or can`t take. It makes it more likely we will accidentally cut-out the global maxima solution. -Note that below the `disk` has become an `Sersic`. +Note that below the `disk` has become a `Sersic`. """ bulge = af.Model(ag.lp_linear.Sersic) disk = af.Model(ag.lp_linear.Sersic) @@ -291,7 +291,7 @@ print( "The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_2 = search_2.fit(model=model_2, analysis=analysis_2) diff --git a/scripts/chapter_optional/tutorial_searches.py b/scripts/chapter_optional/tutorial_searches.py index 40fbe1a..3fc7d0e 100644 --- a/scripts/chapter_optional/tutorial_searches.py +++ b/scripts/chapter_optional/tutorial_searches.py @@ -6,8 +6,8 @@ sampling. In this tutorial, we'll consider how we can change these setting to balance finding the global maxima solution with fast run time. -We will also discuss other types of non-linear searches, such as MCMC and optimizers, which we can use to perform lens -modeling. So far, we have no found any of these alternatives to give anywhere near as robust and efficient results as +We will also discuss other types of non-linear searches, such as MCMC and optimizers, which we can use to perform +modeling. So far, we have not found any of these alternatives to give anywhere near as robust and efficient results as Nautilus, and we recommend users use Nautilus unless they are particularly interested in investigating different model-fitting techniques. @@ -26,11 +26,9 @@ import autofit as af """ -we'll use new galaxying data, where: +we'll use the `simple__sersic` imaging data of a galaxy, where: - - The galaxy's light is an `Sersic`. - - The galaxy's total mass distribution is an `Isothermal` and `ExternalShear`. - - The source galaxy's `LightProfile` is an `Sersic`. + - The galaxy's light is a `Sersic`. """ dataset_name = "simple__sersic" dataset_path = Path("dataset") / "imaging" / dataset_name @@ -59,7 +57,7 @@ ) """ -we'll create and use a smaller 2.0" `Mask2D` again. +we'll create and use a 2.6" `Mask2D`. """ mask = ag.Mask2D.circular( shape_native=dataset.shape_native, pixel_scales=dataset.pixel_scales, radius=2.6 @@ -91,11 +89,11 @@ f_live: -A nested sampling algorithm estimates the *Bayesian Evidence* of the model-fit, which is quantity the non-linear +A nested sampling algorithm estimates the *Bayesian Evidence* of the model-fit, which is a quantity the non-linear search algorithms we introduce later do not. The Bayesian evidence quantifies how well the model as a whole fits the data, following a principle called Occam's Razor (`https://simple.wikipedia.org/wiki/Occam%27s_razor`). This penalizes models for being more complex (e.g. more parameters) and requires that their additional complexity improve -their overall fit to the data compared to a simpler model. By computing the comparing the Bayesian evidence of +their overall fit to the data compared to a simpler model. By comparing the Bayesian evidence of different models one can objectively choose the model that best fits the data. A nested sampling algorithm stops sampling when it estimates that continuing sampling will not increase the Bayesian @@ -106,7 +104,7 @@ A high `f_live` will make the errors estimated on every parameter unreliable and its value must be kept below 0.8 for reliable error estimates. However, when chaining searches, we typically *do not care* about the errors in the first search, therefore setting a high evidence tolerance can be an effective means to make Nautilus converge -faster (we'll estimate reliable errors in the second search when the `f_live is 0.8 or less). +faster (we'll estimate reliable errors in the second search when the `f_live` is 0.8 or less). Lets perform two fits, where: @@ -131,7 +129,7 @@ name="tutorial_searches_slow", unique_tag=dataset_name, n_live=400, - n_batch=50, # GPU lens model fits are batched and run simultaneously, see VRAM section below. + n_batch=50, # GPU model fits are batched and run simultaneously (see chapter 2 tutorial 2). ) analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True) @@ -139,7 +137,7 @@ print( "The non-linear search has begun running - checkout the workspace/output" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_slow = search.fit(model=model, analysis=analysis) @@ -159,17 +157,17 @@ Now lets run the search with fast settings, so we can compare the total number of iterations required. """ search = af.Nautilus( - path_prefix=Path("howtogalaxy", "chapter_2"), + path_prefix=Path("howtogalaxy", "chapter_optional"), name="tutorial_searches_fast", unique_tag=dataset_name, n_live=75, - n_batch=50, # GPU lens model fits are batched and run simultaneously, see VRAM section below. + n_batch=50, # GPU model fits are batched and run simultaneously (see chapter 2 tutorial 2). ) print( "The non-linear search has begun running - checkout the workspace/output" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_fast = search.fit(model=model, analysis=analysis) @@ -257,7 +255,7 @@ model = af.Collection(galaxies=af.Collection(lens=lens, source=source)) search = af.Zeus( - path_prefix=Path("howtogalaxy", "chapter_2"), + path_prefix=Path("howtogalaxy", "chapter_optional"), name="tutorial_searches_zeus", unique_tag=dataset_name, nwalkers=50, @@ -267,7 +265,7 @@ print( "Zeus has begun running - checkout the workspace/output" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_zeus = search.fit(model=model, analysis=analysis) @@ -278,7 +276,7 @@ search = af.Emcee( - path_prefix=Path("howtogalaxy", "chapter_2"), + path_prefix=Path("howtogalaxy", "chapter_optional"), name="tutorial_searches_emcee", unique_tag=dataset_name, nwalkers=50, @@ -288,7 +286,7 @@ print( "The non-linear search has begun running - checkout the workspace/output" " folder for live output of the results, images and model." - " This Jupyter notebook cell with progress once search has completed - this could take some time!" + " This Jupyter notebook cell will progress once search has completed - this could take some time!" ) result_emcee = search.fit(model=model, analysis=analysis) @@ -300,8 +298,8 @@ """ __Wrap Up__ -This tutorial showed how the same model can be fitted using different non-linear searches (e.g. `Nautilus`, `Emcee`, -`PySwarms`). Each search explores parameter space in a different way, and the best choice depends on the +This tutorial showed how the same model can be fitted using different non-linear searches (e.g. `Nautilus`, `Zeus`, +`Emcee`). Each search explores parameter space in a different way, and the best choice depends on the dimensionality and complexity of the model you are fitting. For the vast majority of models in **PyAutoGalaxy**, the default nested sampling search `Nautilus` is recommended, as From 9a1893216b920eff2546825b284084cb6108ef03 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 15:08:47 +0000 Subject: [PATCH 14/15] docs: chapter_optional models a galaxy, not a leftover lens+source system The searches tutorial's model compositions were HowToLens leftovers (lens with Isothermal mass + shear, background source). Both are now a single bulge galaxy matching the simple__sersic dataset, with the MCMC starting-point priors centred on the simulator truth. Runs green under the smoke profile. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- scripts/chapter_optional/tutorial_searches.py | 37 +++---------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/scripts/chapter_optional/tutorial_searches.py b/scripts/chapter_optional/tutorial_searches.py index 3fc7d0e..f5fec37 100644 --- a/scripts/chapter_optional/tutorial_searches.py +++ b/scripts/chapter_optional/tutorial_searches.py @@ -117,10 +117,7 @@ """ model = af.Collection( galaxies=af.Collection( - lens=af.Model( - ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic, mass=ag.mp.Isothermal - ), - source=af.Model(ag.Galaxy, redshift=1.0, bulge=ag.lp.Sersic), + galaxy=af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic), ) ) @@ -219,40 +216,18 @@ I've included an example runs of Emcee and Zeus below, where the model is set up using `UniformPriors` to give the starting point of the MCMC walkers. """ -lens_bulge = af.Model(ag.lp.Sersic) -lens_bulge.centre.centre_0 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1) -lens_bulge.centre.centre_1 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1) -lens_bulge.ell_comps.ell_comps_0 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3) -lens_bulge.ell_comps.ell_comps_1 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3) -lens_bulge.intensity = af.UniformPrior(lower_limit=0.5, upper_limit=1.5) -lens_bulge.effective_radius = af.UniformPrior(lower_limit=0.2, upper_limit=1.6) -lens_bulge.sersic_index = af.UniformPrior(lower_limit=3.0, upper_limit=5.0) - - -mass = af.Model(ag.mp.Isothermal) -mass.centre.centre_0 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1) -mass.centre.centre_1 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1) -mass.ell_comps.ell_comps_0 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3) -mass.ell_comps.ell_comps_1 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3) -mass.einstein_radius = af.UniformPrior(lower_limit=1.0, upper_limit=2.0) - -shear = af.Model(ag.mp.ExternalShear) -shear.gamma_1 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1) -shear.gamma_2 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1) - bulge = af.Model(ag.lp.Sersic) bulge.centre.centre_0 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1) bulge.centre.centre_1 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1) bulge.ell_comps.ell_comps_0 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3) bulge.ell_comps.ell_comps_1 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3) -bulge.intensity = af.UniformPrior(lower_limit=0.1, upper_limit=0.5) -bulge.effective_radius = af.UniformPrior(lower_limit=0.0, upper_limit=0.4) -bulge.sersic_index = af.UniformPrior(lower_limit=0.5, upper_limit=2.0) +bulge.intensity = af.UniformPrior(lower_limit=0.5, upper_limit=1.5) +bulge.effective_radius = af.UniformPrior(lower_limit=0.2, upper_limit=1.6) +bulge.sersic_index = af.UniformPrior(lower_limit=3.0, upper_limit=5.0) -lens = af.Model(ag.Galaxy, redshift=0.5, mass=mass, shear=shear) -source = af.Model(ag.Galaxy, redshift=1.0, bulge=bulge) +galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge) -model = af.Collection(galaxies=af.Collection(lens=lens, source=source)) +model = af.Collection(galaxies=af.Collection(galaxy=galaxy)) search = af.Zeus( path_prefix=Path("howtogalaxy", "chapter_optional"), From 2f55ed890101495296c17a34293b8557cbcec874 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 15:09:06 +0000 Subject: [PATCH 15/15] build: regenerate notebooks and navigator catalogue for the restructure Notebooks rebuilt from the restructured scripts via PyAutoHands generate.py (new chapter 3/4 trees, updated chapters 1-2, simulator notebooks, setup_notebook now active via the PyAutoHands fix); llms-full.txt and workspace_index.json regenerated. check_navigator passes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BxKfSZisjnEn91LRGkN4SU --- llms-full.txt | 60 +- notebooks/chapter_1_introduction/README.md | 2 +- .../tutorial_0_visualization.ipynb | 41 +- .../tutorial_1_grids_and_galaxies.ipynb | 46 +- .../tutorial_2_data.ipynb | 74 +- .../tutorial_3_fitting.ipynb | 25 +- .../tutorial_4_methods.ipynb | 2 +- .../tutorial_5_summary.ipynb | 36 +- .../tutorial_10_prior_passing.ipynb | 81 +- .../tutorial_1_non_linear_search.ipynb | 36 +- .../tutorial_2_practicalities.ipynb | 46 +- .../tutorial_3_realism_and_complexity.ipynb | 30 +- .../tutorial_4_dealing_with_failure.ipynb | 47 +- .../tutorial_5_linear_profiles.ipynb | 77 +- .../tutorial_6_masking.ipynb | 28 +- .../tutorial_7_results.ipynb | 20 +- .../tutorial_8_need_for_speed.ipynb | 150 ++- .../tutorial_9_search_chaining.ipynb | 107 +- .../tutorial_1_pixelizations.ipynb | 14 +- .../tutorial_2_mappers.ipynb | 27 +- .../tutorial_3_inversions.ipynb | 62 +- .../tutorial_4_bayesian_regularization.ipynb | 28 +- .../tutorial_5_bayesian_formalism.ipynb | 972 ++++++++++++++++++ .../tutorial_6_model_fit.ipynb | 76 +- .../tutorial_1_extra_galaxies.ipynb | 702 +++++++++++++ .../tutorial_2_multi_galaxy.ipynb | 637 ++++++++++++ .../tutorial_3_cluster.ipynb | 650 ++++++++++++ .../chapter_optional/tutorial_searches.ipynb | 79 +- notebooks/simulators/cluster.ipynb | 462 +++++++++ notebooks/simulators/extra_galaxy.ipynb | 471 +++++++++ notebooks/simulators/interferometer.ipynb | 367 +++++++ notebooks/simulators/sersic.ipynb | 4 +- notebooks/simulators/sersic_x2.ipynb | 11 +- notebooks/simulators/simple.ipynb | 12 +- workspace_index.json | 286 ++++-- 35 files changed, 5165 insertions(+), 603 deletions(-) create mode 100644 notebooks/chapter_3_pixelizations/tutorial_5_bayesian_formalism.ipynb create mode 100644 notebooks/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.ipynb create mode 100644 notebooks/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.ipynb create mode 100644 notebooks/chapter_4_scaling_up_galaxies/tutorial_3_cluster.ipynb create mode 100644 notebooks/simulators/cluster.ipynb create mode 100644 notebooks/simulators/extra_galaxy.ipynb create mode 100644 notebooks/simulators/interferometer.ipynb diff --git a/llms-full.txt b/llms-full.txt index 990bbf9..8865a60 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -12,18 +12,20 @@ AUTO-GENERATED by PyAutoHands — do not edit by hand; regenerate with generate. - [Tutorial 0: Visualization](scripts/chapter_1_introduction/tutorial_0_visualization.py): In this tutorial, we quickly cover visualization in **PyAutoGalaxy** and make sure images display clearly in your Jupyter notebook and on your computer screen. - Contents: Directories, Dataset, Plot Customization, Subplots, Visuals, Wrap Up - [HowToGalaxy: Introduction](scripts/chapter_1_introduction/tutorial_1_grids_and_galaxies.py): Nearly a century ago, Edwin Hubble famously classified galaxies into three distinct groups: ellipticals, spirals and irregulars. He produced a diagram of these galaxies, called the Hubble Tuning Fork, which is shown below and still discussed by astronomers in the modern day: - - Contents: Grids, Geometry, Light Profiles, Galaxies, One Dimension Projection, Unit Conversion, Wrap Up + - Contents: Grids, Geometry, Light Profiles, One Dimension Projection, Galaxies, Unit Conversion, Wrap Up - [Tutorial 2: Data](scripts/chapter_1_introduction/tutorial_2_data.py): In the previous tutorial, we used light profiles to create images of galaxies. However, those images don't accurately represent what we would observe through a telescope. - - Contents: Initial Setup, Optics Blurring, Poisson Noise, Background Sky, Simulator, Output, Wrap Up + - Contents: Initial Setup, Optics Blurring, Poisson Noise, Background Sky, Simulator, Output, Interferometer Data, Wrap Up - [Tutorial 3: Fitting](scripts/chapter_1_introduction/tutorial_3_fitting.py): In previous tutorials, we used light profiles to create simulated images of galaxies and visualized how these images would appear when captured by a CCD detector on a telescope like the Hubble Space Telescope. - Contents: Dataset, Mask, Masked Grid, Fitting, Incorrect Fit, Model Fitting, Wrap Up - [Tutorial 4: Methods](scripts/chapter_1_introduction/tutorial_4_methods.py): This tutorial is not written yet, but will explain in more detail the different methods used to fit and analyse galaxies. - Contents: Wrap Up -- [Tutorial 9: Summary](scripts/chapter_1_introduction/tutorial_5_summary.py): In this chapter, we have learnt that: +- [Tutorial 5: Summary](scripts/chapter_1_introduction/tutorial_5_summary.py): In this chapter, we have learnt that: - Contents: Initial Setup, Object Composition, Visualization, Code Design, Source Code, Wrap Up ## chapter_2_modeling +- [Tutorial 10: Prior Passing](scripts/chapter_2_modeling/tutorial_10_prior_passing.py): In the previous tutorial, we used non-linear search chaining to break the model-fitting procedure down into two non-linear searches. This used an initial search to fit a simple model, whose results were used to tune and initialize the priors of a more complex model that was fitted by the second search. + - Contents: Initial Setup, Dataset Auto-Simulation, Model, Search, Result (Search 1), Prior Passing, Result, Wrap Up, Detailed Explanation Of Prior Passing, EXAMPLE - [Tutorial 1: Non-linear Search](scripts/chapter_2_modeling/tutorial_1_non_linear_search.py): The starting point for most scientific analysis conducted by an Astronomer is that they have observations of a galaxy using a telescope like the Hubble Space Telescope, and seek to learn about the galaxy and the Universe from these observations. With **PyAutoGalaxy**, we seek to learn about the galaxy's structure and morphology, asking questions like how big is the galaxy, is it disky or bulgy, and how is its light distributed? - Contents: Parameter Space, Non-Linear Search, Search Types, Deeper Background, Data, Model, Priors, Analysis, Searches, Maximum Likelihood Estimation (MLE), Markov Chain Monte Carlo (MCMC), Nested Sampling, Result, Samples, Customizing Searches, Wrap Up - [Tutorial 2: Practicalities](scripts/chapter_2_modeling/tutorial_2_practicalities.py): In the last tutorial, we introduced foundational statistical concepts essential for model-fitting, such as parameter spaces, likelihoods, priors, and non-linear searches. Understanding these statistical concepts is crucial for performing model fits effectively. @@ -33,34 +35,38 @@ AUTO-GENERATED by PyAutoHands — do not edit by hand; regenerate with generate. - Contents: Initial Setup, Approach 1: Prior Tuning, Approach 2: Reducing Complexity, Approach 3: Look Harder - [Tutorial 5: Linear Profiles](scripts/chapter_2_modeling/tutorial_5_linear_profiles.py): In the previous tutorial we learned how to balance model complexity with our non-linear search in order to infer accurate model solutions and avoid failure. We saw how in order to fit a model accurately one may have to parameterize and fit a simpler model with fewer non-linear parameters, at the expense of fitting the data less accurately. - Contents: Initial Setup, Linear Light Profiles, Run Time, Result, Intensities, Visualization, Basis, Model Fit, Disk MGE, Multi Gaussian Expansion Benefits, Positive Only Solver, Other Basis Functions, Wrap Up -- [Tutorial 6: Masking](scripts/chapter_2_modeling/tutorial_6_masking.py): We have learnt everything we need to know about non-linear searches to model a galaxy and infer a good lens model solution. Now, lets consider masking in more detail, something we have not given much consideration previously. We'll also learn a neat trick to improve the speed and accuracy of a non-linear search. +- [Tutorial 6: Masking](scripts/chapter_2_modeling/tutorial_6_masking.py): We have learnt everything we need to know about non-linear searches to model a galaxy and infer a good model solution. Now, lets consider masking in more detail, something we have not given much consideration previously. We'll also learn a neat trick to improve the speed and accuracy of a non-linear search. - Contents: Initial Setup, Mask, Model + Search + Analysis, Discussion, Wrap Up -- [Tutorial 5: Results](scripts/chapter_2_modeling/tutorial_7_results.py): In the previous tutorials, each search returned a `Result` object, which we used to plot the maximum log likelihood fit each model-fit. In this tutorial, we'll take a look at the result object in a little more detail. +- [Tutorial 7: Results](scripts/chapter_2_modeling/tutorial_7_results.py): In the previous tutorials, each search returned a `Result` object, which we used to plot the maximum log likelihood fit of each model-fit. In this tutorial, we'll take a look at the result object in a little more detail. - Contents: Initial Setup, Galaxies & Fit, Samples, Workspace, Database, Wrap Up - [Tutorial 8: Need For Speed](scripts/chapter_2_modeling/tutorial_8_need_for_speed.py): In this chapter, we have learnt how to model galaxies and how to balance complexity and realism to ensure that we infer a good model. - - Contents: Searching Non-linear Parameter Space, Algorithmic Optimization, Data Quantity, Wrap Up + - Contents: Searching Non-linear Parameter Space, Cost Per Evaluation, JAX, Run Time Estimation, Data Quantity, Wrap Up +- [Tutorial 9: Search Chaining](scripts/chapter_2_modeling/tutorial_9_search_chaining.py): Throughout this chapter, we have fitted the data using just one non-linear search. The previous tutorial discussed the run-time cost of doing this: with a single search, the dimensionality of the model, the priors on its parameters and the search settings must all be juggled at once, leaving us little flexibility to trade them off against one another. + - Contents: Initial Setup, Dataset Auto-Simulation, Model, Search + Analysis, Result, Prior Passing, Result, Wrap Up -## chapter_3_search_chaining +## chapter_3_pixelizations -- [Tutorial 1: Search Chaining](scripts/chapter_3_search_chaining/tutorial_1_search_chaining.py): In chapter 2, we learnt how to perform modeling using a non-linear search. In all of the tutorials, we fitted the data using just one non-linear search. In this chapter, we introduce a technique called 'non-linear search chaining', which fits a model using a sequence of non-linear searches. The initial searches fit simpler models whose parameter spaces can be more accurately and efficiently sampled. The results of this search are then passed to later searches which fit models of gradually increasing complexity. - - Contents: Initial Setup, Model, Search + Analysis, Result, Prior Passing, Result, Wrap Up -- [Tutorial 2: Prior Passing](scripts/chapter_3_search_chaining/tutorial_2_prior_passing.py): In the previous tutorial, we used non-linear search chaining to break the model-fitting procedure down into two non-linear searches. This used an initial search to fit a simple model, whose results were used to tune and initialize the priors of a more complex model that was fitted by the second search. - - Contents: Initial Setup, Model, Search, Result (Search 1), Prior Passing, Result, Wrap Up, Detailed Explanation Of Prior Passing, EXAMPLE -- [Tutorial 3: Two Galaxies](scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py): Up to now, all the images we've fitted had one galaxy. However, we saw in chapter 1 that our galaxies object can consist of multiple galaxies which each contribute to the overall emission. Multi-galaxy systems are challenging to model, because they add an extra 5-10 parameters to the non-linear search per galaxy and, more problematically, the degeneracies between the parameters of the light profiles of the galaxies can be severe. - - Contents: Initial Setup, Paths, Search Chaining Approach, Model + Search + Analysis + Model-Fit (Search 1), Result (Search 1), Model (Search 2), Search + Analysis + Model-Fit (Search 2), Result (Search 2), Model + Search + Analysis + Model-Fit (Search 4), Result (Search 3), Wrap Up +- [Tutorial 1: Pixelizations](scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py): In the previous chapters, we used light profiles to model the light of a galaxy, where the light profile was an analytic description of how the luminosity varies as a function of radius. + - Contents: Initial Setup, Mesh, Wrap Up +- [Tutorial 2: Mappers](scripts/chapter_3_pixelizations/tutorial_2_mappers.py): In the previous tutorial, we used a pixelization to create a `Mapper`. However, it was not clear what a `Mapper` does, why it was called a mapper and whether it was mapping anything at all! + - Contents: Initial Setup, Dataset Auto-Simulation, Mappers, Mask, Wrap Up +- [Tutorial 3: Inversions](scripts/chapter_3_pixelizations/tutorial_3_inversions.py): In the previous two tutorials, we introduced: + - Contents: Initial Setup, Dataset Auto-Simulation, Pixelization, Positive Only Solver, Detailed Explanation, Wrap Up +- [Tutorial 4: Bayesian Regularization](scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py): So far, we have: + - Contents: Initial Setup, Dataset Auto-Simulation, Convenience Function, Pixelization, Regularization, Bayesian Evidence, Non-Linear and Linear, Detailed Description, Wrap Up +- [Tutorial 5: Bayesian Formalism](scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py): In tutorials 1 to 4, we built an intuition for how pixelized galaxy reconstruction works: pixelizations place a pixel-grid over the galaxy's image, mappers pair pixelization pixels with image-pixels, inversions solve for the pixel fluxes that best fit the data, and regularization smooths the solution within a Bayesian framework. + - Contents: Initial Setup, Dataset Auto-Simulation, Mask, Over Sampling, Mesh Shape, Galaxy, Image Grid, Mesh Pixel Centres, Interpolation, Mapper, Mapping Matrix, Blurred Mapping Matrix, Data Vector (D), Curvature Matrix (F), Unregularized Solve, Regularization Matrix (H), Galaxy Reconstruction (s), Image Reconstruction, Likelihood Function, Chi Squared, Regularization Term, Complexity Terms, Noise Normalization Term, Log Evidence, Fit, Wrap Up +- [Tutorial 6: Model Fit](scripts/chapter_3_pixelizations/tutorial_6_model_fit.py): In the previous tutorials we used an inversion to reconstruct a complex galaxy. However, from the perspective of a scientific analysis, it is not clear how useful this was. When we fit a galaxy with light profiles, we learn about its brightness (`intensity`), size (`effective_radius`), compactness (`sersic_index`), etc. + - Contents: Initial Setup, Dataset Auto-Simulation, Model + Search + Analysis + Model-Fit (Search 1), Mesh Shape, Model + Search + Analysis + Model-Fit (Search 2), Model + Search (Search 3), Wrap Up -## chapter_4_pixelizations +## chapter_4_scaling_up_galaxies -- [Tutorial 1: pixelizations](scripts/chapter_4_pixelizations/tutorial_1_pixelizations.py): In the previous chapters, we used light profiles to model the light of a galaxy, where the light profile was an analytic description of how the luminosity varies as a function of radius. - - Contents: Initial Setup, Mesh, Wrap Up -- [Tutorial 2: Mappers](scripts/chapter_4_pixelizations/tutorial_2_mappers.py): In the previous tutorial, we used a pixelization to create made a `Mapper`. However, it was not clear what a `Mapper` does, why it was called a mapper and whether it was mapping anything at all! - - Contents: Initial Setup, Mappers, Mask, Wrap Up -- [Tutorial 3: Inversions](scripts/chapter_4_pixelizations/tutorial_3_inversions.py): In the previous two tutorials, we introduced: - - Contents: Initial Setup, Pixelization, Positive Only Solver, Wrap Up, Detailed Explanation -- [Tutorial 4: Bayesian Regularization](scripts/chapter_4_pixelizations/tutorial_4_bayesian_regularization.py): So far, we have: - - Contents: Initial Setup, Convenience Function, Pixelization, Regularization, Bayesian Evidence, Non-Linear and Linear, Detailed Description -- [Tutorial 5: Model-Fit](scripts/chapter_4_pixelizations/tutorial_5_model_fit.py): In the previous tutorials we used an inversion to reconstruct a complex galaxy. However, from the perspective of a scientific analysis, it is not clear how useful this was. When I fit a galaxy with light profiles, I learn about its brightness (`intensity`), size (`effective_radius`), compactness (`sersic_index`), etc. - - Contents: Initial Setup, Model + Search + Analysis + Model-Fit (Search 1), Mesh Shape, Model + Search + Analysis + Model-Fit (Search 2), Model + Search (Search 3), Wrap Up +- [Tutorial 1: Extra Galaxies](scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py): Welcome to chapter 4 of **HowToGalaxy**, where we scale up galaxy modeling beyond a single galaxy. + - Contents: Initial Setup, Dataset Auto-Simulation, The Decision, Mask, Approach 1 Noise Scaling, Noise Scaling Fit, Approach 2 Extra Galaxies Model, Extra Galaxy Centres, Extra Galaxies Model Composition, Extra Galaxies Fit, Which Approach When, Wrap Up +- [Tutorial 2: Multi-Galaxy Blends](scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py): In the previous tutorial, we learned how to deal with extra galaxies near the galaxy we care about — nuisance objects whose light contaminates the data but which are not themselves the subject of our study. We removed their emission from the analysis, or gave them a heavily restricted model, and the single galaxy we were studying remained the star of the show. + - Contents: Initial Setup, Dataset Auto-Simulation, Mask, Over Sampling, Why Not Fit Them Separately?, Model, Fixing the Centres, Model Fit, Result, Light Decomposition Degeneracy, Wrap Up +- [Tutorial 3: Cluster](scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py): Throughout this chapter we have been scaling up: from a single galaxy with extra galaxies nearby, to blended multi-galaxy systems where every galaxy received its own free light model. + - Contents: The Scaling Problem, Dataset, Dataset Auto-Simulation, Member Catalogue, Masking, Model, Search + Analysis, Model Fit, Result, Per-Member Results, Refinements, Wrap Up ## chapter_optional @@ -69,6 +75,12 @@ AUTO-GENERATED by PyAutoHands — do not edit by hand; regenerate with generate. ## simulators +- [Simulator: Cluster](scripts/simulators/cluster.py): This script simulates `Imaging` of a cluster field: a brightest cluster galaxy (BCG) surrounded by ten lower-luminosity member galaxies. It is used in chapter 4 of the **HowToGalaxy** lectures, where the member population is modeled via a **catalogue**: a CSV of member centres and luminosities whose photometry pins the faint galaxies while only a shared normalization stays free. + - Contents: Dataset Paths, Grid, Galaxies, Output, Member Catalogue CSV, Visualize, Plane Output +- [Simulator: Extra Galaxy](scripts/simulators/extra_galaxy.py): This script simulates `Imaging` of a galaxy using light profiles where: + - Contents: Dataset Paths, Grid, Galaxies, Output, Visualize, Mask Extra Galaxies, Plane Output, Extra Galaxies Centres +- [Simulator: Interferometer](scripts/simulators/interferometer.py): This script simulates `Interferometer` data of a galaxy, as would be observed by a radio or sub-mm interferometer like ALMA or the JVLA, where: + - Contents: Dataset Paths, Simulate, Galaxies, Output, Visualize, Plane Output - [Simulator: Sersic](scripts/simulators/sersic.py): This script simulates `Imaging` of a galaxy using light profiles where: - Contents: Dataset Paths, Grid, Galaxies, Output, Visualize, Plane Output - [Simulator: Sersic x2](scripts/simulators/sersic_x2.py): This script simulates `Imaging` of two galaxies where: diff --git a/notebooks/chapter_1_introduction/README.md b/notebooks/chapter_1_introduction/README.md index 8869dad..debdb91 100644 --- a/notebooks/chapter_1_introduction/README.md +++ b/notebooks/chapter_1_introduction/README.md @@ -1,4 +1,4 @@ -In chapter 1, we introduce you to strong gravitational lensing and the core **PyAutoGalaxy** API. +In chapter 1, we introduce you to galaxies, the light they emit and the core **PyAutoGalaxy** API. **Colab** links to every tutorial are included. diff --git a/notebooks/chapter_1_introduction/tutorial_0_visualization.ipynb b/notebooks/chapter_1_introduction/tutorial_0_visualization.ipynb index ef67110..921798a 100644 --- a/notebooks/chapter_1_introduction/tutorial_0_visualization.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_0_visualization.ipynb @@ -63,7 +63,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()" + "from autogalaxy import setup_notebook; setup_notebook()" ], "outputs": [], "execution_count": null @@ -186,12 +186,12 @@ "\n", "Does the figure display correctly on your computer screen?\n", "\n", - "If not, you can customize common matplotlib options by passing them directly to `plot_array`:\n", + "If not, you can customize common plot options by passing them directly to `plot_array`:\n", "\n", " - `title=`: Set the figure title.\n", - " - `figsize=`: Control the figure size as a `(width, height)` tuple.\n", " - `colormap=`: Set the matplotlib colormap name (e.g. `\"jet\"`, `\"gray\"`).\n", - " - `xlabel=`, `ylabel=`: Override the default axis labels." + " - `use_log10=`: Plot the image on a log10 scale, revealing faint features.\n", + " - `vmin=`, `vmax=`: Override the minimum and maximum values of the colormap." ] }, { @@ -210,32 +210,30 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Many matplotlib options can be customized, but for now we're only concerned with making sure figures display clear in\n", + "Many plot options can be customized, but for now we're only concerned with making sure figures display clearly in\n", "your Jupyter Notebooks. Nevertheless, a comprehensive API reference guide of all available plot arguments can\n", "be found in the `autogalaxy_workspace/*/guides/plot` package. You should check this out once you are more familiar with\n", "**PyAutoGalaxy**.\n", "\n", - "Ideally, we would not specify a `figsize` every time we plot an image. Fortunately, default values can be fully\n", + "Ideally, we would not specify these options every time we plot an image. Fortunately, default values can be fully\n", "customized via the config files.\n", "\n", "Checkout the `general.yaml` file in `autogalaxy_workspace/config/visualize`.\n", "\n", - "All default matplotlib values are here. There are a lot of entries, so lets focus on whats important for displaying\n", - "figures:\n", + "All default visualization values are here. There are a lot of entries, so lets focus on whats important for\n", + "displaying figures:\n", "\n", - " - mat_wrap.yaml -> Figure -> figure: -> figsize\n", - " - mat_wrap.yaml -> YLabel -> figure: -> fontsize\n", - " - mat_wrap.yaml -> XLabel -> figure: -> fontsize\n", - " - mat_wrap.yaml -> TickParams -> figure: -> labelsize\n", - " - mat_wrap.yaml -> YTicks -> figure: -> labelsize\n", - " - mat_wrap.yaml -> XTicks -> figure: -> labelsize\n", + " - general.yaml -> general -> backend: the matplotlib backend used for visualization.\n", + " - general.yaml -> colormap: the default colormap of 2D plots.\n", + " - general.yaml -> subplot_shape_to_figsize_factor: controls the size of subplot figures.\n", + " - general.yaml -> colorbar: the font sizes of colorbar labels.\n", "\n", "Don't worry about all the other files or options listed for now, as they'll make a lot more sense once you are familiar\n", "with **PyAutoGalaxy**.\n", "\n", "If you had to change any of the above settings to get the figures to display clearly, you should update their values\n", "in the corresponding config files above (you will need to reset your Jupyter notebook server for these changes to\n", - "take effect, so make sure you have the right values using the `figsize` argument in the cell above beforehand!).\n", + "take effect, so make sure you have the right values using the arguments in the cell above beforehand!).\n", "\n", "__Subplots__\n", "\n", @@ -260,11 +258,12 @@ "source": [ "__Visuals__\n", "\n", - "Visuals can be added to any figure by passing them as keyword arguments directly to `plot_array`.\n", + "Visuals can be added to any figure by passing them as keyword arguments directly to `plot_array`, for\n", + "example `grid=`, `positions=` and `lines=`. The `autogalaxy_workspace/*/guides/plot` examples illustrate every\n", + "overlay argument.\n", "\n", - "For example, we can plot a mask on the image above by passing `mask=mask`.\n", - "\n", - "The `visuals` example illustrates every overlay argument, for example `mask=`, `grid=`, `positions=`, `lines=`, etc." + "Some visuals appear automatically. For example, if we apply a mask to the dataset, figures of it zoom into the\n", + "unmasked region and only show the unmasked data \u2014 as shown by the plot below, which applies an annular mask." ] }, { @@ -278,7 +277,9 @@ " outer_radius=3.0,\n", ")\n", "\n", - "aplt.plot_array(array=dataset.data, title=\"Data\")" + "dataset = dataset.apply_mask(mask=mask)\n", + "\n", + "aplt.plot_array(array=dataset.data, title=\"Data With Annular Mask\")" ], "outputs": [], "execution_count": null diff --git a/notebooks/chapter_1_introduction/tutorial_1_grids_and_galaxies.ipynb b/notebooks/chapter_1_introduction/tutorial_1_grids_and_galaxies.ipynb index 335972e..de4a829 100644 --- a/notebooks/chapter_1_introduction/tutorial_1_grids_and_galaxies.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_1_grids_and_galaxies.ipynb @@ -11,7 +11,7 @@ "ellipticals, spirals and irregulars. He produced a diagram of these galaxies, called the Hubble Tuning Fork, which\n", "is shown below and still discussed by astronomers in the modern day:\n", "\n", - "![HubbleTuning](https://github.com/PyAutoLabs/autogalaxy_workspace/blob/main/scripts/chapter_1_introduction/HubbleTuningFork.jpg)\n", + "![HubbleTuning](https://github.com/PyAutoLabs/HowToGalaxy/blob/main/scripts/chapter_1_introduction/HubbleTuningFork.jpg)\n", "\n", "To make his diagram, Hubble looked at images of each galaxy in his sample, and subjectively judged by eye how\n", "to classify it. Today, Astronomers use computer software, statistical algorithms and image processing techniques to\n", @@ -39,12 +39,12 @@ "- **Grids:** Create a uniform grid of (y,x) coordinates and show how it can be used to measure the light of a galaxy.\n", "- **Geometry:** How to shift and rotate a grid, and convert it to elliptical coordinates.\n", "- **Light Profiles:** Using light profiles, analytic functions that describe how a galaxy's light is distributed.\n", - "- **Galaxies:** Creating galaxies containing light profiles and computing the image of a galaxy.\n", "- **One Dimension Projection:** Create projected 2D radial grids for 1D profile calculations.\n", + "- **Galaxies:** Creating galaxies containing light profiles and computing the image of a galaxy.\n", "- **Unit Conversion:** Converting angular distances to physical distances using cosmology.\n", "- **Wrap Up:** Summary of the key concepts covered in this tutorial.\n", "\n", - "The imports below are required to run the howtogalaxy tutorials in a Jupiter notebook. They also import the\n", + "The imports below are required to run the howtogalaxy tutorials in a Jupyter notebook. They also import the\n", "`autogalaxy` package and the `autogalaxy.plot` module which are used throughout the tutorials." ] }, @@ -91,7 +91,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", @@ -244,7 +244,7 @@ "cell_type": "code", "metadata": {}, "source": [ - "centre = (0.3, 0.5) # Shifting the grid to be centered at y=1.0\", x=2.0\".\n", + "centre = (0.3, 0.5) # Shifting the grid to be centered at y=0.3\", x=0.5\".\n", "\n", "grid_shifted = grid\n", "grid_shifted[:, 0] = grid_shifted[:, 0] - centre[0] # Shift in y-direction.\n", @@ -602,10 +602,10 @@ "source": [ "__One Dimension Projection__\n", "\n", - "We often want to calculative 1D quantities of a light profile, for example to plot how its light changes as\n", + "We often want to calculate 1D quantities of a light profile, for example to plot how its light changes as\n", "a function of radius.\n", "\n", - "To do this, we must still input a 2D grid into the `image_2d_from` method, therefore we create a project 2D \n", + "To do this, we must still input a 2D grid into the `image_2d_from` method, therefore we create a projected 2D\n", "radial grid as follows which has shape [Number_of_1d_coordinates, 2] and where all [:,0] entries are the same.\n", "\n", "A simple example of such a grid is as follows with 4 1D coordinates is:" @@ -634,14 +634,10 @@ "As in this example, we often already have a 2D grid we are using to calculate images of a light profile\n", "and it would be convenient to simply create `grid_2d_projected` from that.\n", "\n", - "For example, we may want the project grid which traces it major axis in uniform radial steps.\n", + "For example, we may want the projected grid which traces the light profile's major axis in uniform radial steps.\n", "\n", - "This is easily computed using the `grid_2d_radial_project_from` function and passing the `centre` and `angle`\n", - "of a light profile we can make it align with the light profile itself.\n", - "\n", - "Note how in this example the two galaxy bulges are not rotationally aligned but we aligned the projected\n", - "grid with the first galaxy. The centres are aligned, but if they were not that would cause similar\n", - "issues." + "This is easily computed using the `grid_2d_radial_projected_from` function: by passing the `centre` and `angle`\n", + "of a light profile we can make the projected grid align with the light profile itself." ] }, { @@ -661,9 +657,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can now plot the 1D radial profile of the light profile. This profile shows how the intensity of the light \n", - "changes as a function of distance from the profile's center. This is a more informative way to visualize the light p\n", - "rofile's distribution.\n", + "We can now plot the 1D radial profile of the light profile. This profile shows how the intensity of the light\n", + "changes as a function of distance from the profile's center. This is a more informative way to visualize the light\n", + "profile's distribution.\n", "\n", "When we plot 1D quantities, we do not use built-in plotting functions as in 2D, but instead use standard\n", "matplotlib functionality.\n", @@ -695,7 +691,7 @@ "Since galaxy light distributions often cover a wide range of values, they are typically better visualized on a log10 \n", "scale. This approach helps highlight details in the faint outskirts of a light profile.\n", "\n", - "The `MatPlot2D` object has a `use_log10` option that applies this transformation automatically. Below, you can see \n", + "The plotting functions have a `use_log10` option that applies this transformation automatically. Below, you can see\n", "that the image plotted in log10 space reveals more details." ] }, @@ -856,8 +852,8 @@ "and disk profiles in this example share the same `centre`, meaning that plotting them together on the same 1D plot \n", "shows how they vary relative to one another. \n", "\n", - "If the `centre` of the profiles were different, when you make the 1D plot you would need to decide if you should the\n", - "profiles offset from one another or plot them both from zero." + "If the `centre` of the profiles were different, when you make the 1D plot you would need to decide if you should plot\n", + "the profiles offset from one another or plot them both from zero." ] }, { @@ -1020,12 +1016,10 @@ "metadata": {}, "source": [ "This `kpc_per_arcsec` can be used as a conversion factor between arcseconds and kiloparsecs when plotting images of\n", - "galaxies.\n", - "\n", - "We compute this value and plot the image in converted units of kiloparsecs.\n", + "galaxies, for example by converting plot tick labels to kiloparsecs.\n", "\n", - "This passes the plotting modules `Units` object a `ticks_convert_factor` and manually specified the new units of the\n", - "plot ticks." + "The `autogalaxy_workspace/*/guides/units` examples show how to perform these conversions when plotting; below we\n", + "simply plot the image in its default units of arc-seconds." ] }, { @@ -1084,7 +1078,7 @@ "on a higher resolution grid than the image data to ensure the calculation is accurate. \n", "\n", "For a new user, the details of over-sampling are not important, therefore just be aware that all calculations use an\n", - "adaptive over sampling scheme which high accuracy across all use cases.\n", + "adaptive over sampling scheme which ensures high accuracy across all use cases.\n", "\n", "Once you are more experienced, you should read up on over-sampling in more detail via \n", "the `autogalaxy_workspace/*/guides/advanced/over_sampling.ipynb` notebook." diff --git a/notebooks/chapter_1_introduction/tutorial_2_data.ipynb b/notebooks/chapter_1_introduction/tutorial_2_data.ipynb index 49e3d57..9466a93 100644 --- a/notebooks/chapter_1_introduction/tutorial_2_data.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_2_data.ipynb @@ -31,6 +31,7 @@ "- **Background Sky:** Add background sky light that introduces noise across the entire image.\n", "- **Simulator:** Use the SimulatorImaging object to simulate imaging data with all effects combined.\n", "- **Output:** Save the simulated data to .fits files for use in future tutorials.\n", + "- **Interferometer Data:** Radio / sub-mm interferometers like ALMA observe visibilities in the uv-plane rather than images.\n", "- **Wrap Up:** Summary of how CCD imaging data is simulated." ] }, @@ -77,7 +78,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from pathlib import Path\n", @@ -309,8 +310,8 @@ "\n", "We also need to consider the units of our image data. Let\u2019s assume that the galaxy image is measured in units of \n", "electrons per second ($e^- s^{-1}$), which is standard for CCD imaging data. To simulate the number of electrons \n", - "actually detected in each pixel, we multiply the image by the observation\u2019s exposure time. This conversion changes t\n", - "he units to the total number of electrons collected per pixel over the entire exposure time.\n", + "actually detected in each pixel, we multiply the image by the observation\u2019s exposure time. This conversion changes\n", + "the units to the total number of electrons collected per pixel over the entire exposure time.\n", "\n", "Once the image is converted, we add Poisson noise, simulating the randomness in the photon-to-electron conversion \n", "process. After adding the noise, we convert the image back to units of electrons per second for analysis, as \n", @@ -609,13 +610,78 @@ "outputs": [], "execution_count": null }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Interferometer Data__\n", + "\n", + "CCD imaging is not the only type of data used to study galaxies. Radio and sub-mm interferometers, like the\n", + "Atacama Large Millimeter Array (ALMA) and the Jansky Very Large Array (JVLA), observe galaxies at wavelengths\n", + "where a CCD cannot.\n", + "\n", + "An interferometer does not observe an image of the galaxy. Each pair of antennas in the array measures a\n", + "\"visibility\", a Fourier component of the sky brightness, at a point in what is called the \"uv-plane\" set by the\n", + "separation of the two antennas. The dataset is therefore a set of complex visibilities in Fourier space, not a 2D\n", + "image, and its noise properties are very different from those of CCD data \u2014 there is no PSF convolution, Poisson\n", + "noise or background sky; instead each visibility has Gaussian noise.\n", + "\n", + "One could Fourier transform the visibilities into an image (called a \"dirty image\") and fit that, but the transform\n", + "correlates the noise between pixels, making the fit statistically incorrect. **PyAutoGalaxy** therefore fits galaxy\n", + "models directly in visibility space: the galaxy's image is evaluated in real space on a grid (defined by a\n", + "real-space mask), Fourier transformed to the uv-plane and compared with the observed visibilities there.\n", + "\n", + "Below, we load a simulated interferometer dataset (creating it first via the `scripts/simulators/interferometer.py`\n", + "script if it does not exist on your hard-disk, using the same auto-simulation idiom as later tutorials) and plot\n", + "its dirty images \u2014 the closest an interferometer dataset comes to the CCD images we simulated above.\n", + "\n", + "HowToGalaxy will not cover interferometry any further than this. The lecture series teaches galaxy modeling using\n", + "CCD imaging, and everything you learn transfers to visibility-space fitting. If you need to model interferometer\n", + "data, go to the `autogalaxy_workspace/scripts/interferometer` package, which is the dedicated resource for\n", + "uv-plane galaxy modeling." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset_path = Path(\"dataset\") / \"interferometer\" / \"simple\"\n", + "\n", + "if ag.util.dataset.should_simulate(str(dataset_path)):\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.run(\n", + " [sys.executable, \"scripts/simulators/interferometer.py\"],\n", + " check=True,\n", + " )\n", + "\n", + "real_space_mask = ag.Mask2D.circular(\n", + " shape_native=(100, 100),\n", + " pixel_scales=0.1,\n", + " radius=3.0,\n", + ")\n", + "\n", + "dataset = ag.Interferometer.from_fits(\n", + " data_path=dataset_path / \"data.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " uv_wavelengths_path=dataset_path / \"uv_wavelengths.fits\",\n", + " real_space_mask=real_space_mask,\n", + " transformer_class=ag.TransformerDFT,\n", + ")\n", + "\n", + "aplt.subplot_interferometer_dirty_images(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, { "cell_type": "markdown", "metadata": {}, "source": [ "__Wrap Up__\n", "\n", - "In this tutorial, you learned how CCD imaging data of a galaxy is collected using real telescopes like the \n", + "In this tutorial, you learned how CCD imaging data of a galaxy is collected using real telescopes like the\n", "Hubble Space Telescope, and how to simulate this data using the `SimulatorImaging` object.\n", "\n", "Let's summarise what we've covered:\n", diff --git a/notebooks/chapter_1_introduction/tutorial_3_fitting.ipynb b/notebooks/chapter_1_introduction/tutorial_3_fitting.ipynb index df78e09..601771a 100644 --- a/notebooks/chapter_1_introduction/tutorial_3_fitting.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_3_fitting.ipynb @@ -78,7 +78,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from pathlib import Path\n", @@ -224,11 +224,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can visualize the mask over the galaxy image using an `Imaging`, which helps us adjust the mask as needed. \n", - "This is useful to ensure that the mask appropriately covers the galaxy's light and does not exclude important regions.\n", - "\n", - "To overlay objects like a mask onto a figure, we use the `Visuals2D` object. This tool allows us to add custom \n", - "visuals to any plot, providing flexibility in creating tailored visual representations." + "We can visualize the mask over the galaxy image, which helps us adjust the mask as needed. This is useful to ensure\n", + "that the mask appropriately covers the galaxy's light and does not exclude important regions." ] }, { @@ -261,8 +258,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "When we plot the masked imaging data again, the mask is now automatically included in the plot, even though we did \n", - "not explicitly pass it using the `Visuals2D` object. The plot also zooms into the unmasked area, showing only the \n", + "When we plot the masked imaging data again, the mask is now automatically included in the plot, even though we did\n", + "not explicitly pass it to the plot function. The plot also zooms into the unmasked area, showing only the\n", "region where we will focus our analysis. This is particularly helpful when working with large images, as it centers \n", "the view on the regions where the galaxy's signal is detected." ] @@ -322,7 +319,7 @@ "print(dataset.data.native.shape)\n", "print(\n", " dataset.data.slim.shape\n", - ") # This should be lower than the total number of pixels, e.g., 100 x 100 = 10,000" + ") # This should be lower than the total number of pixels, e.g., 101 x 101 = 10,201" ], "outputs": [], "execution_count": null @@ -440,8 +437,8 @@ "\n", "Now that our data is masked, we are ready to proceed with the fitting process.\n", "\n", - "Fitting the data is done using the `Galaxy` and `Galaxies objects that we introduced in tutorial 2. We will start by \n", - "setting up a `Galaxies`` object, using the same galaxy configuration that we previously used to simulate the \n", + "Fitting the data is done using the `Galaxy` and `Galaxies` objects that we introduced in tutorial 1. We will start by\n", + "setting up a `Galaxies` object, using the same galaxy configuration that we previously used to simulate the\n", "imaging data. This setup will give us what is known as a 'perfect' fit, as the simulated and fitted models are identical." ] }, @@ -564,7 +561,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The `residual_map` is the different between the observed image and model image, showing where in the image the fit is\n", + "The `residual_map` is the difference between the observed image and model image, showing where in the image the fit is\n", "good (e.g. low residuals) and where it is bad (e.g. high residuals).\n", "\n", "The expression for the residual map is simply:\n", @@ -596,7 +593,7 @@ "metadata": {}, "source": [ "Are these residuals indicative of a good fit to the data? Without considering the noise in the data, it's difficult \n", - "to ascertain. That is, its hard to ascenrtain if a residual value is large or small because this depends on the\n", + "to ascertain. That is, its hard to ascertain if a residual value is large or small because this depends on the\n", "amount of noise in that pixel.\n", "\n", "The `normalized_residual_map` divides the residual-map by the noise-map, giving the residual in units of the noise.\n", @@ -827,7 +824,7 @@ "These metrics are standard ways to quantify the quality of a model fit. They are applicable not only to 1D data but \n", "also to more complex data structures like 2D images, 3D data cubes, or any other multidimensional datasets.\n", "\n", - "__Incorrect Fit___\n", + "__Incorrect Fit__\n", "\n", "In the previous section, we successfully created and fitted a galaxy model to the image data, resulting in an \n", "excellent fit. The residual map and chi-squared map showed no significant discrepancies, indicating that the \n", diff --git a/notebooks/chapter_1_introduction/tutorial_4_methods.ipynb b/notebooks/chapter_1_introduction/tutorial_4_methods.ipynb index 03ae894..03a4f16 100644 --- a/notebooks/chapter_1_introduction/tutorial_4_methods.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_4_methods.ipynb @@ -65,7 +65,7 @@ "\n", "from autogalaxy import jax_wrapper # Sets JAX environment before other imports\n", "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import autogalaxy as ag\n", "import autogalaxy.plot as aplt" diff --git a/notebooks/chapter_1_introduction/tutorial_5_summary.ipynb b/notebooks/chapter_1_introduction/tutorial_5_summary.ipynb index b6b198b..bc26c36 100644 --- a/notebooks/chapter_1_introduction/tutorial_5_summary.ipynb +++ b/notebooks/chapter_1_introduction/tutorial_5_summary.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Tutorial 9: Summary\n", + "Tutorial 5: Summary\n", "===================\n", "\n", "In this chapter, we have learnt that:\n", @@ -12,10 +12,10 @@ " 1) **PyAutoGalaxy** uses Cartesian `Grid2D`'s of $(y,x)$ coordinates to evaluate galaxy luminous emission.\n", " 2) These grids are combined with light profiles to compute images and other quantities.\n", " 3) Profiles are grouped together to make galaxies.\n", - " 4) Collections of galaxies (at the same redshift) can be made..\n", + " 4) Collections of galaxies (at the same redshift) can be made.\n", " 5) The Universe's cosmology can be input into this `Galaxies` to convert its units to kiloparsecs.\n", " 6) The galaxies's image can be used to simulate galaxy `Imaging` like it was observed with a real telescope.\n", - " 7) This data can be fitted, so to as quantify how well a model galaxy system represents the observed image.\n", + " 7) This data can be fitted, so as to quantify how well a model galaxy system represents the observed image.\n", "\n", "In this summary, we'll go over all the different Python objects introduced throughout this chapter and consider how\n", "they come together as one.\n", @@ -73,7 +73,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -138,7 +138,7 @@ "source": [ "__Object Composition__\n", "\n", - "Lets now consider how all of the objects we've covered throughout this chapter (`LightProfile`'s, `MassProfile`'s,\n", + "Lets now consider how all of the objects we've covered throughout this chapter (`LightProfile`'s,\n", "`Galaxy`'s, `Galaxies`'s) come together.\n", "\n", "The `Galaxies` contain the `Galaxy`'s which contains the `Profile`'s:" @@ -189,8 +189,8 @@ "Understanding how these objects decompose into the different components of a galaxy is important for general \n", "**PyAutoGalaxy** use.\n", "\n", - "As the galaxy systems that we analyse become more complex, it is useful to know how to decompose their light \n", - "profiles, galaxies and galaxies to extract different pieces of information about the galaxy. \n", + "As the galaxy systems that we analyse become more complex, it is useful to know how to decompose their light\n", + "profiles and galaxies to extract different pieces of information about the galaxy.\n", "\n", "For example, we made our galaxy above with two light profiles, a `bulge` and `disk`. We can plot the image of \n", "each component individually, now that we know how to break-up the different components of the galaxies." @@ -213,11 +213,11 @@ "source": [ "__Visualization__\n", "\n", - "Furthermore, using the `MatPLot2D` and `Visuals2D` objects we can visualize any aspect we're interested \n", - "in and fully customize the figure. \n", + "Furthermore, using the arguments of the `aplt` plotting functions we can visualize any aspect we're interested\n", + "in and fully customize the figure.\n", "\n", - "Before beginning chapter 2 of **HowToGalaxy**, you should checkout the package `autogalaxy_workspace/plot`. \n", - "This provides a full API reference of every plotting option in **PyAutoGalaxy**, allowing you to create your own \n", + "Before beginning chapter 2 of **HowToGalaxy**, you should checkout the package `autogalaxy_workspace/*/guides/plot`.\n", + "This provides a full API reference of every plotting option in **PyAutoGalaxy**, allowing you to create your own\n", "fully customized figures of galaxies with minimal effort!" ] }, @@ -239,15 +239,15 @@ "__Code Design__\n", "\n", "To end, I want to quickly talk about the **PyAutoGalaxy** code-design and structure, which was really the main topic of\n", - "this tutoriag.\n", + "this tutorial.\n", "\n", - "Throughout this chapter, we never talk about anything like it was code. We didn`t refer to 'variables', 'parameters`' \n", - "'functions' or 'dictionaries', did we? Instead, we talked about 'galaxies'. We discussed \n", + "Throughout this chapter, we never talk about anything like it was code. We didn't refer to 'variables', 'parameters',\n", + "'functions' or 'dictionaries', did we? Instead, we talked about 'galaxies'. We discussed\n", "the objects that we, as scientists, think about when we consider a galaxy system.\n", "\n", - "Software that abstracts the underlying code in this way follows an `object-oriented design`, and it is our hope \n", + "Software that abstracts the underlying code in this way follows an `object-oriented design`, and it is our hope\n", "with **PyAutoGalaxy** that we've made its interface (often called the API for short) very intuitive, whether you were\n", - "previous familiar with galaxy morphology or a complete newcomer!\n", + "previously familiar with galaxy morphology or a complete newcomer!\n", "\n", "__Source Code__\n", "\n", @@ -270,13 +270,13 @@ "\n", "__Wrap Up__\n", "\n", - "You`ve learn a lot in this chapter, but what you have not learnt is how to 'model' a real galaxy.\n", + "You've learnt a lot in this chapter, but what you have not learnt is how to 'model' a real galaxy.\n", "\n", "In the real world, we have no idea what the 'correct' combination of light profiles are that will give a good fit to \n", "a galaxy. Modeling is the process of finding the model which provides a good fit and it is the topic of chapter 2 \n", "of **HowToGalaxy**.\n", "\n", - "Finally, if you enjoyed doing the **HowToGalaxy** tutorials please git us a star on the **PyAutoGalaxy** GitHub\n", + "Finally, if you enjoyed doing the **HowToGalaxy** tutorials please give us a star on the **PyAutoGalaxy** GitHub\n", "repository: \n", "\n", " https://github.com/PyAutoLabs/PyAutoGalaxy\n", diff --git a/notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb b/notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb index 369695d..6daab65 100644 --- a/notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb @@ -4,20 +4,21 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Tutorial 2: Prior Passing\n", - "=========================\n", + "Tutorial 10: Prior Passing\n", + "==========================\n", "\n", "In the previous tutorial, we used non-linear search chaining to break the model-fitting procedure down into two\n", "non-linear searches. This used an initial search to fit a simple model, whose results were used to tune and\n", "initialize the priors of a more complex model that was fitted by the second search.\n", "\n", - "However, the results were passed between searches were passed manually. I explicitly wrote out every result as a prior\n", + "However, the results were passed between searches manually. I explicitly wrote out every result as a prior\n", "containing the values inferred in the first search. **PyAutoGalaxy** has an API for passing priors in a more generalized\n", "way, which is the topic of this tutorial.\n", "\n", "__Contents__\n", "\n", "- **Initial Setup:** Load the dataset and apply a mask.\n", + "- **Dataset Auto-Simulation:** Simulate the dataset via its simulator script if it is not on your hard-disk.\n", "- **Model:** Compose the model for the first search.\n", "- **Search:** Run the first search.\n", "- **Result (Search 1):** Inspect the result of the first search.\n", @@ -71,7 +72,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from pathlib import Path\n", @@ -88,9 +89,9 @@ "source": [ "__Initial Setup__\n", "\n", - "we'll use the same galaxying data as the previous tutorial, where:\n", + "we'll use the same galaxy data as the previous tutorial, where:\n", "\n", - " - The galaxy's bulge is an `Sersic`.\n", + " - The galaxy's bulge is a `Sersic`.\n", " - The galaxy's disk is an `Exponential`.\n", " \n", "All the usual steps for setting up a model fit (masking, analysis, etc.) are included below." @@ -223,11 +224,11 @@ "analysis_1 = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", "\n", "search_1 = af.Nautilus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_3\"),\n", - " name=\"tutorial_1_search_chaining_1\",\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_2\"),\n", + " name=\"tutorial_9_search_chaining_1\",\n", " unique_tag=dataset_name,\n", " n_live=100,\n", - " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", + " n_batch=50, # GPU batching and VRAM use explained in tutorial 2 of this chapter.\n", ")\n", "\n", "result_1 = search_1.fit(model=model_1, analysis=analysis_1)" @@ -263,7 +264,7 @@ "write out the inferred parameter values of each component. The details of how prior passing is performed will be \n", "expanded upon at the end of the tutorial.\n", "\n", - "We start with the bulge, which in the previous search was an `Sersic` with its centre fixed to (0.0, 0.0) \n", + "We start with the bulge, which in the previous search was a `Sersic` with its centre fixed to (0.0, 0.0) \n", "and its `sersic_index` fixed to 4.0. The API for passing priors is shown below and there are two things worth noting:\n", "\n", " 1) We pass the priors using the `model` attribute of the result. This informs **PyAutoGalaxy** to pass the result as a\n", @@ -295,7 +296,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For the disk, we are passing the result of an `Exponential` to an `Sersic`.\n", + "For the disk, we are passing the result of an `Exponential` to a `Sersic`.\n", "\n", "We do not pass the `ell_comps` because this would pair them to the `bulge`, as was performed in the first \n", "model-fit." @@ -366,17 +367,17 @@ "analysis_2 = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", "\n", "search_2 = af.Nautilus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_3\"),\n", - " name=\"tutorial_2_search_chaining_2\",\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_2\"),\n", + " name=\"tutorial_10_prior_passing_2\",\n", " unique_tag=dataset_name,\n", " n_live=100,\n", - " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", + " n_batch=50, # GPU batching and VRAM use explained in tutorial 2 of this chapter.\n", ")\n", "\n", "print(\n", - " \"The non-linear search has begun running - checkout the workspace/output/5_chaining_searches\"\n", + " \"The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_2 = search_2.fit(model=model_2, analysis=analysis_2)\n", @@ -426,34 +427,35 @@ "source": [ "__Wrap Up__\n", "\n", - "We will expand on the prior passing API in the following tutorials. The main thing to note is that we can pass \n", - "entire profiles or galaxies using prior passing, if their model does not change (which for the bulge and disk, was \n", - "not true). The API to pass a whole profile or galaxy is as follows:\n", - " \n", + "This tutorial has covered the core of the prior passing API; the chaining example in the `autogalaxy_workspace`\n", + "(`autogalaxy_workspace/scripts/guides/modeling/chaining.py`) expands on it further. The main thing to note is that\n", + "we can pass entire profiles or galaxies using prior passing, if their model does not change (which for the bulge\n", + "and disk above, was not true). The API to pass a whole profile or galaxy is as follows:\n", + "\n", " bulge = result_1.model.galaxies.galaxy.bulge\n", " galaxy = result_1.model.galaxies.galaxy\n", - " \n", + "\n", "We can also pass priors using an `instance` instead of a `model`. When an `instance` is used, the maximum likelihood\n", "parameter values are passed as fixed values that are therefore not fitted for by the non-linear search (reducing its\n", - "dimensionality). We will use this in the next tutorial to fit data with two galaxies, where fit one galaxy, fix it to \n", - "the best-fit model in a second search that fits the second galaxy, and then go on to fit both simultaneously in the \n", - "final search.\n", + "dimensionality). Chained fits often use this to fix one galaxy's light to the best-fit model of an early search\n", + "before fitting a second galaxy alongside it \u2014 chapter 4 of **HowToGalaxy**, which scales up to modeling multiple\n", + "galaxies, and the workspace's chaining example show this in action.\n", " \n", "Lets now think about how priors are passed. Checkout the `model.info` file of the second search of this tutorial. The \n", "parameters do not use the default priors we saw in search 1 (which are typically broad UniformPriors). Instead, \n", "they use GaussianPrior`s where:\n", "\n", " - The mean values are the median PDF results of every parameter in search 1.\n", - " - The sigma values are specified in the `width_modifier` field of the profile's entry in the `priors.yaml' config \n", - " file (we will discuss why this is used in a moment).\n", + " - The sigma values are specified in the `width_modifier` field of the profile's entry in the `config/priors`\n", + " config files (we will discuss why this is used in a moment).\n", "\n", - "Like the manual `GaussianPrior`'s that were used in tutorial 1, the prior passing API sets up the prior on each \n", + "Like the manual `GaussianPrior`'s that were used in the previous tutorial, the prior passing API sets up the prior on each\n", "parameter with a `GaussianPrior` centred on the high likelihood regions of parameter space!\n", "\n", "__Detailed Explanation Of Prior Passing__\n", "\n", "To end, I provide a detailed overview of how prior passing works and illustrate tools that can be used to customize\n", - "its behaviour. It is up to you whether you want read this, or go ahead to the next tutorial!\n", + "its behaviour. It is up to you whether you want read this, or move on to the summary that concludes the chapter!\n", "\n", "Lets say I chain two parameters as follows:\n", " \n", @@ -461,7 +463,7 @@ "\n", "By invoking the `model` attribute, the prior is passed following 3 rules:\n", "\n", - " 1) The new parameter, in this case the einstein radius, uses a `GaussianPrior`.This is ideal, as the 1D pdf results \n", + " 1) The new parameter, in this case the effective radius, uses a `GaussianPrior`. This is ideal, as the 1D pdf results\n", " we compute at the end of a search are easily summarised as a Gaussian.\n", "\n", " 2) The mean of the `GaussianPrior` is the median PDF value of the parameter estimated in search 1.\n", @@ -470,8 +472,8 @@ " parameter space that correspond to highest log likelihood solutions in the previous search. Our priors therefore \n", " correspond to the `correct` regions of parameter space.\n", "\n", - " 3) The sigma of the Gaussian uses the value specified for the profile in the `config/priors/*.yaml` config file's \n", - " `width_modifer` field (check these files out now).\n", + " 3) The sigma of the Gaussian uses the value specified for the profile in the `config/priors/*.yaml` config file's\n", + " `width_modifier` field (check these files out now).\n", "\n", "The idea here is simple. We want a value of sigma that gives a `GaussianPrior` wide enough to search a broad \n", "region of parameter space, so that the model can change if a better solution is nearby. However, we want it \n", @@ -479,7 +481,7 @@ "into an incorrect solution! \n", "\n", "The `width_modifier` values in the priors config file have been chosen based on our experience as being a good\n", - "balance broadly sampling parameter space but not being so narrow important solutions are missed.\n", + "balance of broadly sampling parameter space but not being so narrow that important solutions are missed.\n", " \n", "There are two ways a value is specified using the priors/width file:\n", "\n", @@ -500,12 +502,9 @@ "\n", "However, there are parameters where using an absolute value does not make sense. Intensity is a good example of this. \n", "The intensity of an image depends on its units, S/N, galaxy brightness, etc. There is no single absolute value that \n", - "one can use to generically chain the intensity of any two proflies. Thus, it makes more sense to chain them using \n", + "one can use to generically chain the intensity of any two profiles. Thus, it makes more sense to chain them using\n", "the relative value from a previous search.\n", "\n", - "We can customize how priors are passed from the results of a search and non-linear search by editing the\n", - " `prior_passer` settings in the `general.yaml` config file.\n", - "\n", "__EXAMPLE__\n", "\n", "Lets go through an example using a real parameter. Lets say in search 1 we fit the galaxy's light with an \n", @@ -517,12 +516,12 @@ "\n", "The prior on the galaxy's bulge sersic index in search 2 would thus be a `GaussianPrior` with mean=4.0. \n", "\n", - "The value of the Sersic index `width_modifier` in the priors config file sets sigma. The prior config file specifies \n", - "that we use an \"Absolute\" value of 0.8 to chain this prior. Thus, the `GaussianPrior` in search 2 would have a \n", - "mean=4.0 and sigma=0.8.\n", + "The value of the Sersic index `width_modifier` in the priors config file sets sigma. The prior config file specifies\n", + "that we use an \"Absolute\" value of 1.5 to chain this prior. Thus, the `GaussianPrior` in search 2 would have a\n", + "mean=4.0 and sigma=1.5.\n", "\n", - "If the prior config file had specified that we use an relative value of 0.8, the GaussianPrior in search 2 would have a \n", - "mean=4.0 and sigma = 4.0 * 0.8 = 3.2.\n", + "If the prior config file had instead specified that we use a relative value of 0.8, the GaussianPrior in search 2 would\n", + "have a mean=4.0 and sigma = 4.0 * 0.8 = 3.2.\n", "\n", "And with that, we're done. Chaining priors is a bit of an art form, but one that works really well. " ] diff --git a/notebooks/chapter_2_modeling/tutorial_1_non_linear_search.ipynb b/notebooks/chapter_2_modeling/tutorial_1_non_linear_search.ipynb index 0ed5584..549355d 100644 --- a/notebooks/chapter_2_modeling/tutorial_1_non_linear_search.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_1_non_linear_search.ipynb @@ -35,7 +35,7 @@ "\n", "__Overview__\n", "\n", - "In this tutorial, we will use a non-linear search to fit a single Serisc light profile to simulated imaging of a\n", + "In this tutorial, we will use a non-linear search to fit a single Sersic light profile to simulated imaging of a\n", "galaxy. We will:\n", "\n", "- Introduce concept like a \"parameter space\", \"likelihood surface\" and \"priors\", and relate them to how a non-linear\n", @@ -45,7 +45,7 @@ " model instance to the data.\n", "\n", "- Fit datasets with different non-linear searches, including a maximum likelihood estimator (MLE),\n", - " Markok Chain Monte Carlo (MCMC) and nested sampling.\n", + " Markov Chain Monte Carlo (MCMC) and nested sampling.\n", "\n", "__Contents__\n", "\n", @@ -88,8 +88,8 @@ "This defines a parameter space in three dimensions, representing the relationships between \\(x\\), \\(y\\), \\(z\\),\n", "and the output \\(f(x, y, z)\\).\n", "\n", - "This concept of parameter space is closely related to how we approach model-fitting. For instance, in chapter 1, w\n", - "e created instances of a `Galaxy` object with\n", + "This concept of parameter space is closely related to how we approach model-fitting. For instance, in chapter 1,\n", + "we created instances of a `Galaxy` object with\n", "parameters like \\( (\\text{`centre_0`}, \\text{`centre_1`}, \\text{`ell_comps_0`}, \\text{`ell_comps_1`}, \\text{`intensity`}, \\text{`effective_radius`}, \\text{`sersic_index`}) \\).\n", "These parameters were used to fit data and compute a log likelihood.\n", "\n", @@ -106,7 +106,7 @@ "\n", "This parameter space is \"non-linear,\" meaning the relationship between the model parameters and the log likelihood is\n", "not a simple linear one. Because of this non-linearity, we cannot predict the log likelihood from a given set of model\n", - "parameters without actually performing a fit to the data, as we did in tutorial 1.\n", + "parameters without actually performing a fit to the data, as we did in the fitting tutorial of chapter 1.\n", "\n", "__Non-Linear Search__\n", "\n", @@ -154,7 +154,7 @@ "\n", "__Deeper Background__\n", "\n", - "**The descriptions of how searches work in this example are simplfied and phoenomenological and do not give a full\n", + "**The descriptions of how searches work in this example are simplified and phenomenological and do not give a full\n", "description of how they work at a deep statistical level. The goal is to provide you with an intuition for how to use\n", "them and when different searches are appropriate for different problems. Later tutorials will provide a more formal\n", "description of how these searches work.**\n", @@ -210,7 +210,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -229,7 +229,7 @@ "[PyAutoFit](https://github.com/PyAutoLabs/PyAutoFit), an open-source project that allows complex model\n", "fitting techniques to be straightforwardly integrated into scientific modeling software. \n", "\n", - "**PyAutoFit** is actually a spin-off project of **PyAutoGalaxy**. whereby we found that the statistic techniques and\n", + "**PyAutoFit** is actually a spin-off project of **PyAutoGalaxy**, whereby we found that the statistical techniques and\n", "methods we applied to model galaxies could be used in a more general setting to many different scientific \n", "problems. Check it out if you are interested in developing your own software to perform advanced model-fitting!\n", "\n", @@ -525,7 +525,7 @@ "2. **Markov Chain Monte Carlo (MCMC)** using the `Emcee` non-linear search.\n", "3. **Nested Sampling** using the `Nautilus` non-linear search.\n", "\n", - "In this example, non-linear search results are stored in memory rather and not written to hard disk because the fits \n", + "In this example, non-linear search results are stored in memory rather than written to hard disk because the fits\n", "are fast and can therefore be easily regenerated. The next tutorial will perform fits which write results to the\n", "hard-disk and discuss the outputs that are generated.\n", "\n", @@ -574,7 +574,7 @@ "print(\n", " \"\"\"\n", " The non-linear search has begun running.\n", - " This Jupyter notebook cell with progress once the search has completed - this could take a few minutes!\n", + " This Jupyter notebook cell will progress once the search has completed - this could take a few minutes!\n", " \"\"\"\n", ")\n", "\n", @@ -595,7 +595,7 @@ "\n", "The `info` attribute shows the result in a readable format.\n", "\n", - "[Above, we discussed that the `info_whitespace_length` parameter in the config files could b changed to make \n", + "[Above, we discussed that the `info_whitespace_length` parameter in the config files could be changed to make\n", "the `model.info` attribute display optimally on your computer. This attribute also controls the whitespace of the\n", "`result.info` attribute.]" ] @@ -667,7 +667,7 @@ "print(\n", " \"\"\"\n", " The non-linear search has begun running.\n", - " This Jupyter notebook cell with progress once the search has completed - this could take a few minutes!\n", + " This Jupyter notebook cell will progress once the search has completed - this could take a few minutes!\n", " \"\"\"\n", ")\n", "\n", @@ -709,7 +709,7 @@ "\n", "However, MLE has its limitations. As seen above, it can get \"stuck\" in a local maximum, particularly if the \n", "starting point is poorly chosen. In complex model-fitting problems, providing a suitable starting point can be \n", - "challenging. While MLE performed well in the example with just three parameters, it struggles with models that have \n", + "challenging. While MLE performed well in the example with just six parameters, it struggles with models that have\n", "many parameters, as the complexity of the likelihood surface makes simply moving \"up\" the gradient less effective.\n", "\n", "The MLE also does not provide any information on the errors on the parameters, which is a significant limitation.\n", @@ -752,7 +752,7 @@ "print(\n", " \"\"\"\n", " The non-linear search has begun running.\n", - " This Jupyter notebook cell with progress once the search has completed - this could take a few minutes!\n", + " This Jupyter notebook cell will progress once the search has completed - this could take a few minutes!\n", " \"\"\"\n", ")\n", "\n", @@ -779,7 +779,7 @@ "\n", "A major advantage of MCMC is that it provides estimates of parameter uncertainties by \"mapping out\" the likelihood \n", "surface, unlike MLE, which only finds the maximum likelihood solution. These error estimates are accessible in \n", - "the `result.info` string and through the `result.samples` object, which is explained fully in tutorial 5.\n", + "the `result.info` string and through the `result.samples` object, which is explained fully in tutorial 7.\n", "\n", "While a good starting point wasn't necessary for this simple model, it becomes essential for efficiently mapping the \n", "likelihood surface in more complex models with many parameters. The code below shows an MCMC fit using a good starting \n", @@ -807,14 +807,14 @@ "\n", "search = af.Emcee(\n", " nwalkers=20, # The number of walkers we'll use to sample parameter space.\n", - " nsteps=300, # The number of steps each walker takes, after which 10 * 200 = 2000 steps the non-linear search ends.\n", + " nsteps=300, # The number of steps each walker takes, after which 20 * 300 = 6000 steps the non-linear search ends.\n", " initializer=initializer,\n", ")\n", "\n", "print(\n", " \"\"\"\n", " The non-linear search has begun running.\n", - " This Jupyter notebook cell with progress once the search has completed - this could take a few minutes!\n", + " This Jupyter notebook cell will progress once the search has completed - this could take a few minutes!\n", " \"\"\"\n", ")\n", "\n", @@ -894,7 +894,7 @@ "print(\n", " \"\"\"\n", " The non-linear search has begun running.\n", - " This Jupyter notebook cell with progress once the search has completed - this could take a few minutes!\n", + " This Jupyter notebook cell will progress once the search has completed - this could take a few minutes!\n", " \"\"\"\n", ")\n", "\n", diff --git a/notebooks/chapter_2_modeling/tutorial_2_practicalities.ipynb b/notebooks/chapter_2_modeling/tutorial_2_practicalities.ipynb index d139f07..87dfa04 100644 --- a/notebooks/chapter_2_modeling/tutorial_2_practicalities.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_2_practicalities.ipynb @@ -27,15 +27,15 @@ " **PyAutoFit:** The parent package of PyAutoGalaxy, which handles practicalities of model-fitting.\n", " **Initial Setup:** Load the dataset we'll fit a model to using a non-linear search.\n", " **Mask:** Apply a mask to the dataset.\n", + " **Over Sampling:** Apply adaptive over sampling to the dataset.\n", " **Model:** Introduce the model we will fit to the data.\n", " **Search:** Setup the non-linear search, Nautilus, used to fit the model to the data.\n", " **Search Settings:** Discuss the settings of the non-linear search, including the number of live points.\n", - " **Number Of Cores:** Discuss how to use multiple cores to fit models faster in parallel.\n", - " **Parallel Script:** Running the model-fit in parallel if a bug occurs in a Jupiter notebook.\n", " **Iterations Per Update:** How often the non-linear search outputs the current results to hard-disk.\n", " **Analysis:** Create the Analysis object which contains the `log_likelihood_function` that the non-linear search calls.\n", + " **Run Times:** Estimate the run-time of the model-fit before initiating it.\n", " **Model-Fit:** Fit the model to the data.\n", - " **Result:** Print the results of the model-fit to the terminal.\n", + " **Result Info:** Print the results of the model-fit to the terminal.\n", " **Output Folder:** Inspect the output folder where results are stored.\n", " **Unique Identifier:** Discussion of the unique identifier of the model-fit which names the folder in the output directory.\n", " **Output Folder Contents:** What is output to the output folder (model results, visualization, etc.).\n", @@ -85,7 +85,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -283,8 +283,8 @@ "more thoroughly, increasing accuracy but also runtime. Fewer live points reduce run-time but may make the search less \n", "reliable, possibly getting stuck in local maxima.\n", "\n", - "The ideal number of live points depends on model complexity. More parameters generally require more live points, but \n", - "the default of 200 is sufficient for most galaxy models. Lower values can still yield reliable results, particularly \n", + "The ideal number of live points depends on model complexity. More parameters generally require more live points, but\n", + "the default of 3000 is more than sufficient for most galaxy models. Lower values can still yield reliable results, particularly\n", "for simpler models. For this example (6 parameters), we reduce the live points to 80 to speed up runtime without \n", "compromising accuracy.\n", "\n", @@ -304,8 +304,8 @@ "Depending on how long it takes for the model to be fitted to the data (see discussion about run times below), \n", "this can take up a large fraction of the run-time of the non-linear search.\n", "\n", - "For this fit, the fit is very fast, thus we set a high value of `iterations_per_quick_update=10000` to ensure these updates\n", - "so not slow down the overall speed of the model-fit. \n", + "For this fit, the fit is very fast, thus we set a high value of `iterations_per_quick_update=2500` to ensure these updates\n", + "do not slow down the overall speed of the model-fit.\n", "\n", "**If the iteration per update is too low, the model-fit may be significantly slowed down by the time it takes to\n", "output results and visualization frequently to hard-disk. If your fit is consistent displaying a log saying that it\n", @@ -321,7 +321,7 @@ " name=\"tutorial_2_practicalities\",\n", " unique_tag=dataset_name,\n", " n_live=80,\n", - " n_batch=50, # GPU lens model fits are batched and run simultaneously, see VRAM section below.\n", + " n_batch=50, # GPU model fits are batched and run simultaneously, see the run times section below.\n", " iterations_per_quick_update=2500,\n", ")" ], @@ -362,15 +362,15 @@ " - **The log likelihood evaluation time:** the time it takes for a single `instance` of the model to be fitted to \n", " the dataset such that a log likelihood is returned.\n", "\n", - " - **The number of iterations (e.g. log likelihood evaluations) performed by the non-linear search:** more complex lens\n", + " - **The number of iterations (e.g. log likelihood evaluations) performed by the non-linear search:** more complex\n", " models require more iterations to converge to a solution (and as discussed above, settings like the number of live\n", " points also control this).\n", "\n", "For this analysis, the log likelihood evaluation time is ~0.05 seconds, which is extremely fast for model fitting. \n", "\n", - "The more advanced fitting techniques discussed at the end of chapter 1 (e.g. shapelets, multi Gaussian expansions, \n", - "pixelizations) have longer log likelihood evaluation times. However, on GPU, they can be so fast they may not produce\n", - "significantly longer overall run-time feasible.\n", + "The more advanced fitting techniques discussed later in this chapter (e.g. multi Gaussian expansions, shapelets) and\n", + "the pixelizations of chapter 3 have longer log likelihood evaluation times. However, on GPU, they can be so fast that\n", + "they may not produce significantly longer overall run-times.\n", "\n", "To estimate the expected overall run time of the model-fit we multiply the log likelihood evaluation time by an \n", "estimate of the number of iterations the non-linear search will perform, which is around 50000 to 100000 for this model.\n", @@ -410,7 +410,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the autogalaxy_workspace/output/\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result = search.fit(model=model, analysis=analysis)\n", @@ -428,7 +428,7 @@ "\n", "A concise readable summary of the results is given by printing its `info` attribute.\n", "\n", - "[Above, we discussed that the `info_whitespace_length` parameter in the config files could b changed to make \n", + "[Above, we discussed that the `info_whitespace_length` parameter in the config files could be changed to make\n", "the `model.info` attribute display optimally on your computer. This attribute also controls the whitespace of the\n", "`result.info` attribute.]" ] @@ -471,9 +471,9 @@ "\n", "__Output Folder Contents__\n", "\n", - "Now this is running you should checkout the `autogalaxy_workspace/output` folder. This is where the results of the \n", - "search are written to hard-disk (in the `start_here` folder), where all outputs are human readable (e.g. as .json,\n", - ".csv or text files).\n", + "Now this is running you should checkout the `autogalaxy_workspace/output` folder. This is where the results of the\n", + "search are written to hard-disk (in the `tutorial_2_practicalities` folder), where all outputs are human readable\n", + "(e.g. as .json, .csv or text files).\n", "\n", "As the fit progresses, results are written to the `output` folder on the fly using the highest likelihood model found\n", "by the non-linear search so far. This means you can inspect the results of the model-fit as it runs, without having to\n", @@ -527,15 +527,15 @@ "\n", "The PDF shows the 1D and 2D probabilities estimated for every parameter after the model-fit. The two dimensional \n", "figures can show the degeneracies between different parameters, for example how increasing the intensity $I$ of the\n", - "source galaxy and decreasing its effective radius $R_{Eff}$ lead to similar likelihoods and probabilities.\n", + "galaxy and decreasing its effective radius $R_{Eff}$ lead to similar likelihoods and probabilities.\n", "\n", "This PDF will be discussed more in the next tutorial.\n", "\n", - "The plot is labeled with short hand parameter names (e.g. `sersic_index` is mapped to the short hand \n", - "parameter `n`). These mappings ate specified in the `config/notation.yaml` file and can be customized by users.\n", + "The plot is labeled with short hand parameter names (e.g. `sersic_index` is mapped to the short hand\n", + "parameter `n`). These mappings are specified in the `config/notation.yaml` file and can be customized by users.\n", "\n", - "The superscripts of labels correspond to the name each component was given in the model (e.g. for the `Isothermal`\n", - "mass its name `mass` defined when making the `Model` above is used).\n", + "The superscripts of labels correspond to the name each component was given in the model (e.g. for the `Sersic`\n", + "bulge its name `bulge` defined when making the `Model` above is used).\n", "\n", "__Other Practicalities__\n", "\n", diff --git a/notebooks/chapter_2_modeling/tutorial_3_realism_and_complexity.ipynb b/notebooks/chapter_2_modeling/tutorial_3_realism_and_complexity.ipynb index 8690a37..3de8f19 100644 --- a/notebooks/chapter_2_modeling/tutorial_3_realism_and_complexity.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_3_realism_and_complexity.ipynb @@ -12,7 +12,7 @@ "\n", "In this tutorial, we'll use a more realistic model, which consists of the following light profiles:\n", "\n", - " - An `Sersic` light profile for the galaxy's bulge [7 parameters].\n", + " - A `Sersic` light profile for the galaxy's bulge [7 parameters].\n", " - An `Exponential` light profile for the galaxy's disk [6 parameters]\n", "\n", "This model has 13 free parameters, meaning that the parameter space and likelihood function it defines has a\n", @@ -74,7 +74,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -90,9 +90,9 @@ "source": [ "__Initial Setup__\n", "\n", - "we'll use new galaxying data, where:\n", + "we'll use new imaging data of a galaxy, where:\n", "\n", - " - The galaxy's bulge is an `Sersic`.\n", + " - The galaxy's bulge is a `Sersic`.\n", " - The galaxy's disk is an `Exponential`." ] }, @@ -238,7 +238,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the autogalaxy_workspace/output/howtogalaxy/chapter_2/tutorial_3_realism_and_complexity\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result = search.fit(model=model, analysis=analysis)\n", @@ -326,9 +326,9 @@ ")\n", "\n", "print(\n", - " \"The non-linear search has begun running - checkout the autogalaxy_workspace/output/3_realism_and_complexity\"\n", + " \"The non-linear search has begun running - checkout the autogalaxy_workspace/output/howtogalaxy/chapter_2/tutorial_3_realism_and_complexity__local_maxima\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_local_maxima = search.fit(model=model, analysis=analysis)\n", @@ -361,7 +361,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Lats look at the fit to the `Imaging` data, which is clearly worse than our original fit above." + "Lets look at the fit to the `Imaging` data, which is clearly worse than our original fit above." ] }, { @@ -379,7 +379,7 @@ "source": [ "Finally, just to be sure we hit a local maxima, lets compare the maximum log likelihood values of the two results \n", "\n", - "The local maxima value is significantly lower, confirming that our non-linear search simply failed to locate lens \n", + "The local maxima value is significantly lower, confirming that our non-linear search simply failed to locate galaxy\n", "models which fit the data better when it searched parameter space." ] }, @@ -406,13 +406,13 @@ "inferring a local maxima is still very real, especially as we make our model more complex.\n", "\n", "Lets think about *complexity*. As we make our model more realistic, we also made it more complex. For this \n", - "tutorial, our non-linear parameter space went from 7 dimensions to 13. This means there was a much larger *volume* of \n", + "tutorial, our non-linear parameter space went from 6 dimensions to 13. This means there was a much larger *volume* of\n", "parameter space to search. As this volume grows, there becomes a higher chance that our non-linear search gets lost \n", "and infers a local maxima, especially if we don't set it up with enough live points!\n", "\n", "At its core, modeling is all about learning how to get a non-linear search to find the global maxima region of \n", - "parameter space, even when the model is complex. This will be the main theme throughout the rest of this chapter\n", - "and is the main subject of chapter 3.\n", + "parameter space, even when the model is complex. This will be the main theme throughout the rest of this chapter,\n", + "culminating in the 'search chaining' technique that closes it.\n", "\n", "In the next exercise, we'll learn how to deal with failure and begin thinking about how we can ensure our non-linear \n", "search finds the global-maximum log likelihood solution. First, think about the following:\n", @@ -423,9 +423,9 @@ " 2) The non-linear search failed because parameter space was too complex. Could we make it less complex, whilst \n", " still keeping our model fairly realistic?\n", " \n", - " 3) The galaxy in this example had only 7 non-linear parameters. Real galaxies may have multiple components (e.g. a \n", - " disk, bulge, bar, star-forming knot) and there may even be more than 1 galaxy! Do you think there is any hope of \n", - " us navigating a parameter space if the galaxies contributes 30+ parameters?" + " 3) The galaxy in this example had only 13 non-linear parameters. Real galaxies may have multiple components (e.g. a\n", + " disk, bulge, bar, star-forming knot) and there may even be more than 1 galaxy! Do you think there is any hope of\n", + " us navigating a parameter space if the galaxies contribute 30+ parameters?" ] } ], diff --git a/notebooks/chapter_2_modeling/tutorial_4_dealing_with_failure.ipynb b/notebooks/chapter_2_modeling/tutorial_4_dealing_with_failure.ipynb index 20dac10..a59ebfa 100644 --- a/notebooks/chapter_2_modeling/tutorial_4_dealing_with_failure.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_4_dealing_with_failure.ipynb @@ -74,7 +74,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from pathlib import Path\n", @@ -93,7 +93,7 @@ "\n", "we'll use the same galaxy data as the previous tutorial, where:\n", "\n", - " - The galaxy's bulge is an `Sersic`.\n", + " - The galaxy's bulge is a `Sersic`.\n", " - The galaxy's disk is an `Exponential`." ] }, @@ -262,10 +262,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "By default, the prior on the $(y,x)$ coordinates of a `LightProfile` is a GaussianPrior with \n", + "By default, the prior on the $(y,x)$ coordinates of a `LightProfile` is a GaussianPrior with\n", "`mean=0.0` and `sigma=0.3`. However, visual inspection of our galaxy image tells us that its centre (based on the\n", - "galaxy's luminous emission) is at x = 0.0\" and y = 0.0\", so lets reduce the `sigma` value on this prior so the\n", - "non-linear search looks over a very narrow range of `centre` values in parameter space." + "galaxy's luminous emission) is at x = 0.0\" and y = 0.0\", so lets replace this prior with a narrow `UniformPrior`, so\n", + "the non-linear search looks over a very narrow range of `centre` values in parameter space." ] }, { @@ -284,8 +284,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "By default, the elliptical components of the of our galaxy's elliptical `LightProfile` are `TruncatedGaussianPrior`'s \n", - "with `mean=0.0` and `sigma=0.5`. Note that the solution `ell_comps=(0.0, 0.0)` corresponds to a spherical system\n", + "By default, the elliptical components of our galaxy's elliptical `LightProfile` are `TruncatedGaussianPrior`'s\n", + "with `mean=0.0` and `sigma=0.3`. Note that the solution `ell_comps=(0.0, 0.0)` corresponds to a spherical system\n", "and that all physical solutions (e.g. with axis-ratios running from 0.0 -> 1.0 and position angles 0.0 -> 180.0 degrees) \n", "are encapsulated for solutions where each component runs from -1.0 -> 1.0). \n", "\n", @@ -357,7 +357,7 @@ "The `sersic_index` defines how concentrated the light profile is. In galaxy structure studies, values of Sersic index\n", "around 1.0 indicate a disk galaxy (which is the value the `Exponential` uses). \n", "\n", - "Higher values of 3 or 4 indicate an elliptical galaxy. **PyAutoGalaxy** assumes a `UniformPrior` between 0.8 and 8.0 \n", + "Higher values of 3 or 4 indicate an elliptical galaxy. **PyAutoGalaxy** assumes a `UniformPrior` between 0.8 and 5.0\n", "by default on this parameter, as a user could model galaxies\n", "where the galaxy is of any morphology.\n", "\n", @@ -380,12 +380,7 @@ "metadata": {}, "source": [ "We now compose the overall model, where the galaxy model uses the `Model` components above which had their\n", - "priors customizes.\n", - "\n", - "In this exercise, I'm not going to change any priors on the galaxy. Whilst modeling experts can look at a \n", - "galaxy and often tell you roughly where the galaxy is located, it is something of art \n", - "form. Furthermore, the source's morphology can be pretty complex, making it difficult to come up with a good source \n", - "prior!" + "priors customized." ] }, { @@ -440,7 +435,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_4_custom_priors\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_custom_priors = search.fit(model=model, analysis=analysis)\n", @@ -491,8 +486,8 @@ "Disadvantages: \n", "\n", " - If we specified a prior incorrectly the non-linear search will infer an incorrect solution.\n", - " - The priors for the search were tailored to the specific galaxy we fitted. If we are fitting multiple galaxies, \n", - " we would have customize the priors for every single fit, for large samples of galaxies this would take a lot of time!\n", + " - The priors for the search were tailored to the specific galaxy we fitted. If we are fitting multiple galaxies,\n", + " we would have to customize the priors for every single fit, for large samples of galaxies this would take a lot of time!\n", "\n", "__Approach 2: Reducing Complexity__\n", "\n", @@ -593,7 +588,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_4_reducing_complexity\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_bulge_disk_align = search.fit(model=model, analysis=analysis)\n", @@ -628,8 +623,8 @@ "\n", "Advantages:\n", "\n", - " - By reducing parameter space`s complexity we again had a higher chance of inferring the global maximum log \n", - " likelihood and the time required by the search to do this is reducing.\n", + " - By reducing parameter space`s complexity we again had a higher chance of inferring the global maximum log\n", + " likelihood and the time required by the search to do this is reduced.\n", " - Unlike tuned priors, the search was not specific to one galaxy and we could run it on many galaxy images.\n", " \n", "Disadvantages:\n", @@ -639,7 +634,7 @@ "__Approach 3: Look Harder__\n", "\n", "In approaches 1 and 2 we extended our non-linear search an olive branch and helped it find the highest log likelihood \n", - "regions of parameter space. In approach 3 ,we're going to tell it to just `look harder`.\n", + "regions of parameter space. In approach 3, we're going to tell it to just `look harder`.\n", "\n", "Every non-linear search has settings which govern how thoroughly it searches parameter space, with the number of live\n", "points that was passed to `Nautilus` an example of such a setting. The more thoroughly the search looks, the more likely \n", @@ -648,8 +643,8 @@ "We create a more thorough `nautilus` search, that uses `n_live=200`. What these settings\n", "are actually changing is discussed in the optional tutorial `HowToGalaxy/chapter_optional/tutorial_searches.ipynb`.\n", "\n", - "Due to the long run times of this search, we comment it output below so it does not run. Feel free to undo these\n", - "comments so the script runs faster." + "Due to the long run times of this search, we comment out the fit below so it does not run. Feel free to undo these\n", + "comments if you wish to perform the search yourself." ] }, { @@ -671,7 +666,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_4_look_harder\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "# result_look_harder = search.fit(model=model, analysis=analysis)\n", @@ -701,12 +696,12 @@ "\n", "Disadvantage:\n", " \n", - " - Its potentially expensive. Very expensive. For very complex models, the run times can hours, days, weeks or, dare \n", + " - Its potentially expensive. Very expensive. For very complex models, the run times can be hours, days, weeks or, dare\n", " I say it, months!\n", "\n", "So, we can now fit galaxies. And when it fails, we know how to get it to work. \n", "\n", - "In chapter 3 of **HowToGalaxy**, we will introduce a technique called 'non-linear search chaining', which performs a \n", + "At the end of this chapter, we will introduce a technique called 'non-linear search chaining', which performs a\n", "model fit by chaining together multiple searches back-to-back . This allows us to combine the 3 different approaches \n", "discussed and exploit the advantages of each, whilst not being hindered by their disadvantages.\n", "\n", diff --git a/notebooks/chapter_2_modeling/tutorial_5_linear_profiles.ipynb b/notebooks/chapter_2_modeling/tutorial_5_linear_profiles.ipynb index 4cf9795..d4f33eb 100644 --- a/notebooks/chapter_2_modeling/tutorial_5_linear_profiles.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_5_linear_profiles.ipynb @@ -88,7 +88,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from pathlib import Path\n", @@ -107,7 +107,7 @@ "\n", "we'll use the same galaxy data as the previous tutorial, where:\n", "\n", - " - The galaxy's bulge is an `Sersic`.\n", + " - The galaxy's bulge is a `Sersic`.\n", " - The galaxy's disk is an `Exponential`." ] }, @@ -311,10 +311,10 @@ "For linear light profiles, the log likelihood evaluation increases to around ~0.05 seconds per likelihood evaluation.\n", "This is still fast, but it does mean that the fit may take around five times longer to run.\n", "\n", - "However, because two free parameters have been removed from the model (the `intensity` of the lens bulge and \n", - "source bulge), the total number of likelihood evaluations will reduce. Furthermore, the simpler parameter space\n", - "likely means that the fit will take less than 10000 per free parameter to converge. This is aided further\n", - "by the reduction in `n_live` to 100.\n", + "However, because two free parameters have been removed from the model (the `intensity` of the bulge and\n", + "disk), the total number of likelihood evaluations will reduce. Furthermore, the simpler parameter space\n", + "likely means that the fit will take less than 10000 likelihood evaluations per free parameter to converge. This is\n", + "aided further by the reduction in `n_live` to 100.\n", "\n", "Fits using standard light profiles and linear light profiles therefore take roughly the same time to run. However,\n", "the simpler parameter space of linear light profiles means that the model-fit is more reliable, less susceptible to\n", @@ -330,7 +330,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_5_linear_light_profile\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_linear_light_profile = search.fit(model=model, analysis=analysis)" @@ -439,11 +439,9 @@ "\n", " - All share the same centre and elliptical components.\n", " - The `sigma` size of the Gaussians increases in log10 increments.\n", - " \n", - "Because `log10(1.0) = 0.0` the first Gaussian `sigma` value is therefore 0.0001, whereas because `log10(10) = 1.0`\n", - "the size of the final Gaussian is 1.0. \n", "\n", - "The equation below has therefore been chosen to provide intuition on the scale of the Gaussians." + "The `sigma` values are spaced in log10 increments from a tenth of the pixel scale (0.01\") to the mask\n", + "radius (3.0\"), ensuring the Gaussians span all scales the data constrains." ] }, { @@ -518,11 +516,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "By plotting the fit, we see that the `Basis` does a reasonable job at capturing the appearance of the lens galaxy.\n", + "By plotting the fit, we see that the `Basis` does a reasonable job at capturing the appearance of the galaxy.\n", "\n", "There are imperfections, but this is because we did not fit the model via a non-linear search in order to determine\n", "the optimal values of the Gaussians in the basis. In particular, the Gaussians above were all spherical, when the\n", - "lens galaxy is elliptical. \n", + "galaxy is elliptical.\n", "\n", "We rectify this below, where we use a non-linear search to determine the optimal values of the Gaussians!" ] @@ -551,7 +549,7 @@ " - The centres and elliptical components of each family of Gaussians are all linked together.\n", " - The `sigma` size of the Gaussians increases in log10 increments.\n", "\n", - "The number of free parameters and therefore the dimensionality of the MGe is just N=4." + "The number of free parameters and therefore the dimensionality of the MGE is just N=4." ] }, { @@ -703,7 +701,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We now fit the model, with just `n_live=50` given the simiplicity of parameter space." + "We now fit the model, with just `n_live=50` given the simplicity of parameter space." ] }, { @@ -721,7 +719,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_2/tutorial_5_basis\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_basis = search.fit(model=model, analysis=analysis)" @@ -755,8 +753,8 @@ "Visualizing the fit shows that we successfully fit the data to the noise level.\n", "\n", "Note that the result objects `max_log_likelihood_galaxies` and `max_log_likelihood_fit` automatically convert\n", - "all linear light profiles to ordinary light profiles, including every single one of the 20 Gaussians fitted\n", - "above. \n", + "all linear light profiles to ordinary light profiles, including every single one of the 40 Gaussians fitted\n", + "above.\n", "\n", "This means we can use them directly to perform the visualization below." ] @@ -783,11 +781,11 @@ "__Multi Gaussian Expansion Benefits__\n", "\n", "Symmetric light profiles (e.g. elliptical Sersics) may leave significant residuals, because they fail to capture\n", - "irregular and asymmetric morphological of galaxies (e.g. isophotal twists, an ellipticity which varies radially).\n", + "irregular and asymmetric morphological features of galaxies (e.g. isophotal twists, an ellipticity which varies radially).\n", "An MGE fully captures these features and can therefore much better represent the emission of complex galaxies.\n", "\n", "The MGE model can be composed in a way that has fewer non-linear parameters than an elliptical Sersic. In this example,\n", - "a groups of Gaussians is used to represent the `bulge` of the galaxy, which in total correspond to just N=4 non-linear \n", + "a group of Gaussians is used to represent the `bulge` of the galaxy, which in total correspond to just N=4 non-linear\n", "parameters (a `bulge` and `disk` comprising two linear Sersics has N=10 parameters).\n", "\n", "The MGE model parameterization is also composed such that neither the `intensity` parameters or any of the\n", @@ -800,7 +798,7 @@ "\n", "__Positive Only Solver__\n", "\n", - "Many codes which use linear algebra typically rely on a linear algabra solver which allows for positive and negative\n", + "Many codes which use linear algebra typically rely on a linear algebra solver which allows for positive and negative\n", "values of the solution (e.g. `np.linalg.solve`), because they are computationally fast. \n", "\n", "This is problematic, as it means that negative surface brightnesses values can be computed to represent a galaxy's \n", @@ -816,19 +814,19 @@ "In addition to the Gaussians used in this example, there is another basis function implemented in PyAutoGalaxy \n", "that is commonly used to represent the light of galaxies, called a `Shapelet`. \n", "\n", - "Shapelets are basis functions with analytic properties that are appropriate for capturing the exponential / disk-like \n", - "features of a galaxy. They do so over a wide range of scales, and can often represent features in source galaxies \n", + "Shapelets are basis functions with analytic properties that are appropriate for capturing the exponential / disk-like\n", + "features of a galaxy. They do so over a wide range of scales, and can often represent features in galaxies\n", "that a single Sersic function or MGE cannot.\n", "\n", "An example using shapelets is given at `autogalaxy_workspace/scripts/imaging/features/shapelets/modeling.py`.\n", - " \n", - "Feel free to experiment with using shapelets as the galaxy by yourself. However they incur higher computational \n", - "overheads than the MGE and include a free parameter which governs the size of the basis functions and therefore source,\n", - "slowing down convergence of the non-linear search. We have found that MGEs perform better than shapelets in most \n", - "lens modeling problems. \n", "\n", - "If you have a desire to fit sources with even more complex morphologies we recommend you look at how to reconstruct \n", - "sources using pixelizations in the `modeling/features` section or chapter 4 of **HowToGalaxy**.\n", + "Feel free to experiment with using shapelets to model the galaxy yourself. However they incur higher computational\n", + "overheads than the MGE and include a free parameter which governs the size of the basis functions and therefore galaxy,\n", + "slowing down convergence of the non-linear search. We have found that MGEs perform better than shapelets in most\n", + "galaxy modeling problems.\n", + "\n", + "If you have a desire to fit galaxies with even more complex morphologies we recommend you look at how to reconstruct\n", + "galaxies using pixelizations in the `modeling/features` section or chapter 3 of **HowToGalaxy**.\n", "\n", "__Wrap Up__\n", "\n", @@ -837,25 +835,14 @@ "and robust.\n", "\n", "Perhaps the biggest downside to basis functions is that they are only as good as the features they can capture\n", - "in the data. For example, a baiss of Gaussians still assumes that they have a well defined centre, but there are\n", - "galaxies which may have multiple components with multiple centres (e.g. many star forming knots) which such a \n", - "basis cannot catprue.\n", + "in the data. For example, a basis of Gaussians still assumes that they have a well defined centre, but there are\n", + "galaxies which may have multiple components with multiple centres (e.g. many star forming knots) which such a\n", + "basis cannot capture.\n", "\n", - "In chapter 4 of **HowToGalaxy** we introduce non-parametric pixelizations, which reconstruct the data in way\n", + "In chapter 3 of **HowToGalaxy** we introduce non-parametric pixelizations, which reconstruct the data in way\n", "that does not make assumptions like a centre and can thus reconstruct even more complex, asymmetric and irregular\n", "galaxy morphologies." ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "basis = ag.lp_basis.Basis(\n", - " profile_list=gaussian_list, regularization=ag.reg.Constant(coefficient=1.0)\n", - ")\n" - ], - "outputs": [], - "execution_count": null } ], "metadata": { diff --git a/notebooks/chapter_2_modeling/tutorial_6_masking.ipynb b/notebooks/chapter_2_modeling/tutorial_6_masking.ipynb index 4c1ceeb..2cb955e 100644 --- a/notebooks/chapter_2_modeling/tutorial_6_masking.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_6_masking.ipynb @@ -7,7 +7,7 @@ "Tutorial 6: Masking\n", "===================\n", "\n", - "We have learnt everything we need to know about non-linear searches to model a galaxy and infer a good lens\n", + "We have learnt everything we need to know about non-linear searches to model a galaxy and infer a good\n", "model solution. Now, lets consider masking in more detail, something we have not given much consideration previously.\n", "We'll also learn a neat trick to improve the speed and accuracy of a non-linear search.\n", "\n", @@ -63,7 +63,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -81,7 +81,7 @@ "\n", "we'll use the same galaxy data as tutorials 1 & 2, where:\n", "\n", - " - The galaxy's `LightProfile` is an `Sersic`." + " - The galaxy's `LightProfile` is a `Sersic`." ] }, { @@ -187,7 +187,7 @@ "source": [ "__Model + Search + Analysis__\n", "\n", - "Lets fit the data using this mask, by creating the search as per usuag. Note that the `imaging` data with this mask\n", + "Lets fit the data using this mask, by creating the search as per usual. Note that the `imaging` data with this mask\n", "applied is passed into the `AnalysisImaging` object, ensuring that this is the mask the model-fit uses. " ] }, @@ -195,7 +195,7 @@ "cell_type": "code", "metadata": {}, "source": [ - "galaxy = af.Model(ag.Galaxy, redshift=1.0, bulge=ag.lp.Sersic)\n", + "galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic)\n", "model = af.Collection(galaxies=af.Collection(galaxy=galaxy))\n", "\n", "search = af.Nautilus(\n", @@ -219,7 +219,7 @@ "source": [ "__Discussion__\n", "\n", - "So, we can choose the mask we use in a model-fit. We know that we want the mask remove as little of the galaxy's light, \n", + "So, we can choose the mask we use in a model-fit. We know that we want the mask to remove as little of the galaxy's light,\n", "but is this the 'right' mask? What is the 'right' mask? Maybe we want a bigger mask? a smaller mask?\n", "\n", "When it comes to choosing a mask, we are essentially balancing two things: computational run-time and accuracy. When we\n", @@ -230,7 +230,7 @@ "If run-time was not a consideration we would always choose a bigger mask, for two reasons:\n", "\n", " 1) The galaxy may have very faint emission that when you choose the mask you simply do not notice. Overly aggressive \n", - " masking runs the risk of us inadvertantly masking out some of the galaxy's light, which would otherwise better \n", + " masking runs the risk of us inadvertently masking out some of the galaxy's light, which would otherwise better\n", " constrain the model!\n", " \n", " 2) When the data is fitted with a model image, the fit is performed only within the masked region. For certain galaxies\n", @@ -244,19 +244,19 @@ "use as large of a mask as possible (whilst still achieving reasonable run-times). Aggressive masking will make \n", "**PyAutoGalaxy** run very fast, but could lead you to infer an incorrect model! \n", "\n", - "In chapter 3, where we introduce 'non-linear search chaining' we will see how we can use tighter masks in earlier \n", - "searches to achieve faster run times.\n", + "At the end of this chapter, where we introduce 'non-linear search chaining', we will see how chaining makes it\n", + "possible to use tighter masks in earlier searches to achieve faster run times.\n", "\n", - "If your data includes the light of additional galaxies nearby you may much have no choice but to use a smaller \n", - "circular mask, because it is important these objects do not interfere with the fit. \n", + "If your data includes the light of additional galaxies nearby you may have no choice but to use a smaller\n", + "circular mask, because it is important these objects do not interfere with the fit.\n", "\n", - "In fact, you can drawcustom masks that remove their light entirely. You may now wish to checkout \n", - "the `autogalaxy_workspace/*/imaging/data_preparation` package. This includes tools for creating custom masks and \n", + "In fact, you can draw custom masks that remove their light entirely. You may now wish to checkout\n", + "the `autogalaxy_workspace/*/imaging/data_preparation` package. This includes tools for creating custom masks and\n", "marking the positions on a galaxy (via a GUI) so you can use them in a model-fit.\n", "\n", "__Wrap Up__\n", "\n", - "There are is one thing you should bare in mind in terms of masking:\n", + "There is one thing you should bear in mind in terms of masking:\n", "\n", " 1) Customizing the mask for the analysis of one galaxy gets the analysis running fast and can provide accurate \n", " non-linear sampling. However, for a large sample of galaxies, this high level of customization may take a lot of time. " diff --git a/notebooks/chapter_2_modeling/tutorial_7_results.ipynb b/notebooks/chapter_2_modeling/tutorial_7_results.ipynb index f3b9700..d14fcc1 100644 --- a/notebooks/chapter_2_modeling/tutorial_7_results.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_7_results.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Tutorial 5: Results\n", + "Tutorial 7: Results\n", "===================\n", "\n", "In the previous tutorials, each search returned a `Result` object, which we used to plot the maximum log likelihood\n", - "fit each model-fit. In this tutorial, we'll take a look at the result object in a little more detail.\n", + "fit of each model-fit. In this tutorial, we'll take a look at the result object in a little more detail.\n", "\n", "__Contents__\n", "\n", @@ -63,7 +63,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -79,7 +79,7 @@ "source": [ "__Initial Setup__\n", "\n", - "Lets use the model-fit performed in tutorial 1 to get a `Result` object." + "Lets repeat the simple Sersic model-fit performed earlier in this chapter to get a `Result` object." ] }, { @@ -139,12 +139,12 @@ "dataset = dataset.apply_over_sampling(over_sample_size_lp=over_sample_size)\n", "\n", "model = af.Collection(\n", - " galaxies=af.Collection(galaxy=af.Model(ag.Galaxy, redshift=0.5, mass=ag.lp.Sersic))\n", + " galaxies=af.Collection(galaxy=af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic))\n", ")\n", "\n", "search = af.Nautilus(\n", " path_prefix=Path(\"howtogalaxy\", \"chapter_2\"),\n", - " name=\"tutorial_1_non_linear_search\",\n", + " name=\"tutorial_7_results\",\n", " unique_tag=dataset_name,\n", " n_live=80,\n", " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", @@ -192,7 +192,7 @@ "\n", "For example, the `Samples` object contains all of the non-linear search samples, including the parameters \n", "of every successful model evaluation and their log likelihood values. These are used for computing information \n", - "about the model-fit, such as the most probably parameter estimates and the error inferred for every parameter." + "about the model-fit, such as the most probable parameter estimates and the error inferred for every parameter." ] }, { @@ -220,7 +220,7 @@ "\n", " `autogalaxy_workspace/*/results`\n", " \n", - "The results API for CCD imaging data is the same as for other data types (e.g. interferometer, point-soures). This\n", + "The results API for CCD imaging data is the same as for other data types (e.g. interferometer). This\n", "package can therefore be used to learn the API and then translate to other data types.\n", "\n", "__Database__\n", @@ -248,7 +248,7 @@ " - Use the `Samples` to produce many different results from the fit, including error estimates on parameters and \n", " plots of the probability density function of parameters in 1D and 2D.\n", " \n", - " - Visualize results, for example the fit to a lens dataset.\n", + " - Visualize results, for example the fit to a galaxy dataset.\n", "\n", "\n", "__Wrap Up__\n", @@ -256,7 +256,7 @@ "Even if you are only modeling a small sample of galaxies, if you anticipate using **PyAutoGalaxy** for the long-term I \n", "strongly recommend you begin using the database to inspect and analyse your result. \n", "\n", - "This is because it makes it simple to perform all analyse in a Jupyter notebook, which is the most flexible and \n", + "This is because it makes it simple to perform all analysis in a Jupyter notebook, which is the most flexible and\n", "versatile way to check results and make figures." ] } diff --git a/notebooks/chapter_2_modeling/tutorial_8_need_for_speed.ipynb b/notebooks/chapter_2_modeling/tutorial_8_need_for_speed.ipynb index bb8d393..600140a 100644 --- a/notebooks/chapter_2_modeling/tutorial_8_need_for_speed.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_8_need_for_speed.ipynb @@ -10,67 +10,141 @@ "In this chapter, we have learnt how to model galaxies and how to balance complexity and realism to ensure that we\n", "infer a good model.\n", "\n", - "__Contents__\n", - "\n", - "- **Searching Non-linear Parameter Space:** How dimensionality, priors and settings drive search run-times.\n", - "- **Algorithmic Optimization:** How PyAutoGalaxy uses numba and JAX for fast computation.\n", - "- **Data Quantity:** How the number of image pixels affects run-time.\n", - "- **Wrap Up:** Summary of strategies for keeping model-fit run-times manageable.\n", - "\n", "For fitting more complex models, the final challenge that we face is keeping the run-time low. One can easily end\n", "up in a situation where a model-fit takes days, or longer, to fit just one image. For fitting complex models and high\n", "resolution datasets this is somewhat unavoidable. However, it is worth us discussing what drives the long run-times of\n", "the modeling process and how we might speed it up.\n", "\n", + "We have kept an eye on the log likelihood evaluation times and therefore expected overall run-time of the fits\n", + "performed in this chapter. Lets quickly remind ourselves of the main factors that drive the run-time, and how we\n", + "might reduce it.\n", + "\n", + "The overall run-time of a model-fit is, to a good approximation, a simple product of two numbers:\n", + "\n", + " - The number of log likelihood evaluations the non-linear search performs before it converges.\n", + "\n", + " - The time each individual log likelihood evaluation takes.\n", + "\n", + "Everything we discuss below reduces one (or both) of these numbers.\n", + "\n", + "__Contents__\n", + "\n", + "- **Searching Non-linear Parameter Space:** What drives the number of likelihood evaluations a search performs.\n", + "- **Cost Per Evaluation:** What drives the time a single log likelihood evaluation takes.\n", + "- **JAX:** How **PyAutoGalaxy** compiles the likelihood function with JAX and runs it on CPUs and GPUs.\n", + "- **Run Time Estimation:** How to estimate the overall run-time of a fit before committing to it.\n", + "- **Data Quantity:** How the amount of data fitted, and the mask applied to it, drives run-times.\n", + "- **Wrap Up:** Summary of the script and next steps.\n", + "\n", "__Searching Non-linear Parameter Space__\n", "\n", "The time it takes for the non-linear search to sample parameter space and find the high likelihood models is driven by:\n", "\n", " - Dimensionality: A more complex parameter space (e.g. more parameters) takes longer to search.\n", " - Priors: The broader the priors on each parameter the longer the search.\n", - " - Settings: Non-linear search settings which sample parameter space more thoroughly lead to longer run-times.\n", + " - Settings: Non-linear search settings which sample parameter space more thoroughly (e.g. a higher `n_live`) lead to\n", + " longer run-times.\n", "\n", "When we use only one search to fit a model, we are somewhat restricted in how we can try to achieve faster run\n", - "times by changing these 3 aspects of the search.\n", + "times by changing these 3 aspects of the search. We have already seen one powerful exception: the linear light\n", + "profiles of tutorial 5 remove every `intensity` parameter from the non-linear parameter space, reducing its\n", + "dimensionality and removing some of its most difficult degeneracies at the same time.\n", "\n", - "In the next chapter, we introduce 'non-linear search chaining', which fits a model using multiple searches that\n", - "are performed back-to-back. A key motivation for this is that it gives us a lot more flexibility in juggling the\n", - "dimensionality, priors and settings so as to perform faster and more efficient modeling.\n", + "The final two tutorials of this chapter (tutorials 9 and 10) introduce 'non-linear search chaining', which fits\n", + "a model using multiple searches that are performed back-to-back, passing the results of earlier searches to\n", + "later ones. A key motivation for this is that it gives us a lot more flexibility in juggling the dimensionality,\n", + "priors and settings so as to perform faster and more efficient modeling.\n", "\n", - "In the optional **HowToGalaxy** tutorial `chapter_optional/tutorial_searches.ipynb` we discuss other non-linear\n", - "searches supported by **HowToGalaxy** which use a different approach to sample parameter sample than `nautilus`. For\n", - "those familiar with statistical inference, this includes maximum likelihood estimators and MCMC algorithms.\n", + "In the optional **HowToGalaxy** tutorial `chapter_optional/tutorial_searches.py` we discuss other non-linear searches\n", + "supported by **PyAutoGalaxy** which use a different approach to sample parameter space than `Nautilus`. For those\n", + "familiar with statistical inference, this includes maximum likelihood estimators and MCMC algorithms.\n", "\n", "For galaxy modeling, there are maximum likelihood estimator methods (e.g. the Levenberg-Marquardt) that for simple\n", - "models (e.g. a single `Sersic`) can often reliable infer the maximum likelihood model, using 10 times or fewer\n", - "likelihood evaluations than `nautilus` therefore running over ten times or more faster). However, these methods do\n", - "not infer reliable errors and are subject to inferring local maximum. Nevertheless, users modeling large galaxy\n", - "samples may wish to investigate these methods.\n", + "models (e.g. a single `Sersic`) can often reliably infer the maximum likelihood model, using ten times or fewer\n", + "likelihood evaluations than `Nautilus` and therefore running over ten times faster. However, these methods do\n", + "not infer reliable errors and are prone to becoming trapped in local maxima. Nevertheless, users modeling large\n", + "galaxy samples may wish to investigate these methods.\n", + "\n", + "__Cost Per Evaluation__\n", + "\n", + "The second number is the time a single log likelihood evaluation takes, which is set by every operation\n", + "**PyAutoGalaxy** performs to fit the galaxy data with a model, for example:\n", + "\n", + " - Computing the intensity values of every light profile on the image's grid of (y,x) coordinates.\n", + " - Over-sampling: sub-dividing image pixels into finer sub-pixels where the light profile changes rapidly (e.g. the\n", + " centre of the galaxy), which multiplies the number of intensity evaluations performed.\n", + " - Convolving the model image of the galaxies with the telescope's PSF to compare it to the data. The larger the\n", + " PSF, the more expensive this convolution becomes.\n", + " - For linear light profiles, basis functions like the MGE of tutorial 5 and the pixelizations of chapter 3, solving\n", + " the linear algebra of the inversion, which computes the `intensity` values that best fit the data.\n", + "\n", + "More complex fitting techniques therefore cost more per evaluation: a couple of standard light profiles are cheap,\n", + "an MGE costs more (its inversion solves for many Gaussians), and pixelized reconstructions cost the most.\n", + "As we saw in tutorial 5, this trade often still favours the more advanced technique, because the simpler parameter\n", + "space it produces reduces the number of evaluations the search needs.\n", + "\n", + "__JAX__\n", + "\n", + "How are these operations made fast? **PyAutoGalaxy** uses JAX (https://github.com/jax-ml/jax), Google's numerical\n", + "computing library, which just-in-time (JIT) compiles the entire log likelihood function into optimized machine code.\n", + "\n", + "JIT compilation means the first time the likelihood function is called, JAX traces through every operation it\n", + "performs (light profile evaluation, over-sampling, PSF convolution, the linear algebra of an inversion) and compiles\n", + "them into a single optimized program. This compilation is a one-off overhead, typically taking seconds to a couple\n", + "of minutes depending on model complexity. Every one of the tens of thousands of likelihood evaluations that follow\n", + "then reuses the compiled program and runs dramatically faster than ordinary Python. When a search begins you will\n", + "see a log message like `JAX jit compiling vectorized (vmap) likelihood function, could take seconds or minutes...`\n", + "-- that is the compilation happening.\n", + "\n", + "JAX's second superpower is that the same compiled code runs on either a CPU or a GPU. On a GPU, likelihood\n", + "evaluations are not only faster individually but can be batched, with many evaluated simultaneously (the\n", + "`n_batch` search input we have set throughout this chapter controls the size of these batches). GPU speed-ups\n", + "over CPU of an order of magnitude or more are common, and they grow with the size of the dataset and model. If you\n", + "have access to a GPU (even a modest laptop GPU), it is the single biggest speed-up available to you.\n", + "\n", + "You do not need to do anything to switch this on. If JAX is installed (`pip install autogalaxy[jax]`), every\n", + "`AnalysisImaging` object we have created in this chapter defaults to `use_jax=True` and the search compiles and\n", + "batches the likelihood function automatically. You can force the plain NumPy path with\n", + "`ag.AnalysisImaging(dataset=dataset, use_jax=False)` (or by setting the environment variable\n", + "`PYAUTO_DISABLE_JAX=1`), which is mainly useful for debugging, as NumPy errors and stack traces are much easier\n", + "to read than JAX ones.\n", + "\n", + "The one trade-off to keep in mind is the compilation overhead itself. For a full model-fit performing tens of\n", + "thousands of likelihood evaluations it is negligible. For a quick one-off calculation (e.g. fitting galaxies to\n", + "data once via a `FitImaging`, as we did in chapter 1) compilation can take longer than the calculation it speeds\n", + "up, which is why short interactive computations sometimes feel slower than you might expect the first time they run.\n", + "\n", + "Long-time users may remember that **PyAutoGalaxy** previously used `numba`, which recompiles individual Python\n", + "functions into fast machine code. JAX supersedes it: rather than accelerating functions one at a time, it compiles\n", + "and optimizes the likelihood function end-to-end, and adds GPU support and batching on top.\n", "\n", - "__Algorithmic Optimization__\n", + "Therefore, **PyAutoGalaxy** is pretty well optimized and there are no 'low hanging fruit' speed ups available by\n", + "writing the code in a different language.\n", "\n", - "Every operation **PyAutoGalaxy** performs to fit galaxy data with a model takes time, for example:\n", + "__Run Time Estimation__\n", "\n", - " - Computing the intensity values from a light profile.\n", - " - Convolving the galaxies image with the PSF to compare it to the data.\n", + "Because run-time is the product of the two numbers above, we can estimate it before committing to a fit:\n", "\n", - "One can therefore in principle make **PyAutoGalaxy** run faster by using more efficient algorithms. However, I am\n", - "confident that for many tasks and operations we have written code that is already very fast!\n", + " - Time the log likelihood evaluation: perform a fit of the model to the data once (e.g. via a `FitImaging`) and\n", + " time it, remembering to discard the first call, which includes JIT compilation.\n", "\n", - "I often get asked, given that **PyAutoGalaxy** is written in Python (a synonymously slow programming language), is it\n", - "not really slow? **PyAutoGalaxy** uses a library called `numba` to ensure that it runs fast, which recompiles Python\n", - "functions into C functions before **PyAutoGalaxy** runs. This gives us C-like speed, but in Python code. If you`ve got\n", - "your own code that needs speeding up, I strongly recommend that you look up Numba:\n", + " - Estimate the number of evaluations: for `Nautilus` a reasonable rule of thumb is of order 10000 evaluations per\n", + " free parameter, with simpler parameter spaces (e.g. linear light profiles, chained searches with tight priors)\n", + " converging in fewer.\n", "\n", - "http://numba.pydata.org/\n", + "Multiplying the two gives a ballpark overall run-time. In tutorial 2 we used exactly this arithmetic: a log\n", + "likelihood evaluation time of ~0.05 seconds and 50000-100000 evaluations gave expected run-times of roughly 30\n", + "minutes on CPU and 10 minutes on GPU. If your own estimate comes out at days or weeks, that is the moment to\n", + "simplify the model, tighten the search settings, reach for search chaining or find a GPU -- before starting the\n", + "fit, not after.\n", "\n", - "Therefore, **PyAutoGalaxy** is pretty well optimized and there are no 'low hanging fruit' speed ups available by\n", - "writing the code in a different language.\n", + "On a GPU, there is one other pre-flight check: `analysis.print_vram_use(model=model, batch_size=search.batch_size)`\n", + "estimates whether the fit will fit within the GPU's memory for your chosen batch size.\n", "\n", "__Data Quantity__\n", "\n", "The final factor driving run-speed is the quantity of data that is fitted. For every image-pixel that we fit,\n", - "we have to compute the light profile intensities and convolve it with the telescope's PSF. The larger that PSF is,\n", + "we have to compute the light profile intensities and convolve them with the telescope's PSF. The larger that PSF is,\n", "the more convolution operations we have to perform too.\n", "\n", "In the previous exercises, we used images with a pixel scale of 0.1\". This value is relatively low resolution: most\n", @@ -80,11 +154,17 @@ "\n", "There are ways that we can reduce the number of image-pixels we fit, via masking. If we mask out more of the image,\n", "we will fit fewer pixels and **PyAutoGalaxy** will run faster. If you want the best, most perfect model possible,\n", - "aggressive masking and cutting the data in this way is a bad idea, as discussed in tutorial 5.\n", + "aggressive masking and cutting the data in this way is a bad idea, as discussed in tutorial 6.\n", "\n", "__Wrap Up__\n", "\n", - "This tutorial simply wanted to get you thinking about *why* a model takes as long to fit as it does." + "This tutorial simply wanted to get you thinking about *why* a model takes as long to fit as it does: the number\n", + "of likelihood evaluations the search performs, times the cost of each evaluation, with JAX's compiled likelihood\n", + "function (and a GPU, if you have one) driving the latter down as far as it will go.\n", + "\n", + "The remaining tutorials of this chapter attack the other half of the product: search chaining (tutorial 9) and prior\n", + "passing (tutorial 10) reduce the number of evaluations needed to fit complex models, laying the groundwork for the\n", + "pixelized reconstructions of chapter 3." ] }, { diff --git a/notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb b/notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb index 836ba07..f06251b 100644 --- a/notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb +++ b/notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb @@ -4,16 +4,20 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Tutorial 1: Search Chaining\n", + "Tutorial 9: Search Chaining\n", "===========================\n", "\n", - "In chapter 2, we learnt how to perform modeling using a non-linear search. In all of the tutorials, we fitted the\n", - "data using just one non-linear search. In this chapter, we introduce a technique called 'non-linear search chaining',\n", - "which fits a model using a sequence of non-linear searches. The initial searches fit simpler models whose parameter\n", - "spaces can be more accurately and efficiently sampled. The results of this search are then passed to later searches\n", - "which fit models of gradually increasing complexity.\n", + "Throughout this chapter, we have fitted the data using just one non-linear search. The previous tutorial discussed\n", + "the run-time cost of doing this: with a single search, the dimensionality of the model, the priors on its parameters\n", + "and the search settings must all be juggled at once, leaving us little flexibility to trade them off against one\n", + "another.\n", "\n", - "Lets think back to tutorial 4 of chapter 2. We learnt there were three approaches one could take fitting a model\n", + "In this closing part of the chapter, we introduce a technique called 'non-linear search chaining', which fits a\n", + "model using a sequence of non-linear searches. The initial searches fit simpler models whose parameter spaces can\n", + "be more accurately and efficiently sampled. The results of this search are then passed to later searches which fit\n", + "models of gradually increasing complexity.\n", + "\n", + "Lets think back to tutorial 4 of this chapter. We learnt there were three approaches one could take fitting a model\n", "accurately if we found that a model fit failed. These were:\n", "\n", " 1) Tuning our priors to the galaxy we're fitting.\n", @@ -25,24 +29,25 @@ "much on searching parameter space for longer, we could end up with search`s that take days, weeks or months to run.\n", "\n", "In this tutorial, we are going to show how search chaining combines these 3 approaches such that we can fit\n", - "complex and realistic models in a way that that can be generalized to many different galaxies. To do this,\n", - "we'll run 2 searches, and chain the model inferred in the first search to the priors of the second search`s lens\n", + "complex and realistic models in a way that can be generalized to many different galaxies. To do this,\n", + "we'll run 2 searches, and chain the model inferred in the first search to the priors of the second search`s\n", "model.\n", "\n", - "Our first search will make the same bulge-disk alignment assumption we made in the previous tutorial. We saw that this\n", - "gives a reasonable model. However, we'll make a couple of extra simplifying assumptions, to really try and bring\n", - "our model complexity down and get the non-linear search running fast.\n", + "Our first search will make the same bulge-disk alignment assumption we made in tutorial 4 of this chapter. We saw\n", + "that this gives a reasonable model. However, we'll make a couple of extra simplifying assumptions, to really try and\n", + "bring our model complexity down and get the non-linear search running fast.\n", "\n", "The model we infer above will therefore be a lot less realistic. But it does not matter, because in the second search\n", "we are going to relax these assumptions and fit the more realistic model. The beauty is that, by running the first\n", "search, we can use its results to tune the priors of our second search. For example:\n", "\n", " 1) The first search should give us a pretty good idea of the galaxy's bulge and disk profiles, for example its\n", - " centre, intensity, effective radius.\n", + " centre, ellipticity and effective radius.\n", "\n", "__Contents__\n", "\n", "- **Initial Setup:** Load the dataset and apply a mask.\n", + "- **Dataset Auto-Simulation:** Simulate the dataset via its simulator script if it is not on your hard-disk.\n", "- **Model:** Compose a simplified model for the first search with aligned bulge-disk assumptions.\n", "- **Search + Analysis:** Run the first search with the simplified model.\n", "- **Result:** Inspect the result of the first search.\n", @@ -94,7 +99,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import numpy as np\n", "from pathlib import Path\n", @@ -111,9 +116,9 @@ "source": [ "__Initial Setup__\n", "\n", - "we'll use the same galaxy data as tutorial 4 of chapter 2, where:\n", + "we'll use the same galaxy data as tutorial 4 of this chapter, where:\n", "\n", - " - The galaxy's bulge is an `Sersic`.\n", + " - The galaxy's bulge is a `Sersic`.\n", " - The galaxy's disk is an `Exponential`." ] }, @@ -222,7 +227,41 @@ "bulge.centre_0 = 0.0\n", "bulge.centre_1 = 0.0\n", "disk.centre_0 = 0.0\n", - "disk.centre_1 = 0.0\n", + "disk.centre_1 = 0.0" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Lets also use the same approach as tutorial 4 of this chapter and align the ellipticity of the bulge and disk,\n", + "removing another two parameters from the model." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "disk.ell_comps = bulge.ell_comps" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also know that the bulges of many galaxies are well described by a Sersic index of around 4 (a de Vaucouleurs\n", + "profile). Lets fix it to 4 this time." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "bulge.sersic_index = 4.0\n", "\n", "galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge, disk=disk)\n", "\n", @@ -261,11 +300,11 @@ "metadata": {}, "source": [ "search_1 = af.Nautilus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_3\"),\n", - " name=\"tutorial_1_search_chaining_1\",\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_2\"),\n", + " name=\"tutorial_9_search_chaining_1\",\n", " unique_tag=dataset_name,\n", " n_live=100,\n", - " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", + " n_batch=50, # GPU batching and VRAM use explained in tutorial 2 of this chapter.\n", ")\n", "\n", "analysis_1 = ag.AnalysisImaging(dataset=dataset, use_jax=True)" @@ -277,8 +316,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Lets run the search, noting that our liberal approach to reducing the model complexity has reduced it to just \n", - "6 parameters." + "Lets run the search, noting that our liberal approach to reducing the model complexity has reduced it to just\n", + "4 parameters." ] }, { @@ -286,9 +325,9 @@ "metadata": {}, "source": [ "print(\n", - " \"The non-linear search has begun running - checkout the workspace/output/5_chaining_searches\"\n", + " \"The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_1 = search_1.fit(model=model_1, analysis=analysis_1)\n", @@ -345,7 +384,7 @@ "possibility that there might be a better solution nearby. In contrast, `UniformPrior`'s put hard limits on what values a \n", "parameter can or can`t take. It makes it more likely we will accidentally cut-out the global maxima solution.\n", "\n", - "Note that below the `disk` has become an `Sersic`." + "Note that below the `disk` has become a `Sersic`." ] }, { @@ -366,7 +405,7 @@ "parameter was fixed in the previous search, its prior is based around the previous value. Don't worry about the sigma \n", "values for now, I've chosen values that I know will ensure reasonable sampling, but we'll cover this later.\n", "\n", - "__LENS BULGE PRIORS:__" + "__BULGE PRIORS:__" ] }, { @@ -399,7 +438,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "__LENS DISK PRIORS:__" + "__DISK PRIORS:__" ] }, { @@ -474,22 +513,20 @@ "cell_type": "code", "metadata": {}, "source": [ - "batch_size = 50 # Explained chapter 2 tutorial 2\n", - "\n", "search_2 = af.Nautilus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_3\"),\n", - " name=\"tutorial_1_search_chaining_2\",\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_2\"),\n", + " name=\"tutorial_9_search_chaining_2\",\n", " unique_tag=dataset_name,\n", " n_live=100,\n", - " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", + " n_batch=50, # GPU batching and VRAM use explained in tutorial 2 of this chapter.\n", ")\n", "\n", "analysis_2 = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", "\n", "print(\n", - " \"The non-linear search has begun running - checkout the workspace/output/5_chaining_searches\"\n", + " \"The non-linear search has begun running - checkout the output/howtogalaxy/chapter_2\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_2 = search_2.fit(model=model_2, analysis=analysis_2)\n", @@ -544,7 +581,7 @@ "pretty much any galaxy and therefore get ourselves a decent model with which to tune search 2`s priors.\n", "\n", "You are probably thinking though that there is one huge, giant, glaring flaw in all of this that I've not mentioned. \n", - "Search 2 can`t be generalized to another lens, because its priors are tuned to the image we fitted. If we had a lot \n", + "Search 2 can`t be generalized to another galaxy, because its priors are tuned to the image we fitted. If we had a lot\n", "of galaxies, we`d have to write a new search for every single one. This isn't ideal, is it?\n", "\n", "Fortunately, we can pass priors in **PyAutoGalaxy** without specifying the specific values. The API for this technique,\n", diff --git a/notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb b/notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb index 77c6482..3b417d9 100644 --- a/notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb +++ b/notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Tutorial 1: pixelizations\n", + "Tutorial 1: Pixelizations\n", "=========================\n", "\n", "In the previous chapters, we used light profiles to model the light of a galaxy, where the light profile was an\n", @@ -63,7 +63,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "import autogalaxy as ag\n", "import autogalaxy.plot as aplt\n", @@ -141,7 +141,7 @@ "By itself, a pixelization does not tell us much. It has no grid of $(y,x)$ coordinates, no image, and no information\n", "about the galaxy we are fitting. \n", "\n", - "This information comes when we use the pixelization to create up a `Mapper`, which we perform below using the grid \n", + "This information comes when we use the pixelization to create a `Mapper`, which we perform below using the grid\n", "that we created above." ] }, @@ -162,7 +162,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This `Mapper` is a `RectangularMapper` -- every `Mesh` and `Pixelization` generates it owns mapper." + "This `Mapper` is a `RectangularMapper` -- every `Mesh` and `Pixelization` generates its own mapper." ] }, { @@ -229,7 +229,7 @@ "plot_mapper(\n", " mapper=mapper,\n", " mesh_grid=mapper.source_plane_mesh_grid,\n", - " title=\"Recntagular Grid With Pixel Cenres\",\n", + " title=\"Rectangular Grid With Pixel Centres\",\n", ")" ], "outputs": [], @@ -311,8 +311,8 @@ "This was a relatively gentle overview of pixelizations, but one that was hopefully easy to follow. Think about the \n", "following questions before moving on to the next tutorial:\n", "\n", - " 1) The rectangular pixelization`s edges are aligned with the most exterior coordinates of the source-grid. This is \n", - " intentional, why do you think this is?" + " 1) The rectangular pixelization`s edges are aligned with the most exterior coordinates of the grid it was created\n", + " from. This is intentional, why do you think this is?" ] } ], diff --git a/notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb b/notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb index dabb7d2..d126e91 100644 --- a/notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb +++ b/notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb @@ -7,7 +7,7 @@ "Tutorial 2: Mappers\n", "===================\n", "\n", - "In the previous tutorial, we used a pixelization to create made a `Mapper`. However, it was not clear what a `Mapper`\n", + "In the previous tutorial, we used a pixelization to create a `Mapper`. However, it was not clear what a `Mapper`\n", "does, why it was called a mapper and whether it was mapping anything at all!\n", "\n", "Therefore, in this tutorial, we'll cover mappers in more detail.\n", @@ -15,6 +15,7 @@ "__Contents__\n", "\n", "- **Initial Setup:** Load the dataset for illustration.\n", + "- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist.\n", "- **Mappers:** Understand how mappers map image-plane pixels to pixelization pixels.\n", "- **Mask:** Apply a mask and see how it affects the mapper.\n", "- **Wrap Up:** Summary of mapper concepts." @@ -63,7 +64,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -79,11 +80,10 @@ "source": [ "__Initial Setup__\n", "\n", - "we'll use complex galaxy data, where:\n", + "we'll use galaxy data, where:\n", "\n", " - The galaxy's bulge is an `Sersic`.\n", - " - The galaxy's disk is an `Exponential`.\n", - " - The galaxy's has four star forming clumps which are `Sersic` profiles." + " - The galaxy's disk is an `Exponential`." ] }, { @@ -153,7 +153,7 @@ "source": [ "__Mappers__\n", "\n", - "We now setup a `Pixelization` and use it to create a `Mapper` via the plane`s source-plane grid, just like we did in\n", + "We now setup a `Pixelization` and use it to create a `Mapper` via the image's grid, just like we did in\n", "the previous tutorial.\n", "\n", "We will make its pixelization resolution half that of the grid above." @@ -182,7 +182,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We now plot the `Mapper` alongside the image we used to generate the source-plane grid.\n", + "We now plot the `Mapper` alongside the image we used to generate the grid.\n", "\n", "Using the `Visuals2D` object we are also going to highlight specific grid coordinates certain colors, such that we\n", "can see how they map from the image grid to the pixelization grid. " @@ -259,8 +259,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can now use the masked grid to create a new `Mapper` (using the same rectangular 25 x 25 pixelization \n", - "as before)." + "We can now use the masked grid to create a new `Mapper` (using the same rectangular pixelization as before)." ] }, { @@ -295,8 +294,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "First, We can see a red circle of dots in both the image and pixelization, showing where the edge of the mask\n", - "maps too in the pixelization.\n", + "First, we can see a red circle of dots in both the image and pixelization, showing where the edge of the mask\n", + "maps to in the pixelization.\n", "\n", "Now lets show that when we plot pixelization pixel indexes, they still appear in the same place in the image." ] @@ -330,9 +329,9 @@ "In this tutorial, we learnt about mappers, and we used them to understand how the image and pixelization map to one \n", "another. Your exercises are:\n", " \n", - " 1) Think about how this could help us actually model galaxies. We have said we're going to reconstruct our galaxies \n", - " on the pixel-grid. So, how does knowing how each pixel maps to the image actually help us? If you`ve not got \n", - " any bright ideas, then worry not, that exactly what we're going to cover in the next tutorial." + " 1) Think about how this could help us actually model galaxies. We have said we're going to reconstruct our galaxies\n", + " on the pixel-grid. So, how does knowing how each pixel maps to the image actually help us? If you`ve not got\n", + " any bright ideas, then worry not, that's exactly what we're going to cover in the next tutorial." ] } ], diff --git a/notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb b/notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb index d232e4f..ae38c66 100644 --- a/notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb +++ b/notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb @@ -12,16 +12,17 @@ " - `Pixelization`'s: which place a pixel-grid over the image data.\n", " - `Mappers`'s: which describe how each pixelization pixel maps to one or more image pixels.\n", "\n", - "However, non of this has actually helped us fit galaxy data or reconstruct the galaxy. This is the subject\n", + "However, none of this has actually helped us fit galaxy data or reconstruct the galaxy. This is the subject\n", "of this tutorial, where the process of reconstructing the galaxy's light on the pixelization is called an `Inversion`.\n", "\n", "__Contents__\n", "\n", "- **Initial Setup:** Load the dataset for illustration.\n", + "- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist.\n", "- **Pixelization:** Create a pixelization and perform an inversion to reconstruct the galaxy.\n", "- **Positive Only Solver:** Ensure the reconstruction has only positive intensity values.\n", - "- **Wrap Up:** Summary of inversion concepts.\n", - "- **Detailed Explanation:** In-depth explanation of the linear algebra behind inversions." + "- **Detailed Explanation:** Where the linear algebra behind inversions is derived in full.\n", + "- **Wrap Up:** Summary of inversion concepts." ] }, { @@ -67,7 +68,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -83,11 +84,10 @@ "source": [ "__Initial Setup__\n", "\n", - "we'll use the same complex galaxy data as the previous tutorial, where:\n", + "we'll use the same galaxy data as the previous tutorial, where:\n", "\n", " - The galaxy's bulge is an `Sersic`.\n", - " - The galaxy's disk is an `Exponential`.\n", - " - The galaxy's has four star forming clumps which are `Sersic` profiles." + " - The galaxy's disk is an `Exponential`." ] }, { @@ -266,16 +266,17 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "There we have it, we have successfully reconstructed the galaxy using a rectangular pixel-grid. This has reconstructed\n", - "the complex blobs of light of the galaxy.\n", + "There we have it, we have successfully reconstructed the galaxy using a rectangular pixel-grid, capturing the\n", + "structure of its light without assuming an analytic form for it.\n", "\n", - "Pretty great, huh? If you ran the complex source pipeline in chapter 3, you'll remember that getting a model image \n", - "that looked this good simply *was not possible*. With an inversion, we can do this with ease and without having to \n", + "Pretty great, huh? If you fitted this galaxy via search chaining at the end of chapter 2 (tutorials 9-10), you'll\n", + "remember how much effort a good model image took. With an inversion, we can do this with ease and without having to\n", "perform model-fitting with 20+ parameters for the galaxy's light!\n", "\n", - "We will now briefly discuss how an inversion actually works, however the explanation I give in this tutorial will be \n", - "overly-simplified. To be good at modeling you do not need to understand the details of how an inversion works, you \n", - "simply need to be able to use an inversion to model a galaxy. \n", + "We will now briefly discuss how an inversion actually works, however the explanation given in this tutorial will be\n", + "overly-simplified. (The full linear algebra -- the mapping matrices, the linear solve and the equations behind\n", + "them -- is derived step-by-step in tutorial 5.) To be good at modeling you do not need to understand the details of\n", + "how an inversion works, you simply need to be able to use an inversion to model a galaxy.\n", "\n", "To begin, lets consider some random mappings between our mapper`s pixelization pixels and the image." ] @@ -371,34 +372,35 @@ "source": [ "__Positive Only Solver__\n", "\n", - "All pixelized source reconstructions use a positive-only solver, meaning that every source-pixel is only allowed\n", - "to reconstruct positive flux values. This ensures that the source reconstruction is physical and that we don't\n", - "reconstruct negative flux values that don't exist in the real source galaxy (a common systematic solution in lens\n", - "analysis).\n", + "All pixelized galaxy reconstructions use a positive-only solver, meaning that every pixelization pixel is only\n", + "allowed to reconstruct positive flux values. This ensures that the reconstruction is physical and that we don't\n", + "reconstruct negative flux values that don't exist in the real galaxy (a common unphysical systematic in methods\n", + "without this constraint).\n", "\n", "It may be surprising to hear that this is a feature worth pointing out, but it turns out setting up the linear algebra\n", "to enforce positive reconstructions is difficult to make efficient. A lot of development time went into making this\n", "possible, where a bespoke fast non-negative linear solver was developed to achieve this.\n", "\n", - "Other methods in the literature often do not use a positive only solver, and therefore suffer from these \n", - "unphysical solutions, which can degrade the results of lens model in general.\n", + "Other methods in the literature often do not use a positive only solver, and therefore suffer from these\n", + "unphysical solutions, which can degrade the results of galaxy models in general.\n", + "\n", + "__Detailed Explanation__\n", + "\n", + "The linear algebra sketched above -- setting up the mappings as a matrix and solving for the pixelization pixel\n", + "fluxes -- is derived in full in tutorial 5 of this chapter, where we build every matrix by hand and perform the\n", + "solve ourselves. The file `autogalaxy_workspace/*/imaging/features/pixelization/likelihood_function.ipynb` gives a\n", + "further visual step-by-step guide of the process alongside equations and references to literature on the subject.\n", "\n", "__Wrap Up__\n", "\n", "And, we're done, here are a few questions to get you thinking about inversions:\n", "\n", - " 1) The inversion provides the maximum log likelihood solution to the observed image. Is there a problem with seeking \n", - " the highest likelihood solution? Is there a risk that we're going to fit other things in the image than just the \n", + " 1) The inversion provides the maximum log likelihood solution to the observed image. Is there a problem with seeking\n", + " the highest likelihood solution? Is there a risk that we're going to fit other things in the image than just the\n", " galaxy? What happens if you reduce the `coefficient` of the regularization object above to zero?\n", "\n", - " 2) The exterior pixels in the rectangular pixel-grid have no image-pixels in them. However, they are still given a \n", - " reconstructed flux. Given these pixels do not map to the data, where is this value coming from?\n", - " \n", - "__Detailed Explanation__\n", - "\n", - "If you are interested in a more detailed description of how inversions work, then checkout the file\n", - "`autogalaxy_workspace/*/imaging/features/pixelization/likelihood_function.ipynb` which gives a visual step-by-step\n", - "guide of the process alongside equations and references to literature on the subject." + " 2) The exterior pixels in the rectangular pixel-grid have no image-pixels in them. However, they are still given a\n", + " reconstructed flux. Given these pixels do not map to the data, where is this value coming from?" ] } ], diff --git a/notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb b/notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb index cc1c58f..daf9ab3 100644 --- a/notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb +++ b/notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb @@ -20,12 +20,14 @@ "__Contents__\n", "\n", "- **Initial Setup:** Load the dataset for illustration.\n", + "- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist.\n", "- **Convenience Function:** A helper function for performing inversions.\n", "- **Pixelization:** Perform inversions with different regularization coefficients.\n", "- **Regularization:** Understand how regularization smooths the reconstruction.\n", "- **Bayesian Evidence:** Use the Bayesian evidence to objectively choose the regularization coefficient.\n", "- **Non-Linear and Linear:** Discussion of how regularization interacts with the non-linear search.\n", - "- **Detailed Description:** In-depth explanation of how the Bayesian evidence penalizes overfitting." + "- **Detailed Description:** In-depth explanation of how the Bayesian evidence penalizes overfitting.\n", + "- **Wrap Up:** Summary of Bayesian regularization concepts." ] }, { @@ -71,7 +73,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -87,11 +89,10 @@ "source": [ "__Initial Setup__\n", "\n", - "we'll use the same complex galaxy data as the previous tutorial, where:\n", + "we'll use the same galaxy data as the previous tutorial, where:\n", "\n", " - The galaxy's bulge is an `Sersic`.\n", - " - The galaxy's disk is an `Exponential`.\n", - " - The galaxy's has four star forming clumps which are `Sersic` profiles." + " - The galaxy's disk is an `Exponential`." ] }, { @@ -233,9 +234,9 @@ "So, what is happening here? Why does reducing the `coefficient` do this to our reconstruction? First, we need\n", "to understand what regularization actually does!\n", "\n", - "When the inversion reconstructs the galaxy, it does not *only* compute the set of pixelization pixel fluxes that \n", - "best-fit the image. It also regularizes this solution, whereby it goes to every pixel on the rectangular grid \n", - "and computes the different between the reconstructed flux values of every pixel with its 4 neighboring pixels. \n", + "When the inversion reconstructs the galaxy, it does not *only* compute the set of pixelization pixel fluxes that\n", + "best-fit the image. It also regularizes this solution, whereby it goes to every pixel on the rectangular grid\n", + "and computes the difference between the reconstructed flux values of every pixel with its 4 neighboring pixels.\n", "If the difference in flux is large the solution is penalized, reducing its log likelihood. You can think of this as \n", "us applying a 'smoothness prior' on the reconstructed galaxy's light.\n", "\n", @@ -277,7 +278,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The figure above shows that we completely remove over-fitting. However, we now fit the image data less poorly,\n", + "The figure above shows that we completely remove over-fitting. However, we now fit the image data more poorly,\n", "due to the much higher level of smoothing.\n", "\n", "So, we now understand what regularization is and why it is necessary. There is one nagging question that remains, how \n", @@ -321,9 +322,12 @@ " Bayesian evidence will decrease. The evidence penalizes solutions which are complex, which, in a Bayesian sense, are \n", " less probable (you may want to look up `Occam`s Razor`).\n", "\n", - "The Bayesian evidence therefore ensures we only invoke a more complex galaxy reconstruction when the data absolutely \n", + "The Bayesian evidence therefore ensures we only invoke a more complex galaxy reconstruction when the data absolutely\n", "necessitates it.\n", "\n", + "(This tutorial describes the Bayesian evidence in words; tutorial 5 writes it down as an equation, deriving each of\n", + "its terms via the linear algebra of the inversion.)\n", + "\n", "Lets take a look at the Bayesian evidence of the fits that we performed above, which is accessible from a `FitImaging` \n", "object via the `log_evidence` property:" ] @@ -353,8 +357,8 @@ "\n", "Before we end, lets consider which aspects of an inversion are linear and which are non-linear.\n", "\n", - "The linear part of the inversion is the step that solves for the reconstruct pixelization pixel fluxes, including \n", - "accounting for the smoothing via regularizaton. We do not have to perform a non-linear search to determine the pixel\n", + "The linear part of the inversion is the step that solves for the reconstructed pixelization pixel fluxes, including\n", + "accounting for the smoothing via regularization. We do not have to perform a non-linear search to determine the pixel\n", "fluxes or compute the Bayesian evidence discussed above.\n", "\n", "However, determining the regularization `coefficient` that maximizes the Bayesian log evidence is a non-linear problem \n", diff --git a/notebooks/chapter_3_pixelizations/tutorial_5_bayesian_formalism.ipynb b/notebooks/chapter_3_pixelizations/tutorial_5_bayesian_formalism.ipynb new file mode 100644 index 0000000..22e869b --- /dev/null +++ b/notebooks/chapter_3_pixelizations/tutorial_5_bayesian_formalism.ipynb @@ -0,0 +1,972 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Tutorial 5: Bayesian Formalism\n", + "==============================\n", + "\n", + "In tutorials 1 to 4, we built an intuition for how pixelized galaxy reconstruction works: pixelizations place a\n", + "pixel-grid over the galaxy's image, mappers pair pixelization pixels with image-pixels, inversions solve for the\n", + "pixel fluxes that best fit the data, and regularization smooths the solution within a Bayesian framework.\n", + "\n", + "This tutorial collects the linear algebra behind all of that in one place. The hands-on tutorials built intuition,\n", + "and now we write down the equations. We will construct, step-by-step and in code, every matrix and vector the\n", + "inversion uses, solve for the galaxy reconstruction ourselves and compute the Bayesian evidence by hand, comparing\n", + "our answer at the end to the `FitImaging` object which performs this calculation internally.\n", + "\n", + "None of this is required to *use* **PyAutoGalaxy** for galaxy modeling. However, if you publish results which use a\n", + "pixelization, this is the calculation your paper's likelihood function section will describe, and understanding it\n", + "removes any sense that the galaxy reconstruction is a \"black box\".\n", + "\n", + "The formalism follows Warren & Dye 2003 (https://arxiv.org/abs/astro-ph/0302587), hereafter WD03, with the data\n", + "vector notation of Nightingale & Dye 2015 (https://arxiv.org/abs/1412.7436), hereafter N15. These papers derived\n", + "the method for reconstructing gravitationally lensed sources, but the linear algebra is identical for a galaxy's\n", + "own light -- the only lensing-specific step (ray tracing image-pixel coordinates to a source-plane) is simply\n", + "absent, with the mesh laid directly over the galaxy's image. The Bayesian evidence was derived by Suyu et al. 2006\n", + "(https://arxiv.org/abs/astro-ph/0601493) and translated to **PyAutoGalaxy** notation in Dye et al. 2008\n", + "(https://arxiv.org/abs/0804.4002).\n", + "\n", + "__Contents__\n", + "\n", + "- **Initial Setup:** Load the dataset the previous tutorials fitted.\n", + "- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist.\n", + "- **Mask:** Mask the data so the likelihood is evaluated only where the galaxy's light is.\n", + "- **Over Sampling:** Disable over sampling so the algebra stays simple.\n", + "- **Mesh Shape:** Fix the rectangular mesh's shape and note why edge pixels are zeroed.\n", + "- **Galaxy:** Create the galaxy whose pixelization pairs the mesh with `Constant` regularization.\n", + "- **Image Grid:** The masked image-pixel coordinates the mesh is laid over -- no ray tracing occurs.\n", + "- **Mesh Pixel Centres:** Overlay the rectangular mesh over the masked image grid.\n", + "- **Interpolation:** Pair every image-pixel with mesh pixels via bilinear interpolation.\n", + "- **Mapper:** Package the interpolation into a `Mapper` describing all image-to-mesh mappings.\n", + "- **Mapping Matrix:** Express the mappings as the 2D matrix $f$.\n", + "- **Blurred Mapping Matrix:** Convolve every column of $f$ with the imaging PSF.\n", + "- **Data Vector (D):** Compute the data vector $D$ from the blurred mapping matrix, data and noise-map.\n", + "- **Curvature Matrix (F):** Compute the curvature matrix $F$.\n", + "- **Unregularized Solve:** Solve $s = F^{-1} D$ and see the over-fitted mess this produces.\n", + "- **Regularization Matrix (H):** Compute the regularization matrix $H$ encoding the smoothness prior.\n", + "- **Galaxy Reconstruction (s):** Solve the regularized system $s = [F + H]^{-1} D$.\n", + "- **Image Reconstruction:** Map the reconstruction back to image resolution via the blurred mapping matrix.\n", + "- **Likelihood Function:** The five terms which combine into the log evidence.\n", + "- **Chi Squared:** The goodness-of-fit of the reconstructed image to the data.\n", + "- **Regularization Term:** The penalty $s^{T} H s$ applied by the smoothness prior.\n", + "- **Complexity Terms:** The log determinant terms which penalize complex galaxy reconstructions.\n", + "- **Noise Normalization Term:** The Gaussian noise normalization.\n", + "- **Log Evidence:** Combine all five terms into the log evidence.\n", + "- **Fit:** Compare our by-hand log evidence to the `FitImaging` object's internal calculation.\n", + "- **Wrap Up:** Summary and next steps." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Google Colab Setup__\n", + "\n", + "This cell sets up the environment when the notebook is run on Google Colab: it installs the\n", + "required PyAuto packages, clones the workspace (configuration files and example datasets) and\n", + "points the configuration at it. If you are running the notebook elsewhere (e.g. locally via\n", + "your own installation) it does nothing, and you can run it safely.\n", + "\n", + "Colab tip: model-fits run much faster on a GPU \u2014 enable one via \"Runtime\" -> \"Change runtime\n", + "type\" -> \"Hardware accelerator\" before running the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import google.colab\n", + "except ImportError:\n", + " from autogalaxy import setup_colab as _setup_colab\n", + "else:\n", + " import importlib\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.check_call(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"autonerves\", \"--no-deps\"]\n", + " )\n", + " _setup_colab = importlib.import_module(\"autonerves.setup_colab\")\n", + "\n", + "_setup_colab.setup(\"howtogalaxy\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from pathlib import Path\n", + "\n", + "import autogalaxy as ag\n", + "import autogalaxy.plot as aplt\n", + "from autoarray.inversion.plot.mapper_plots import plot_mapper" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Initial Setup__\n", + "\n", + "we'll use the same galaxy data as the previous tutorials, where:\n", + "\n", + " - The galaxy's bulge is an `Sersic`.\n", + " - The galaxy's disk is an `Exponential`.\n", + "\n", + "For simplicity, the model in this tutorial is a pixelization only, with no light profiles: the mesh absorbs the\n", + "bulge and disk together. In WD03's notation this means the light profile model image $b_{j}$ is zero everywhere. If\n", + "light profiles are included in the galaxy model, their PSF-convolved image is computed first and subtracted from\n", + "the data before the steps below -- nothing else about the formalism changes." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset_name = \"simple\"\n", + "dataset_path = Path(\"dataset\") / \"imaging\" / dataset_name" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Dataset Auto-Simulation__\n", + "\n", + "If the dataset does not already exist on your system, it will be created by running the corresponding\n", + "simulator script. This ensures that all example scripts can be run without manually simulating data first." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "if ag.util.dataset.should_simulate(str(dataset_path)):\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.run(\n", + " [sys.executable, \"scripts/simulators/simple.py\"],\n", + " check=True,\n", + " )\n", + "\n", + "dataset = ag.Imaging.from_fits(\n", + " data_path=dataset_path / \"data.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " psf_path=dataset_path / \"psf.fits\",\n", + " pixel_scales=0.1,\n", + ")\n", + "\n", + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Mask__\n", + "\n", + "The likelihood is only evaluated within a mask, which we define as the same 2.0\" circle used in the previous\n", + "tutorials, containing all of the galaxy's light." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mask = ag.Mask2D.circular(\n", + " shape_native=dataset.shape_native,\n", + " pixel_scales=dataset.pixel_scales,\n", + " radius=2.0,\n", + ")\n", + "\n", + "masked_dataset = dataset.apply_mask(mask=mask)\n", + "\n", + "aplt.subplot_imaging_dataset(dataset=masked_dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Over Sampling__\n", + "\n", + "Over sampling splits every image pixel into a sub-grid of sub-pixels, each of which is paired with mesh pixels\n", + "individually. It improves accuracy, but it also multiplies the number of rows in every matrix below by the number\n", + "of sub-pixels.\n", + "\n", + "To keep the algebra as easy to follow as possible, we disable over sampling by setting both sub-grid sizes to 1, so\n", + "each image pixel is represented by the single coordinate at its centre." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "masked_dataset = masked_dataset.apply_over_sampling(\n", + " over_sample_size_lp=1,\n", + " over_sample_size_pixelization=1,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Mesh Shape__\n", + "\n", + "The `mesh_shape` defines the number of pixels in the rectangular mesh used to reconstruct the galaxy, set below\n", + "to 20 x 20 = 400 mesh pixels.\n", + "\n", + "We use the `RectangularUniform` mesh, where all rectangular mesh pixels have the same size, rather than the\n", + "`RectangularAdaptDensity` mesh used in the previous tutorials. The uniform mesh keeps the geometry simple, and every\n", + "equation below applies unchanged to the adaptive meshes -- only the mesh pixel centres move.\n", + "\n", + "By default, mesh pixels at the edge of the mesh are forced to solutions of zero flux by the linear algebra solver.\n", + "This prevents unphysical solutions where the mesh edge lights up to fit residuals, and does not change any of the\n", + "formalism below." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mesh_pixels_yx = 20\n", + "mesh_shape = (mesh_pixels_yx, mesh_pixels_yx)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Galaxy__\n", + "\n", + "We create the galaxy whose `Pixelization` pairs the `RectangularUniform` mesh with `Constant` regularization (whose\n", + "role appears later, when we reach the matrix $H$).\n", + "\n", + "The galaxy has no light profiles -- the mesh alone will reconstruct its bulge and disk." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "pixelization = ag.Pixelization(\n", + " mesh=ag.mesh.RectangularUniform(shape=mesh_shape),\n", + " regularization=ag.reg.Constant(coefficient=1.0),\n", + ")\n", + "\n", + "galaxy = ag.Galaxy(redshift=0.5, pixelization=pixelization)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Image Grid__\n", + "\n", + "In the lensing version of this formalism (see **PyAutoLens**), every image-pixel coordinate is first ray-traced to\n", + "a source-plane, and the mesh is laid over the traced coordinates. For a galaxy there is no lensing, so this step\n", + "simply does not exist: the mesh is laid directly over the galaxy's own image, and the grid the inversion uses is\n", + "the grid of masked image-pixel centres itself.\n", + "\n", + "This grid is `masked_dataset.grids.pixelization`, with one coordinate at the centre of every masked image pixel\n", + "(because we disabled over sampling above).\n", + "\n", + "(The `Mapper` attributes below carry `source_plane_` prefixes because **PyAutoGalaxy** and **PyAutoLens** share one\n", + "inversion implementation. For a galaxy, the \"source plane\" is just the image plane the galaxy lives in.)" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.plot_grid(grid=masked_dataset.grids.pixelization, title=\"Masked Image Grid\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Mesh Pixel Centres__\n", + "\n", + "To reconstruct the galaxy on a `RectangularUniform` mesh, we need the centres of its rectangular mesh pixels.\n", + "\n", + "We compute these by overlaying a uniform rectangular grid over the masked image grid, sized so the mesh fully\n", + "contains the masked region without wasted edge pixels beyond it." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from autoarray.inversion.mesh.mesh.rectangular_adapt_density import overlay_grid_from\n", + "\n", + "mesh_grid = overlay_grid_from(\n", + " shape_native=mesh_shape, grid=masked_dataset.grids.pixelization\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Interpolation__\n", + "\n", + "We now combine the two grids to create an `Interpolator`, which describes how every image-pixel coordinate maps to\n", + "the rectangular mesh pixels.\n", + "\n", + "For a rectangular mesh the scheme is bilinear interpolation: every image pixel maps to the rectangular pixel it\n", + "lands in *and* its three nearest neighbours, with weights set by how close the coordinate is to each pixel centre.\n", + "Interpolation is what lets the mesh reconstruct smooth galaxy morphologies. We can print the mappings and weights\n", + "of the first image pixel to confirm it pairs with four mesh pixels." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "interpolator = pixelization.mesh.interpolator_from(\n", + " source_plane_data_grid=masked_dataset.grids.pixelization,\n", + " source_plane_mesh_grid=mesh_grid,\n", + ")\n", + "\n", + "print(interpolator.mappings[0])\n", + "print(interpolator.weights[0])" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Mapper__\n", + "\n", + "The interpolator creates a `Mapper`, the object tutorials 1 and 2 introduced. It packages up the mapping between\n", + "every image pixel and every rectangular mesh pixel.\n", + "\n", + "The key attribute is `pix_indexes_for_sub_slim_index`, mapping every image-pixel index (`sub_slim_index`) to the\n", + "mesh-pixel indexes (`pix_indexes`) it interpolates onto, alongside the number of mappings per image pixel and\n", + "their interpolation weights." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mapper = ag.Mapper(interpolator=interpolator)\n", + "\n", + "pix_indexes_for_sub_slim_index = mapper.pix_indexes_for_sub_slim_index\n", + "\n", + "print(pix_indexes_for_sub_slim_index[0:9])\n", + "print(mapper.pix_sizes_for_sub_slim_index[0:9])\n", + "print(mapper.pix_weights_for_sub_slim_index[0:9])" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Mapping Matrix__\n", + "\n", + "The `mapping_matrix` expresses these image-pixel to mesh-pixel mappings as a single 2D matrix, with\n", + "dimensions `(total_image_pixels, total_mesh_pixels)`.\n", + "\n", + "Each column is the \"image\" of one mesh pixel: entries are the interpolation weights for image pixels which map to\n", + "that mesh pixel and zero everywhere else." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mapping_matrix = ag.util.mapper.mapping_matrix_from(\n", + " pix_indexes_for_sub_slim_index=pix_indexes_for_sub_slim_index,\n", + " pix_size_for_sub_slim_index=mapper.pix_sizes_for_sub_slim_index,\n", + " pix_weights_for_sub_slim_index=mapper.pix_weights_for_sub_slim_index,\n", + " pixels=mapper.pixels,\n", + " total_mask_pixels=mapper.source_plane_data_grid.mask.pixels_in_mask,\n", + " slim_index_for_sub_slim_index=mapper.slim_index_for_sub_slim_index,\n", + " sub_fraction=mapper.over_sampler.sub_fraction,\n", + ")\n", + "\n", + "plt.imshow(mapping_matrix, aspect=(mapping_matrix.shape[1] / mapping_matrix.shape[0]))\n", + "plt.show()\n", + "plt.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Because each column is an image of zeros and interpolation weights, we can plot any column as a 2D image showing\n", + "all image pixels its mesh pixel maps to. For a mesh pixel near the mesh centre, these image pixels sit at the\n", + "centre of the galaxy, where its bulge is brightest." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "central_mesh_pixel = (mesh_shape[0] // 2) * mesh_shape[1] + mesh_shape[1] // 2\n", + "\n", + "array_2d = ag.Array2D(\n", + " values=mapping_matrix[:, central_mesh_pixel], mask=masked_dataset.mask\n", + ")\n", + "\n", + "aplt.plot_array(array=array_2d, title=\"Image of Central Mesh Pixel\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Blurred Mapping Matrix__\n", + "\n", + "The imaging data is blurred by the telescope's PSF, so the model must be too. Since each column of the mapping\n", + "matrix is an image, we simply convolve each column with the PSF via 2D convolution.\n", + "\n", + "This produces the `blurred_mapping_matrix`, with the same dimensions `(total_image_pixels, total_mesh_pixels)`.\n", + "\n", + "In WD03 this matrix is denoted $f_{ij}$, where $i$ runs over all $I$ mesh pixels and $j$ over all $J$ image\n", + "pixels. For example:\n", + "\n", + " - $f_{0, 2} = 0.3$ indicates that image-pixel $2$ maps to mesh-pixel $0$ with a weight of $0.3$ after PSF\n", + " convolution.\n", + " - $f_{4, 8} = 0$ indicates that image-pixel $8$ does not map to mesh-pixel $4$, even after PSF convolution.\n", + "\n", + "(The indexing of the code's `mapping_matrix` is transposed relative to WD03's $f$: image pixels are the first index\n", + "in the code, but the second index in the equations.)\n", + "\n", + "PSF blurring has an important consequence: it correlates neighbouring columns, so the images of nearby mesh pixels\n", + "now overlap far more than the interpolation alone produced." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "blurred_mapping_matrix = masked_dataset.psf.convolved_mapping_matrix_from(\n", + " mapping_matrix=mapping_matrix, mask=masked_dataset.mask\n", + ")\n", + "\n", + "plt.imshow(\n", + " blurred_mapping_matrix,\n", + " aspect=(blurred_mapping_matrix.shape[1] / blurred_mapping_matrix.shape[0]),\n", + ")\n", + "plt.colorbar()\n", + "plt.show()\n", + "plt.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Data Vector (D)__\n", + "\n", + "We now pose the reconstruction as a linear inversion, converting the blurred mapping matrix, data and noise-map\n", + "into two objects: the data vector $D$ and the curvature matrix $F$.\n", + "\n", + "The data vector has dimensions `(total_mesh_pixels,)` and is given by (WD03 / N15):\n", + "\n", + " $\\vec{D}_{i} = \\sum_{j=1}^{J} f_{ij} (d_{j} - b_{j}) / \\sigma_{j}^2 \\, \\, .$\n", + "\n", + "Where:\n", + "\n", + " - $d_{j}$ are the image-pixel data values.\n", + " - $b_{j}$ are the model images of any light profiles in the galaxy model (zero here, because our model is a\n", + " pixelization only).\n", + " - $\\sigma_{j}^2$ are the statistical uncertainties of each image pixel.\n", + "\n", + "Each entry of $D$ is therefore the noise-weighted overlap between one mesh pixel's blurred image and the data:\n", + "it measures how much evidence the data provides for flux in that mesh pixel, with the PSF fully accounted for." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "data_vector = ag.util.inversion_imaging.data_vector_via_blurred_mapping_matrix_from(\n", + " blurred_mapping_matrix=blurred_mapping_matrix,\n", + " image=np.array(masked_dataset.data),\n", + " noise_map=np.array(masked_dataset.noise_map),\n", + ")\n", + "\n", + "plt.imshow(\n", + " data_vector.reshape(data_vector.shape[0], 1), aspect=10.0 / data_vector.shape[0]\n", + ")\n", + "plt.colorbar()\n", + "plt.show()\n", + "plt.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Curvature Matrix (F)__\n", + "\n", + "The curvature matrix has dimensions `(total_mesh_pixels, total_mesh_pixels)` and is given by (WD03):\n", + "\n", + " ${F}_{ik} = \\sum_{j=1}^{J} f_{ij} f_{kj} / \\sigma_{j}^2 \\, \\, .$\n", + "\n", + "Every entry of $F$ is the noise-weighted overlap between the blurred images of two mesh pixels: $F_{ik}$ sums the\n", + "product of columns $i$ and $k$ of $f$ over all image pixels. For $F_{ik}$ to be non-zero, mesh pixels $i$ and $k$\n", + "must share at least one image pixel, which happens for neighbouring pixels via interpolation and for pixels\n", + "further apart via PSF blurring.\n", + "\n", + "$F$ describes how degenerate pairs of mesh pixels are with one another: two mesh pixels whose blurred images\n", + "overlap heavily can trade flux between themselves whilst fitting the data almost equally well." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "curvature_matrix = ag.util.inversion.curvature_matrix_via_mapping_matrix_from(\n", + " mapping_matrix=blurred_mapping_matrix, noise_map=masked_dataset.noise_map\n", + ")\n", + "\n", + "plt.imshow(curvature_matrix)\n", + "plt.colorbar()\n", + "plt.show()\n", + "plt.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Unregularized Solve__\n", + "\n", + "The inversion seeks the mesh-pixel fluxes $s$ (a vector with one entry per mesh pixel) that minimize the\n", + "chi-squared:\n", + "\n", + " $\\chi^2 = \\sum_{j=1}^{J} \\bigg[ \\frac{(\\sum_{i=1}^{I} s_{i} f_{ij}) + b_{j} - d_{j}}{\\sigma_{j}} \\bigg]^2$\n", + "\n", + "Setting the derivative of $\\chi^2$ with respect to each $s_{i}$ to zero gives the linear system whose solution is\n", + "(equation 5 of WD03):\n", + "\n", + " $s = F^{-1} D$\n", + "\n", + "We can solve this directly with NumPy. (Without regularization the curvature matrix is often singular, so the loop\n", + "below adds a tiny value to its diagonal to avoid a `LinAlgError` -- it is a numerical crutch, not part of the\n", + "formalism.)" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "for i in range(curvature_matrix.shape[0]):\n", + " curvature_matrix[i, i] += 1e-8\n", + "\n", + "reconstruction = np.linalg.solve(curvature_matrix, data_vector)\n", + "\n", + "plot_mapper(mapper=mapper, solution_vector=reconstruction)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The reconstructed mesh-pixel fluxes are a noisy, unsmooth mess -- exactly the over-fitting we saw in tutorial 4\n", + "when we lowered the regularization coefficient towards zero. The linear inversion is fitting the noise in the\n", + "data, because this system of equations is ill-posed: we need a smoothness prior.\n", + "\n", + "__Regularization Matrix (H)__\n", + "\n", + "Regularization adds a linear regularization term $G_{L}$ to the merit function we minimize (equation 11 of WD03):\n", + "\n", + " $G = \\chi^2 + \\lambda \\, G_{L}$\n", + "\n", + "where $\\lambda$ is the `regularization_coefficient` controlling the degree of smoothing. The `Constant` scheme uses\n", + "gradient regularization (equation 14 of WD03):\n", + "\n", + " $G_{L} = \\sum_{i}^{I} \\sum_{n=1}^{N} [s_{i} - s_{i, n}]^2$\n", + "\n", + "In words: for every mesh pixel, compare its flux with each of its $N$ neighbours $n$, and penalize solutions where\n", + "the differences are large. This is precisely the \"smoothness prior\" of tutorial 4, now written as an equation.\n", + "\n", + "To fold this into the linear algebra we define the regularization matrix $H$, with\n", + "dimensions `(total_mesh_pixels, total_mesh_pixels)` (equation 13 of WD03):\n", + "\n", + " $H_{ik} = \\frac{1}{2} \\frac{\\partial^{2} G_{L}}{\\partial s_{i} \\partial s_{k}}$\n", + "\n", + "$H$ has the coefficient $\\lambda$ folded into it. Its non-zero off-diagonal entries mark pairs of mesh pixels\n", + "which are neighbours and therefore regularized with one another; most entries are zero because most mesh pixels\n", + "are not neighbours." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "regularization_matrix = ag.util.regularization.constant_regularization_matrix_from(\n", + " coefficient=galaxy.pixelization.regularization.coefficient,\n", + " neighbors=mapper.neighbors,\n", + " neighbors_sizes=mapper.neighbors.sizes,\n", + ")\n", + "\n", + "plt.imshow(regularization_matrix)\n", + "plt.colorbar()\n", + "plt.show()\n", + "plt.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Galaxy Reconstruction (s)__\n", + "\n", + "$H$ enters the linear system as follows (equation 12 of WD03):\n", + "\n", + " $s = [F + H]^{-1} D$\n", + "\n", + "We add the two matrices and solve again. The diagonal jitter used above is no longer needed, because $H$ makes the\n", + "system well-posed." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "curvature_reg_matrix = np.add(curvature_matrix, regularization_matrix)\n", + "\n", + "reconstruction = np.linalg.solve(curvature_reg_matrix, data_vector)\n", + "\n", + "plot_mapper(mapper=mapper, solution_vector=reconstruction)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The reconstructed fluxes are now smooth and physical: regularization has suppressed the noisy solution and the\n", + "reconstruction actually looks like the galaxy's bulge and disk, without over-fitting the noise.\n", + "\n", + "__Image Reconstruction__\n", + "\n", + "Using the reconstructed mesh-pixel fluxes, we map the reconstruction back to image resolution via the blurred\n", + "mapping matrix (so the reconstructed image includes PSF blurring) to produce the model image of the galaxy." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mapped_reconstructed_data = (\n", + " ag.util.inversion.mapped_reconstructed_data_via_mapping_matrix_from(\n", + " mapping_matrix=blurred_mapping_matrix, reconstruction=reconstruction\n", + " )\n", + ")\n", + "\n", + "mapped_reconstructed_data = ag.Array2D(\n", + " values=mapped_reconstructed_data, mask=masked_dataset.mask\n", + ")\n", + "\n", + "aplt.plot_array(array=mapped_reconstructed_data, title=\"Reconstructed Image\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Likelihood Function__\n", + "\n", + "We now quantify the goodness-of-fit of the galaxy reconstruction, computing the quantity tutorial 4 called the\n", + "Bayesian evidence. The log evidence consists of five terms:\n", + "\n", + " $-2 \\, \\mathrm{ln} \\, \\epsilon = \\chi^2 + s^{T} H s + \\mathrm{ln} \\, [ \\mathrm{det} (F + H) ] - \\mathrm{ln} \\, [ \\mathrm{det} (H) ] + \\sum_{j=1}^{J} \\mathrm{ln} \\, [2 \\pi (\\sigma_{j})^2 ] \\, .$\n", + "\n", + "This expression was first derived by Suyu et al. 2006 (https://arxiv.org/abs/astro-ph/0601493), equation (19), and\n", + "is given in **PyAutoGalaxy** notation by Dye et al. 2008 (https://arxiv.org/abs/0804.4002), equation (5).\n", + "\n", + "We now compute each term in turn.\n", + "\n", + "__Chi Squared__\n", + "\n", + "The first term is the $\\chi^2$ statistic from the merit function above, computed as:\n", + "\n", + " - `model_data` = the reconstructed image of the galaxy (plus any light profile model images, zero here).\n", + " - `residual_map` = (`data` - `model_data`)\n", + " - `normalized_residual_map` = (`data` - `model_data`) / `noise_map`\n", + " - `chi_squared_map` = (`normalized_residual_map`) ** 2.0\n", + " - `chi_squared` = sum(`chi_squared_map`)\n", + "\n", + "High chi-squared values indicate image pixels the reconstruction fits poorly, lowering the likelihood." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "model_image = mapped_reconstructed_data\n", + "\n", + "residual_map = masked_dataset.data - model_image\n", + "normalized_residual_map = residual_map / masked_dataset.noise_map\n", + "chi_squared_map = normalized_residual_map**2.0\n", + "\n", + "chi_squared = np.sum(chi_squared_map)\n", + "\n", + "print(chi_squared)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Regularization Term__\n", + "\n", + "The second term, $s^{T} H s$, is the $\\lambda \\, G_{L}$ regularization penalty evaluated at the solution: the\n", + "summed difference in flux between all neighbouring mesh pixels, weighted by the regularization coefficient (which\n", + "is already folded into $H$).\n", + "\n", + "Less smooth solutions have larger values of this term and therefore lower likelihoods." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "regularization_term = np.matmul(\n", + " reconstruction.T, np.matmul(regularization_matrix, reconstruction)\n", + ")\n", + "\n", + "print(regularization_term)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Complexity Terms__\n", + "\n", + "Up to this point, nothing has justified our choice of `regularization_coefficient=1.0`. We cannot choose it using\n", + "the two terms above, because increasing the coefficient smooths the solution more, which *both* worsens the\n", + "chi-squared *and* (for a fixed solution) raises the regularization penalty. Optimizing those two terms alone would\n", + "drive the coefficient to zero and put us right back at the over-fitted mess.\n", + "\n", + "The two log determinant terms, $\\mathrm{ln} \\, [ \\mathrm{det} (F + H) ]$ and $- \\mathrm{ln} \\, [ \\mathrm{det} (H) ]$,\n", + "fix this. Together they measure how *complex* the galaxy reconstruction is -- roughly, how many effective degrees\n", + "of freedom the mesh uses after regularization correlates its pixels -- and penalize more complex solutions.\n", + "Lowering the regularization coefficient frees the mesh to use more of its flexibility, increasing this complexity\n", + "penalty.\n", + "\n", + "These terms therefore counteract the chi-squared and regularization terms, so the highest evidence goes to\n", + "solutions which fit the data well with the *simplest* galaxy reconstruction. This is the Occam's razor behaviour\n", + "that tutorial 4 demonstrated empirically." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "log_curvature_reg_matrix_term = np.linalg.slogdet(curvature_reg_matrix)[1]\n", + "log_regularization_matrix_term = np.linalg.slogdet(regularization_matrix)[1]\n", + "\n", + "print(log_curvature_reg_matrix_term)\n", + "print(log_regularization_matrix_term)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Noise Normalization Term__\n", + "\n", + "The likelihood function assumes the imaging data consists of independent Gaussian noise in every image pixel, and\n", + "the final term is the normalization of those Gaussians: the sum of the log of every noise-map value squared.\n", + "\n", + "Because the noise-map is fixed, this term is constant throughout galaxy modeling and has no impact on the model\n", + "we infer -- it simply normalizes the likelihood." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "noise_normalization = float(np.sum(np.log(2 * np.pi * masked_dataset.noise_map**2.0)))\n", + "\n", + "print(noise_normalization)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Log Evidence__\n", + "\n", + "We can now combine the five terms into the log evidence of the galaxy reconstruction." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "log_evidence = float(\n", + " -0.5\n", + " * (\n", + " chi_squared\n", + " + regularization_term\n", + " + log_curvature_reg_matrix_term\n", + " - log_regularization_matrix_term\n", + " + noise_normalization\n", + " )\n", + ")\n", + "\n", + "print(log_evidence)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Fit__\n", + "\n", + "Everything above is what the `FitImaging` object does internally when it fits a galaxy with a pixelization. We can\n", + "see this by performing the fit and comparing its `log_evidence` to ours.\n", + "\n", + "The two values do not agree exactly, because the real fit improves on our simplified solve in two ways mentioned\n", + "along the way: it uses the positive-only solver (tutorial 3), which forbids the negative mesh-pixel fluxes our\n", + "unconstrained `np.linalg.solve` permits, and it zeroes the pixels at the edge of the mesh. Our unconstrained solve\n", + "exploits that extra (unphysical) freedom to push its chi-squared lower than the real solver allows, which is why\n", + "our by-hand log evidence comes out somewhat higher. Neither constraint changes the formalism -- the same $f$, $D$,\n", + "$F$ and $H$ feed a solver with extra conditions on $s$." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxies = ag.Galaxies(galaxies=[galaxy])\n", + "\n", + "fit = ag.FitImaging(dataset=masked_dataset, galaxies=galaxies)\n", + "\n", + "print(fit.log_evidence)\n", + "\n", + "aplt.subplot_fit_imaging(fit=fit)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Wrap Up__\n", + "\n", + "We have walked through the complete linear algebra of a pixelized galaxy reconstruction:\n", + "\n", + " - The `mapping_matrix` and PSF-blurred mapping matrix $f$, whose columns are the blurred images of each\n", + " mesh pixel.\n", + "\n", + " - The data vector $D$ and curvature matrix $F$, the noise-weighted overlaps of those images with the data and with\n", + " each other.\n", + "\n", + " - The regularization matrix $H$, which encodes the smoothness prior, and the linear solve $s = [F + H]^{-1} D$ for\n", + " the galaxy reconstruction.\n", + "\n", + " - The five terms of the Suyu et al. 2006 log evidence -- chi-squared, the regularization penalty, the two log\n", + " determinant complexity terms and the noise normalization -- and their Bayesian interpretation as an Occam's\n", + " razor which favours the simplest galaxy reconstruction the data allows.\n", + "\n", + "During galaxy modeling, this whole calculation is one likelihood evaluation: the non-linear search varies the\n", + "model's parameters (for example the mesh resolution, the regularization coefficient, or the light profiles fitted\n", + "alongside the mesh), and each sample triggers the full solve and evidence computation above.\n", + "\n", + "Two simplifications are worth remembering: real fits use over sampling (each image pixel contributes several\n", + "sub-pixel rows to $f$) and **PyAutoGalaxy** uses a positive-only solver for $s$ rather than the unconstrained\n", + "`np.linalg.solve` used here (see tutorial 3). The workspace\n", + "guide `autogalaxy_workspace/*/imaging/features/pixelization/likelihood_function.ipynb` repeats this walk-through\n", + "with additional visualization of every step, including how light profiles combine with the mesh.\n", + "\n", + "In the next tutorial, we return to hands-on territory and use pixelizations in an actual model-fit, combining light\n", + "profiles and an inversion via search chaining." + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb b/notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb index c930e2b..5d9dfe6 100644 --- a/notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb +++ b/notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb @@ -4,29 +4,30 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Tutorial 5: Model-Fit\n", + "Tutorial 6: Model Fit\n", "=====================\n", "\n", "In the previous tutorials we used an inversion to reconstruct a complex galaxy. However, from the perspective of\n", - "a scientific analysis, it is not clear how useful this was. When I fit a galaxy with light profiles, I learn about\n", + "a scientific analysis, it is not clear how useful this was. When we fit a galaxy with light profiles, we learn about\n", "its brightness (`intensity`), size (`effective_radius`), compactness (`sersic_index`), etc.\n", "\n", - "What did I learn about the galaxy I reconstructed? Not a lot, perhaps.\n", + "What did we learn about the galaxy we reconstructed? Not a lot, perhaps.\n", "\n", - "Inversions are most useful when combined with light profiles. For the complex galaxy we used throughout this tutorial,\n", - "we can fit it with light profiles to quantify the properties of its `bulge` and `disk` components, whilst\n", - "simultaneously fitting the clumps with the inversion so as to ensure they do not impact the fit.\n", + "Inversions are most useful when combined with light profiles. For the complex galaxy we have used throughout this\n", + "chapter, we can fit it with light profiles to quantify the properties of its `bulge` and `disk` components, whilst\n", + "simultaneously fitting any residual structure with the inversion so as to ensure it does not impact the fit.\n", "\n", - "To illustrate modeling using an inversion this tutorial therefore revisits the complex galaxy model-fit that we\n", - "performed in tutorial 4 of chapter 3. This time, as you have probably guessed, we will fit part of the complex galaxy\n", - "using an inversion.\n", + "To illustrate modeling using an inversion, this tutorial therefore revisits the complex galaxy model-fit that we\n", + "performed via search chaining at the end of chapter 2 (tutorials 9-10). This time, as you have probably guessed, we\n", + "will fit part of the galaxy using an inversion.\n", "\n", "We will use search chaining to do this, first fitting the main galaxy components with light profiles, thereby\n", "initializing the bulge and disk components. In the later searches we will switch to an `Inversion`.\n", "\n", "__Contents__\n", "\n", - "- **Initial Setup:** Load the complex galaxy dataset and apply a mask.\n", + "- **Initial Setup:** Load the galaxy dataset and apply a mask.\n", + "- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist.\n", "- **Model + Search + Analysis + Model-Fit (Search 1):** Fit light profiles to the main galaxy components.\n", "- **Mesh Shape:** Discussion of how mesh shape affects the inversion.\n", "- **Model + Search + Analysis + Model-Fit (Search 2):** Fit with a pixelization for residual structure.\n", @@ -77,7 +78,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autofit as af\n", @@ -93,11 +94,10 @@ "source": [ "__Initial Setup__\n", "\n", - "we'll use complex galaxy data, where:\n", + "we'll use the same galaxy data as the previous tutorials, where:\n", "\n", " - The galaxy's bulge is an `Sersic`.\n", - " - The galaxy's disk is an `Exponential`.\n", - " - The galaxy's has four star forming clumps which are `Sersic` profiles." + " - The galaxy's disk is an `Exponential`." ] }, { @@ -193,7 +193,7 @@ "model_1 = af.Collection(galaxies=af.Collection(galaxy=galaxy))\n", "\n", "search_1 = af.Nautilus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_4\"),\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_3\"),\n", " name=\"search[1]\",\n", " unique_tag=dataset_name,\n", " n_live=100,\n", @@ -213,11 +213,11 @@ "source": [ "__Mesh Shape__\n", "\n", - "The `mesh_shape` parameter defines number of pixels used by the rectangular mesh to reconstruct the source,\n", - "set below to 28 x 28. \n", + "The `mesh_shape` parameter defines the number of pixels used by the rectangular mesh to reconstruct the galaxy,\n", + "set below to 28 x 28.\n", "\n", "The `mesh_shape` must be fixed before modeling and cannot be a free parameter of the model, because JAX uses the\n", - "mesh shape to define static shaped arrays which use the mesh to reconstruct the source. For a rectangular\n", + "mesh shape to define static shaped arrays which use the mesh to reconstruct the galaxy. For a rectangular\n", "mesh, the same number of pixels must be used in the y and x directions." ] }, @@ -240,17 +240,18 @@ "We use the results of search 1 to create the model fitted in search 2, where:\n", "\n", " - The galaxy's bulge is an `Sersic` [0 parameters: parameters fixed from search 1].\n", - " \n", + "\n", " - The galaxy's disk is an `Exponential` [0 parameters: parameters fixed from search 1].\n", "\n", - " - The galaxy's clumps are reconstructed `RectangularAdaptDensity` mesh with resolution as free parameters [2 parameters].\n", + " - Residual galaxy structure is reconstructed using a `RectangularAdaptDensity` mesh with a fixed 28 x 28\n", + " shape [0 parameters].\n", "\n", - " - This pixelization is regularized using a `Constant` scheme [1 parameter]. \n", + " - This pixelization is regularized using a `GaussianKernel` scheme [2 parameters].\n", "\n", - "The number of free parameters and therefore the dimensionality of non-linear parameter space is N=3.\n", + "The number of free parameters and therefore the dimensionality of non-linear parameter space is N=2.\n", "\n", - "This search allows us to very efficiently set up the resolution of the mesh and regularization coefficient \n", - "of the regularization scheme, before using these models to refit the galaxy mass model." + "This search allows us to very efficiently set up the regularization scheme's parameters, before refitting the\n", + "galaxy's bulge and disk alongside the inversion in search 3." ] }, { @@ -274,7 +275,7 @@ "model_2 = af.Collection(galaxies=af.Collection(galaxy=galaxy))\n", "\n", "search_2 = af.Nautilus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_4\"),\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_3\"),\n", " name=\"search[2]\",\n", " unique_tag=dataset_name,\n", " n_live=50,\n", @@ -301,16 +302,17 @@ "We use the results of searches 1 and 2 to create the model fitted in search 3, where:\n", "\n", " - The galaxy's bulge is an `Sersic` [7 parameters: priors initialized from search 1].\n", - " \n", - " - The galaxy's disk is an `Exponential` [6 parameters: priors initialized from search 1].\n", "\n", - " - The galaxy's light uses a `RectangularAdaptDensity` mesh[parameters fixed to results of search 2].\n", + " - The galaxy's disk is an `Sersic` [7 parameters: priors initialized from search 1].\n", "\n", - " - This pixelization is regularized using a `Constant` scheme [parameters fixed to results of search 2]. \n", + " - Residual galaxy structure uses a `RectangularAdaptDensity` mesh with a fixed 28 x 28 shape [0 parameters].\n", "\n", - "The number of free parameters and therefore the dimensionality of non-linear parameter space is N=13.\n", + " - This pixelization is regularized using a `GaussianKernel` scheme [2 parameters].\n", "\n", - "This search allows us to refit the bulge and disk components with an inversion that takes care of the clumps." + "The number of free parameters and therefore the dimensionality of non-linear parameter space is N=16.\n", + "\n", + "This search allows us to refit the bulge and disk components with an inversion that takes care of any residual\n", + "structure the light profiles do not capture." ] }, { @@ -341,7 +343,7 @@ "model_3 = af.Collection(galaxies=af.Collection(galaxy=galaxy))\n", "\n", "search_3 = af.Nautilus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_4\"),\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_3\"),\n", " name=\"search[3]\",\n", " unique_tag=dataset_name,\n", " n_live=100,\n", @@ -361,7 +363,15 @@ "source": [ "__Wrap Up__\n", "\n", - "And with that, we now have a pipeline to model galaxies using an inversion! " + "And with that, we now have a pipeline to model galaxies using an inversion!\n", + "\n", + "This concludes the pixelizations chapter. You now have every tool this chapter set out to teach: pixelizations and\n", + "mappers (tutorials 1-2), inversions (tutorial 3), Bayesian regularization and the evidence (tutorials 4-5), and a\n", + "search-chained pipeline which combines light profiles with an inversion (this tutorial).\n", + "\n", + "In chapter 4, we scale up from a single galaxy to the busier fields real imaging surveys deliver: datasets with\n", + "extra galaxies near the main galaxy, blended multi-galaxy systems modeled simultaneously, and cluster fields with\n", + "many member galaxies." ] } ], diff --git a/notebooks/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.ipynb b/notebooks/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.ipynb new file mode 100644 index 0000000..21b7450 --- /dev/null +++ b/notebooks/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.ipynb @@ -0,0 +1,702 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Tutorial 1: Extra Galaxies\n", + "==========================\n", + "\n", + "Welcome to chapter 4 of **HowToGalaxy**, where we scale up galaxy modeling beyond a single galaxy.\n", + "\n", + "In chapters 1 to 3, every dataset we studied had the same simple anatomy: one galaxy, alone at the centre of the\n", + "image, whose light we modeled with parametric profiles, basis functions or pixelizations. This is the cleanest\n", + "possible configuration, and it was the right place to learn the fundamentals of light profiles, non-linear searches\n", + "and Bayesian inference.\n", + "\n", + "Real galaxies are rarely this tidy. Galaxies live in crowded fields: a galaxy may have a faint companion a few\n", + "arc-seconds away, be blended with a close neighbour of comparable brightness, or sit in a cluster containing\n", + "hundreds of members. The science of galaxy modeling scales up through this hierarchy, and so must our modeling:\n", + "\n", + "- **Extra galaxies (this tutorial)**: a single galaxy of interest, with one or more nearby interloper galaxies\n", + " whose light may contaminate the analysis.\n", + "\n", + "- **Multi-galaxy blends (next tutorial)**: systems where two or more galaxies are blended together and all are\n", + " subjects of the analysis, so no single galaxy can be called \"the\" galaxy.\n", + "\n", + "- **Cluster fields (final tutorial)**: images containing many galaxies, for example a brightest cluster galaxy\n", + " surrounded by a population of cluster members, which must be modeled together.\n", + "\n", + "The same ladder exists in strong gravitational lensing, where interlopers, multi-galaxy deflectors and cluster-scale\n", + "lenses complicate the analysis in analogous ways. The **HowToLens** lectures scale up lens modeling through an\n", + "equivalent chapter 4, using the same **PyAutoFit** API you will learn here.\n", + "\n", + "In this first tutorial, we take the first step up in scale: a galaxy with one extra galaxy nearby. We will learn\n", + "how to decide whether the extra galaxy matters, and the two approaches to dealing with it when it does: removing\n", + "its light from the data, or including it in the model.\n", + "\n", + "__Contents__\n", + "\n", + "- **Initial Setup:** Load the imaging dataset of a galaxy with an extra galaxy and inspect the interloper.\n", + "- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist.\n", + "- **The Decision:** The core question: does the extra galaxy's light overlap the region of the image we fit?\n", + "- **Mask:** Define a circular mask large enough to include the extra galaxy's emission.\n", + "- **Approach 1 Noise Scaling:** Remove the extra galaxy's light from the fit by scaling its data and noise values.\n", + "- **Noise Scaling Fit:** Fit a model to the noise-scaled data, without the extra galaxy in the model.\n", + "- **Approach 2 Extra Galaxies Model:** Include the extra galaxy's light in the model explicitly.\n", + "- **Extra Galaxy Centres:** Why the extra galaxy's centre is fixed to its observed light centre.\n", + "- **Extra Galaxies Model Composition:** Compose the model including the extra galaxy via the extra galaxies API.\n", + "- **Extra Galaxies Fit:** Fit the model which includes the extra galaxy.\n", + "- **Which Approach When:** Guidance on choosing between noise scaling and explicit modeling.\n", + "- **Wrap Up:** Summary of the script and next steps." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Google Colab Setup__\n", + "\n", + "This cell sets up the environment when the notebook is run on Google Colab: it installs the\n", + "required PyAuto packages, clones the workspace (configuration files and example datasets) and\n", + "points the configuration at it. If you are running the notebook elsewhere (e.g. locally via\n", + "your own installation) it does nothing, and you can run it safely.\n", + "\n", + "Colab tip: model-fits run much faster on a GPU \u2014 enable one via \"Runtime\" -> \"Change runtime\n", + "type\" -> \"Hardware accelerator\" before running the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import google.colab\n", + "except ImportError:\n", + " from autogalaxy import setup_colab as _setup_colab\n", + "else:\n", + " import importlib\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.check_call(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"autonerves\", \"--no-deps\"]\n", + " )\n", + " _setup_colab = importlib.import_module(\"autonerves.setup_colab\")\n", + "\n", + "_setup_colab.setup(\"howtogalaxy\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "\n", + "from autogalaxy import jax_wrapper # Sets JAX environment before other imports\n", + "\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", + "\n", + "from pathlib import Path\n", + "import autogalaxy as ag\n", + "import autogalaxy.plot as aplt\n", + "import autofit as af" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Initial Setup__\n", + "\n", + "Lets load the `Imaging` dataset we'll fit in this tutorial. It is similar to the `simple__sersic` dataset used\n", + "throughout chapter 2, where:\n", + "\n", + " - The galaxy's bulge is an `Sersic`.\n", + "\n", + "However, there is one addition: an extra galaxy, with its own light (an `ExponentialSph`), located a few\n", + "arc-seconds from the main galaxy." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset_name = \"extra_galaxy\"\n", + "dataset_path = Path(\"dataset\") / \"imaging\" / dataset_name" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Dataset Auto-Simulation__\n", + "\n", + "If the dataset does not already exist on your system, it will be created by running the corresponding\n", + "simulator script. This ensures that all example scripts can be run without manually simulating data first." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "if ag.util.dataset.should_simulate(str(dataset_path)):\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.run(\n", + " [sys.executable, \"scripts/simulators/extra_galaxy.py\"],\n", + " check=True,\n", + " )\n", + "\n", + "dataset = ag.Imaging.from_fits(\n", + " data_path=dataset_path / \"data.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " psf_path=dataset_path / \"psf.fits\",\n", + " pixel_scales=0.1,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When we plot the dataset, the familiar sight of a single galaxy at the centre of the image is joined by a blob of\n", + "light in the upper-right of the image.\n", + "\n", + "This is the extra galaxy. It is not the galaxy we are studying: it is an interloper, a galaxy that happens to lie\n", + "close to ours on the sky. Interlopers like this are extremely common in real imaging of galaxies, and everyone who\n", + "models galaxies has to decide what to do about them." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__The Decision__\n", + "\n", + "The decision of what to do about an extra galaxy hinges on one question:\n", + "\n", + "**Does its light overlap the region of the image we fit?**\n", + "\n", + "The extra galaxy's emission may extend into the region of the image we fit. If it does, and we fit a model\n", + "containing only the main galaxy, the model has no component that can produce this emission. The fit will respond by\n", + "distorting the main galaxy's light profiles to soak it up \u2014 for example inflating the `effective_radius` or skewing\n", + "the `ell_comps` towards the interloper \u2014 biasing every parameter we infer about the galaxy's structure.\n", + "\n", + "If the extra galaxy's light does not reach the fitted region (it is faint, or far away, or both), we can simply\n", + "ignore it. When it does, we have two approaches:\n", + "\n", + "- **Approach 1 (remove its light)**: We can remove the extra galaxy's emission from the fit entirely, without\n", + " adding anything to the model.\n", + "\n", + "- **Approach 2 (model it explicitly)**: If its light blends too closely with the main galaxy's emission to cleanly\n", + " remove, we include the extra galaxy in the model, with its own light profile.\n", + "\n", + "We will now perform both, and at the end of the tutorial discuss when each is appropriate.\n", + "\n", + "__Mask__\n", + "\n", + "We first define the circular mask used to fit the data. In chapter 2 we typically used a 2.5\" - 3.0\" mask, which\n", + "tightly contained the galaxy.\n", + "\n", + "Here, we use a larger 6.0\" mask, so that the region containing the extra galaxy is included in the fit. If we\n", + "simply shrank the mask to exclude the extra galaxy, we would also throw away pixels containing the main galaxy's\n", + "outer emission, and the mask's hard edge could still cut through the extra galaxy's light." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mask = ag.Mask2D.circular(\n", + " shape_native=dataset.shape_native,\n", + " pixel_scales=dataset.pixel_scales,\n", + " radius=6.0,\n", + ")\n", + "\n", + "dataset = dataset.apply_mask(mask=mask)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plotting the masked dataset confirms the extra galaxy's emission is inside the mask, and will therefore impact the\n", + "model-fit unless we do something about it." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Approach 1 Noise Scaling__\n", + "\n", + "Our first approach removes the extra galaxy's light from the fit.\n", + "\n", + "The most obvious way to do this would be to mask the extra galaxy's pixels, removing them from the fit entirely.\n", + "However, removing pixels changes the fit in subtle ways: their coordinates are no longer used when evaluating the\n", + "galaxy's light, and for certain models (e.g. the pixelized reconstructions of chapter 3) removing interior pixels\n", + "creates discontinuities in the pixelization that introduce unexpected systematics.\n", + "\n", + "Instead, we use **noise scaling**: the pixels stay in the fit, but their data values are set to zero and their\n", + "noise-map values are increased to very large values. A pixel with enormous noise contributes negligibly to the\n", + "likelihood, so the extra galaxy's light cannot influence the model, while the pixels themselves remain part of the\n", + "fit's geometry.\n", + "\n", + "To do this we need a mask of the extra galaxy's region. For real data, you would create this yourself by\n", + "inspecting the image (the `autogalaxy_workspace`'s `data_preparation` package includes a GUI for drawing it); for\n", + "this simulated dataset the simulator script has already output a `mask_extra_galaxies.fits` circle covering the\n", + "extra galaxy.\n", + "\n", + "We reload the dataset first, because noise scaling must be applied before the circular mask." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset = ag.Imaging.from_fits(\n", + " data_path=dataset_path / \"data.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " psf_path=dataset_path / \"psf.fits\",\n", + " pixel_scales=0.1,\n", + ")\n", + "\n", + "mask_extra_galaxies = ag.Mask2D.from_fits(\n", + " file_path=dataset_path / \"mask_extra_galaxies.fits\",\n", + " pixel_scales=0.1,\n", + " invert=True, # Note that we invert the mask here as `True` means a pixel is scaled.\n", + ")\n", + "\n", + "dataset = dataset.apply_noise_scaling(mask=mask_extra_galaxies)\n", + "\n", + "mask = ag.Mask2D.circular(\n", + " shape_native=dataset.shape_native,\n", + " pixel_scales=dataset.pixel_scales,\n", + " radius=6.0,\n", + ")\n", + "\n", + "dataset = dataset.apply_mask(mask=mask)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plotting the dataset shows the extra galaxy's emission has vanished: its data values are zero and the\n", + "signal-to-noise of its pixels is effectively zero, so the fit will simply ignore that region of the image." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Noise Scaling Fit__\n", + "\n", + "We now fit this noise-scaled dataset with a model that does **not** include the extra galaxy. The model is a\n", + "single linear `Sersic` bulge, using the linear light profiles introduced in chapter 2's linear profiles\n", + "tutorial." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "bulge = af.Model(ag.lp_linear.Sersic)\n", + "\n", + "galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge)\n", + "\n", + "model = af.Collection(galaxies=af.Collection(galaxy=galaxy))\n", + "\n", + "search = af.Nautilus(\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_4\"),\n", + " name=\"tutorial_1_extra_galaxies_noise_scaling\",\n", + " unique_tag=dataset_name,\n", + " n_live=100,\n", + " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", + ")\n", + "\n", + "analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", + "\n", + "print(\n", + " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_4/tutorial_1_extra_galaxies_noise_scaling\"\n", + " \" folder for live output of the results, images and model.\"\n", + " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + ")\n", + "\n", + "result_noise_scaling = search.fit(model=model, analysis=analysis)\n", + "\n", + "print(\"The search has finished run - you may now continue the notebook.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plotting the maximum log likelihood fit shows the galaxy is fitted well, with the noise-scaled region contributing\n", + "nothing to the residuals.\n", + "\n", + "The extra galaxy's light has been dealt with, and no complexity was added to the model: the fit had exactly the\n", + "same free parameters as a fit to an isolated galaxy. This is the great appeal of noise scaling." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_fit_imaging(fit=result_noise_scaling.max_log_likelihood_fit)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Approach 2 Extra Galaxies Model__\n", + "\n", + "Our second approach includes the extra galaxy in the model, fitting its light explicitly so we no longer need to\n", + "remove it from the data. Its emission is subtracted by the model itself, including any faint light that spills\n", + "towards the main galaxy which a noise-scaling mask cannot cleanly separate.\n", + "\n", + "We reload the dataset and apply the 6.0\" circular mask, but this time we do **not** apply noise scaling, because\n", + "the extra galaxy's emission is now something the model itself will fit." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset = ag.Imaging.from_fits(\n", + " data_path=dataset_path / \"data.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " psf_path=dataset_path / \"psf.fits\",\n", + " pixel_scales=0.1,\n", + ")\n", + "\n", + "mask = ag.Mask2D.circular(\n", + " shape_native=dataset.shape_native,\n", + " pixel_scales=dataset.pixel_scales,\n", + " radius=6.0,\n", + ")\n", + "\n", + "dataset = dataset.apply_mask(mask=mask)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Extra Galaxy Centres__\n", + "\n", + "To include the extra galaxy in the model, we input the centre of its light, as observed in the image.\n", + "\n", + "In principle, we could add the extra galaxy to the model with a completely free centre, and let the non-linear\n", + "search figure out where it is. In practice this is a bad idea: the extra parameters make parameter space more\n", + "complex, and fits commonly go wrong in characteristic ways. For example, the extra galaxy's light profile may\n", + "wander off and try to fit part of the main galaxy's asymmetric emission instead of the interloper, leaving the\n", + "interloper unfitted and the main galaxy's light model biased.\n", + "\n", + "Fixing each extra galaxy's light centre to its observed light centre removes these failure modes and keeps the\n", + "model as simple as possible. The observed centre is an excellent estimate of the true centre, because a galaxy's\n", + "brightest pixel closely traces the peak of its light distribution.\n", + "\n", + "For real data you would measure these centres from the image (the `autogalaxy_workspace`'s `data_preparation`\n", + "package shows how, including a GUI for marking them); for this simulated dataset the simulator has output them to\n", + "a .json file, which we load below." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "extra_galaxies_centres = ag.Grid2DIrregular(\n", + " ag.from_json(file_path=dataset_path / \"extra_galaxies_centres.json\")\n", + ")\n", + "\n", + "print(extra_galaxies_centres)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Extra Galaxies Model Composition__\n", + "\n", + "We compose the main galaxy model exactly as before." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "bulge = af.Model(ag.lp_linear.Sersic)\n", + "\n", + "galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We now compose the model of the extra galaxy, using the dedicated `extra_galaxies` modeling API.\n", + "\n", + "For each extra galaxy centre (there is only one here, but the loop below scales to any number) we create a `Galaxy`\n", + "model with:\n", + "\n", + " - A linear `SersicSph` light profile, with its `centre` fixed to the observed centre [2 free\n", + " parameters: `effective_radius` and `sersic_index`, as the `intensity` is solved for by the linear inversion].\n", + "\n", + "The extra galaxies are grouped into their own `af.Collection`, which is passed to the overall model via its\n", + "`extra_galaxies` input, alongside the `galaxies` collection containing the main galaxy. This is the same API used\n", + "throughout the `autogalaxy_workspace` for extra galaxies, and it is how **PyAutoGalaxy** knows these galaxies are\n", + "nuisance components of the fit rather than the subject of the analysis." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Extra Galaxies:\n", + "\n", + "extra_galaxies_list = []\n", + "\n", + "for extra_galaxy_centre in extra_galaxies_centres:\n", + "\n", + " extra_galaxy = af.Model(\n", + " ag.Galaxy,\n", + " redshift=0.5,\n", + " bulge=ag.lp_linear.SersicSph,\n", + " )\n", + "\n", + " extra_galaxy.bulge.centre = extra_galaxy_centre\n", + "\n", + " extra_galaxies_list.append(extra_galaxy)\n", + "\n", + "extra_galaxies = af.Collection(extra_galaxies_list)\n", + "\n", + "# Overall Model:\n", + "\n", + "model = af.Collection(\n", + " galaxies=af.Collection(galaxy=galaxy), extra_galaxies=extra_galaxies\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `info` attribute confirms the model includes the extra galaxy, with its fixed centre and its\n", + "free `effective_radius` and `sersic_index` parameters." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(model.info)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Extra Galaxies Fit__\n", + "\n", + "We fit this model with the same search set up as before. The model has only two more free parameters than the\n", + "noise-scaling fit, thanks to the fixed centre and linear light profile, so the fit remains fast." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "search = af.Nautilus(\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_4\"),\n", + " name=\"tutorial_1_extra_galaxies_model\",\n", + " unique_tag=dataset_name,\n", + " n_live=100,\n", + " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", + ")\n", + "\n", + "analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", + "\n", + "print(\n", + " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_4/tutorial_1_extra_galaxies_model\"\n", + " \" folder for live output of the results, images and model.\"\n", + " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + ")\n", + "\n", + "result_extra_galaxies = search.fit(model=model, analysis=analysis)\n", + "\n", + "print(\"The search has finished run - you may now continue the notebook.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plotting the maximum log likelihood fit shows the extra galaxy's emission is now fitted and subtracted by its own\n", + "light profile, leaving the main galaxy's light cleanly fitted by the `Sersic` bulge." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_fit_imaging(fit=result_extra_galaxies.max_log_likelihood_fit)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The result's `info` shows the inferred `effective_radius` and `sersic_index` of the extra galaxy, alongside the\n", + "main galaxy's parameters, which are no longer at risk of being biased by the interloper's light." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(result_extra_galaxies.info)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Which Approach When__\n", + "\n", + "We have seen the two extremes of dealing with an extra galaxy. Choosing between them comes back to the question at\n", + "the start of this tutorial \u2014 does its light overlap the region we fit? \u2014 which in practice is settled by two\n", + "properties of the interloper:\n", + "\n", + "- **Distance from the main galaxy's emission**: An extra galaxy whose light is well separated from the main galaxy\n", + " (as in this tutorial) can be cleanly noise-scaled away. If its light blends into the main galaxy's emission,\n", + " noise scaling would also delete main-galaxy emission we need to fit \u2014 biasing exactly the outer isophotes that\n", + " constrain the `effective_radius` and `sersic_index` \u2014 and the light must be modeled instead.\n", + "\n", + "- **Brightness**: A faint interloper whose emission barely rises above the noise can often be ignored entirely, or\n", + " noise-scaled with no consequence. A bright one must be removed or modeled, or it will bias the fit. The brighter\n", + " the interloper, the further its light extends above the noise, and the more likely it blends with the main\n", + " galaxy.\n", + "\n", + "When in doubt, fit both approaches and compare the inferred models of the main galaxy: if its parameters shift\n", + "appreciably, the extra galaxy's light is leaking into the fit and explicit modeling is the safer choice.\n", + "\n", + "A middle ground also exists and is fully supported by the API: noise-scale the interloper's bright central regions\n", + "but also include it in the model, so its faint outer light under the main galaxy is still subtracted. The\n", + "`extra_galaxies` collection simply contains whatever galaxies you give it.\n", + "\n", + "Finally, the `SersicSph` profile we used for the extra galaxy can be swapped for any light model from the earlier\n", + "chapters. In particular, a Multi Gaussian Expansion basis (chapter 2) captures irregular interloper morphologies\n", + "with no extra non-linear parameters, and is the recommended choice once the number of extra galaxies grows beyond\n", + "a handful \u2014 see `autogalaxy_workspace/*/imaging/features/extra_galaxies/modeling.py` for this extension.\n", + "\n", + "__Wrap Up__\n", + "\n", + "In this tutorial, we took the first step up in scale from the single galaxy of chapters 1 to 3, and learnt:\n", + "\n", + "1. Real galaxies live in crowded fields, and interloping extra galaxies are the first complication real data\n", + " throws at us.\n", + "\n", + "2. Whether an extra galaxy matters hinges on one question: does its **light** overlap the region of the image we\n", + " fit? If it does, an unmodeled interloper biases the main galaxy's inferred structure.\n", + "\n", + "3. Noise scaling removes an extra galaxy's light from the fit without adding model complexity, by zeroing its data\n", + " and inflating its noise, while keeping the pixels in the fit's geometry.\n", + "\n", + "4. The `extra_galaxies` modeling API includes extra galaxies in the model with their own light profiles, with\n", + " their centres fixed to the observed light centres to keep parameter space simple and well behaved.\n", + "\n", + "5. Which approach is appropriate depends on the interloper's distance from the main galaxy's emission and its\n", + " brightness, and the two approaches can be combined.\n", + "\n", + "Throughout, the extra galaxy was a nuisance: something to remove or account for, so that our analysis of the main\n", + "galaxy remained accurate. In the next tutorial we meet systems where that framing breaks down entirely, because a\n", + "second galaxy is not a nuisance but a co-equal subject of the analysis, blended with the first and of comparable\n", + "brightness. There, no single galaxy is \"the\" galaxy, and the model must treat them all on an equal footing." + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.ipynb b/notebooks/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.ipynb new file mode 100644 index 0000000..b733673 --- /dev/null +++ b/notebooks/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.ipynb @@ -0,0 +1,637 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Tutorial 2: Multi-Galaxy Blends\n", + "===============================\n", + "\n", + "In the previous tutorial, we learned how to deal with extra galaxies near the galaxy we care about \u2014 nuisance\n", + "objects whose light contaminates the data but which are not themselves the subject of our study. We removed their\n", + "emission from the analysis, or gave them a heavily restricted model, and the single galaxy we were studying\n", + "remained the star of the show.\n", + "\n", + "In this tutorial, we meet systems where that picture breaks down entirely: **multi-galaxy blends**, where two\n", + "(or more) galaxies of comparable brightness overlap on the sky and *every one of them* is a science subject.\n", + "Neither galaxy is a minor contaminant we can mask away or simplify \u2014 they are **co-dominant**, and each needs\n", + "its own free light model, fitted simultaneously in a single analysis.\n", + "\n", + "How do such systems arise physically? There are two main channels:\n", + "\n", + "- **Interacting pairs and mergers**: two galaxies at the same redshift caught orbiting one another or in the\n", + " act of merging. Their projected separation is small compared to the extent of their light, so their surface\n", + " brightness distributions genuinely overlap \u2014 and the interaction itself (tidal features, triggered star\n", + " formation, morphological disturbance) is often exactly the science we want to measure, which is why both\n", + " galaxies are subjects of the fit.\n", + "\n", + "- **Chance projections**: two physically unrelated galaxies at different redshifts that happen to lie along\n", + " nearly the same line of sight. There is no physical interaction, but on the image their light still blends,\n", + " and measuring either galaxy's morphology or photometry requires decomposing the blend.\n", + "\n", + "In both cases the modeling challenge is the same: the flux in each pixel is the sum of both galaxies'\n", + "emission (blurred by the PSF), so the only principled way to measure either galaxy is to fit a model in which\n", + "both galaxies are present at once. This tutorial shows how to compose and fit that model, how to stop its\n", + "parameter space growing out of control, and what the data can \u2014 and cannot \u2014 tell us about a blend.\n", + "\n", + "__Contents__\n", + "\n", + "- **Initial Setup:** Load imaging of two blended galaxies.\n", + "- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist.\n", + "- **Mask:** Define a mask which encloses the light of both galaxies.\n", + "- **Over Sampling:** Centre the adaptive over sampling grid on every galaxy, not just one.\n", + "- **Why Not Fit Them Separately?:** Why masking out one galaxy and fitting the other biases both measurements.\n", + "- **Model:** Compose a model with one free light model per galaxy, and count its parameters.\n", + "- **Fixing the Centres:** Fix each galaxy's centre to its observed light centre, and why this is standard.\n", + "- **Model Fit:** Fit the two-galaxy model to the data with a non-linear search.\n", + "- **Result:** Decompose the blend into each galaxy's light and measure per-galaxy photometry.\n", + "- **Light Decomposition Degeneracy:** The total flux is well constrained, but its split between the galaxies is not.\n", + "- **Wrap Up:** Summary and the road to cluster fields." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Google Colab Setup__\n", + "\n", + "This cell sets up the environment when the notebook is run on Google Colab: it installs the\n", + "required PyAuto packages, clones the workspace (configuration files and example datasets) and\n", + "points the configuration at it. If you are running the notebook elsewhere (e.g. locally via\n", + "your own installation) it does nothing, and you can run it safely.\n", + "\n", + "Colab tip: model-fits run much faster on a GPU \u2014 enable one via \"Runtime\" -> \"Change runtime\n", + "type\" -> \"Hardware accelerator\" before running the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import google.colab\n", + "except ImportError:\n", + " from autogalaxy import setup_colab as _setup_colab\n", + "else:\n", + " import importlib\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.check_call(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"autonerves\", \"--no-deps\"]\n", + " )\n", + " _setup_colab = importlib.import_module(\"autonerves.setup_colab\")\n", + "\n", + "_setup_colab.setup(\"howtogalaxy\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "\n", + "from autogalaxy import jax_wrapper # Sets JAX environment before other imports\n", + "\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", + "\n", + "from pathlib import Path\n", + "import autofit as af\n", + "import autogalaxy as ag\n", + "import autogalaxy.plot as aplt" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Initial Setup__\n", + "\n", + "We begin with `Imaging` of two galaxies, separated by 2.0\", whose light blends together:\n", + "\n", + " - The first galaxy's bulge is a `Sersic` centred at (0.0\", -1.0\"), with effective radius 0.8\" and Sersic\n", + " index 2.5.\n", + " - The second galaxy's bulge is a `Sersic` centred at (0.0\", 1.0\"), with effective radius 0.6\" and Sersic\n", + " index 3.0.\n", + "\n", + "Because both galaxies have Sersic indices well above 1, their light falls off slowly with radius \u2014 each\n", + "galaxy's outer envelope extends far past the midpoint between them, so a significant fraction of the flux in\n", + "every central pixel comes from *both* galaxies at once. This is the blend we must decompose.\n", + "\n", + "__Dataset Auto-Simulation__\n", + "\n", + "If the dataset does not already exist on your system, it will be created by running the corresponding\n", + "simulator script. This ensures that all example scripts can be run without manually simulating data first." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset_name = \"sersic_x2\"\n", + "dataset_path = Path(\"dataset\") / \"imaging\" / dataset_name\n", + "\n", + "if ag.util.dataset.should_simulate(str(dataset_path)):\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.run(\n", + " [sys.executable, \"scripts/simulators/sersic_x2.py\"],\n", + " check=True,\n", + " )\n", + "\n", + "dataset = ag.Imaging.from_fits(\n", + " data_path=dataset_path / \"data.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " psf_path=dataset_path / \"psf.fits\",\n", + " pixel_scales=0.1,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When we plot the data, the difference from every dataset we have modeled so far is obvious: there are two\n", + "bright galaxies, and between them their light merges into a continuous bridge of emission. There is no radius\n", + "at which we could draw a boundary and say \"this flux belongs to galaxy 0, that flux to galaxy 1\"." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Mask__\n", + "\n", + "We define a 3.0\" circular mask centred between the two galaxies. For a blended system, the mask must enclose\n", + "the light of **both** galaxies \u2014 the fit is going to decompose the blend, so it needs to see the whole blend.\n", + "A mask sized by eye around either galaxy individually would cut through the other galaxy's light, guaranteeing\n", + "a biased fit." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mask_radius = 3.0\n", + "\n", + "mask = ag.Mask2D.circular(\n", + " shape_native=dataset.shape_native,\n", + " pixel_scales=dataset.pixel_scales,\n", + " radius=mask_radius,\n", + ")\n", + "\n", + "dataset = dataset.apply_mask(mask=mask)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Over Sampling__\n", + "\n", + "We use the adaptive over sampling scheme introduced in earlier chapters, which evaluates the steep central\n", + "regions of a galaxy's light at high resolution. The one multi-galaxy specific point is that the adaptive grid\n", + "is centred on **every** galaxy, not just one \u2014 each galaxy has its own steep central light profile needing\n", + "accurate evaluation, and `centre_list` takes as many centres as we give it.\n", + "\n", + "The two centres below are the observed centres of the two galaxies, which for this simulated dataset we know\n", + "exactly. For real data you would measure them from the image itself \u2014 the `autogalaxy_workspace`'s\n", + "`multi_galaxy` package loads them from a `galaxy_centres.json` file, which a GUI in the workspace writes from\n", + "mouse clicks on the image." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxy_centres = [(0.0, -1.0), (0.0, 1.0)]\n", + "\n", + "over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from(\n", + " grid=dataset.grid,\n", + " sub_size_list=[4, 2, 2],\n", + " radial_list=[0.3, 0.6],\n", + " centre_list=galaxy_centres,\n", + ")\n", + "\n", + "dataset = dataset.apply_over_sampling(over_sample_size_lp=over_sample_size)\n", + "\n", + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Why Not Fit Them Separately?__\n", + "\n", + "Before composing the joint model, it is worth asking the obvious question: why not just fit each galaxy on its\n", + "own? We could mask out galaxy 1, fit galaxy 0, then swap and repeat \u2014 turning one hard problem into two easy\n", + "ones, each identical to the single-galaxy fits of chapter 2.\n", + "\n", + "The answer is that this biases **both** measurements, for two compounding reasons:\n", + "\n", + "- **Contaminated flux**: whatever mask we draw, the pixels we keep still contain the other galaxy's light \u2014\n", + " the blend has no clean boundary. The fit attributes that extra flux to the galaxy being modeled, inflating\n", + " its measured brightness and dragging its effective radius and Sersic index towards the contaminating\n", + " neighbour. The neighbour's light is brightest exactly on the side facing it, so the fitted ellipticity and\n", + " centre are skewed in that direction too.\n", + "\n", + "- **Amputated flux**: the mask that removes the neighbour also removes part of the modeled galaxy's own outer\n", + " envelope, precisely where the two overlap. The fit never sees that light, so it underestimates how far the\n", + " galaxy extends \u2014 biasing the effective radius and Sersic index in the *opposite* direction to the first\n", + " effect. The two biases do not cancel; they combine into measurements that are simply wrong, in ways that\n", + " depend on the mask choice.\n", + "\n", + "The joint fit has neither problem. Every pixel in the mask is modeled as the sum of both galaxies' light, so\n", + "no flux is wrongly attributed and none is thrown away. Decomposing the blend *is* the fit \u2014 this is the whole\n", + "point of the multi-galaxy regime, and it is why both galaxies must be in the model even if only one of them is\n", + "the galaxy we ultimately care about scientifically." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Model__\n", + "\n", + "We now compose the model, and here the multi-galaxy regime shows its teeth: every galaxy needs its own free\n", + "light model, so the model has one `Galaxy` entry per galaxy.\n", + "\n", + "We build each galaxy in a loop over the observed centres and store them in a dictionary as `galaxy_0`,\n", + "`galaxy_1`, etc. This list-based composition scales to any number of blended galaxies, and it is the same API\n", + "the `autogalaxy_workspace`'s `multi_galaxy` package uses \u2014 so nothing needs re-learning later.\n", + "\n", + "For each galaxy's light we use a Multi Gaussian Expansion (MGE) made of linear light profiles, built by the\n", + "utility function `ag.model_util.mge_model_from` (this hides the longer `Basis` composition API we stepped\n", + "through in the linear profiles tutorial of chapter 2). Each galaxy's 20 Gaussians add just **4 non-linear\n", + "parameters** \u2014 a shared centre and shared elliptical components, with every `sigma` fixed to log-spaced values\n", + "and every `intensity` solved for by the linear inversion.\n", + "\n", + "The `centre_prior_is_uniform=True` input places a uniform prior of width 0.2\" on each galaxy's centre,\n", + "centred on its observed light centre \u2014 so `galaxy_0`'s centre priors are centred on (0.0\", -1.0\") and\n", + "`galaxy_1`'s on (0.0\", 1.0\"). This is essential in a blend: if both galaxies had identical broad centre\n", + "priors, the model would not know which component is which, and the search would waste time exploring\n", + "solutions where the two galaxies have swapped places (or worse, piled on top of one another).\n", + "\n", + "The MGE choice matters more here than anywhere we have used it before, because the multi-galaxy regime\n", + "multiplies whatever light model we choose by the number of galaxies:\n", + "\n", + " - Two full `Sersic` bulges (`ag.lp.Sersic`): **14** free parameters (7 each).\n", + " - Two linear `Sersic` bulges (`ag.lp_linear.Sersic`, intensities solved for): **12** free parameters.\n", + " - Two MGEs: **8** free parameters (4 each) \u2014 and each MGE is far more flexible than a single Sersic,\n", + " capturing the asymmetries and radially-varying ellipticity that real (especially interacting!) galaxies\n", + " show.\n", + "\n", + "A light model that is both cheap and flexible is close to essential in this regime \u2014 this is exactly where\n", + "the MGE shines. On top of the parameter count, solving the intensities by linear algebra has a second, subtler\n", + "benefit for blends that we will return to at the end of this tutorial." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxy_dict = {}\n", + "\n", + "for i, centre in enumerate(galaxy_centres):\n", + "\n", + " bulge = ag.model_util.mge_model_from(\n", + " mask_radius=mask_radius,\n", + " total_gaussians=20,\n", + " centre_prior_is_uniform=True,\n", + " centre=(centre[0], centre[1]),\n", + " sigma_min=dataset.pixel_scales[0] / 10.0,\n", + " )\n", + "\n", + " galaxy_dict[f\"galaxy_{i}\"] = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge)\n", + "\n", + "model = af.Collection(galaxies=af.Collection(**galaxy_dict))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The model's `info` shows `galaxy_0` and `galaxy_1` each carrying their own free MGE \u2014 the signature of the\n", + "multi-galaxy regime \u2014 and its `prior_count` gives the total number of free parameters." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(model.info)\n", + "\n", + "print(f\"Free parameters (free centres): {model.prior_count}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The count is **8**: each galaxy contributes its MGE's 2 centre parameters and 2 elliptical component\n", + "parameters. Every further blended galaxy will add 4 more \u2014 the growth is linear, and with plain Sersic light\n", + "profiles instead of MGEs it would grow at 7 per galaxy instead.\n", + "\n", + "__Fixing the Centres__\n", + "\n", + "Before fitting, we make one change that is standard practice for blended systems: we **fix each galaxy's\n", + "centre to its observed light centre**, removing 2 free parameters per galaxy. The previous tutorial fixed the\n", + "interloper's centre for the same reason; in a blend the case is even stronger.\n", + "\n", + "Why? With a single galaxy, the data constrains the centre extremely well: it is simply the peak of the light,\n", + "and there is only one plausible culprit for every photon. With blended light this breaks down. Each pixel's\n", + "flux is the sum of both galaxies' light, so a small shift of one galaxy's centre can be compensated by changes\n", + "to the other galaxy's model \u2014 its ellipticity, its Gaussians' relative intensities, or a shift of its own\n", + "centre. The free centres become degenerate with one another and with the rest of the model, and the search\n", + "wanders these degeneracies, converging slowly and sometimes settling in unphysical corners of parameter space\n", + "where one model galaxy has drifted between the two real ones.\n", + "\n", + "Fixing each centre to the observed light peak resolves this at minimal cost. Unlike almost every other light\n", + "profile parameter, the centre is something we can measure directly and robustly from the image before any\n", + "fitting \u2014 a peak position is hard to fake, even in a blend. (Measuring a genuine offset \u2014 say, of a nucleus\n", + "displaced during an interaction \u2014 is real science, but it is a *follow-up* fit performed after a robust model\n", + "with fixed centres has been found.)\n", + "\n", + "The `centre_fixed` input of `mge_model_from` fixes every Gaussian's centre to the input tuple, so the centre\n", + "is no longer a free parameter with a prior." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxy_dict = {}\n", + "\n", + "for i, centre in enumerate(galaxy_centres):\n", + "\n", + " bulge = ag.model_util.mge_model_from(\n", + " mask_radius=mask_radius,\n", + " total_gaussians=20,\n", + " centre_fixed=(centre[0], centre[1]),\n", + " sigma_min=dataset.pixel_scales[0] / 10.0,\n", + " )\n", + "\n", + " galaxy_dict[f\"galaxy_{i}\"] = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge)\n", + "\n", + "model = af.Collection(galaxies=af.Collection(**galaxy_dict))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Printing the model's `info` again, each centre is now listed as a fixed value with no prior, and the free\n", + "parameter count has dropped from 8 to **4** \u2014 just each galaxy's two elliptical components. Pause on that\n", + "number: we are about to decompose a fully blended pair of galaxies, capturing both morphologies with 40\n", + "Gaussians, by searching a 4-dimensional parameter space. Compare it to the 14 dimensions two full Sersics\n", + "would have cost, and it is clear how the combination of linear profiles, the MGE and fixed centres is what\n", + "makes this regime tractable at all." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(model.info)\n", + "\n", + "print(f\"Free parameters (fixed centres): {model.prior_count}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Model Fit__\n", + "\n", + "We fit the model with the nested sampling algorithm `Nautilus`, as in previous chapters. Thanks to the MGE and\n", + "the fixed centres the parameter space is small and well behaved, so `n_live=100` (the chapter 2 default) is\n", + "ample.\n", + "\n", + "Note that the `AnalysisImaging` object is completely unchanged from single-galaxy fitting \u2014 the multi-galaxy\n", + "regime changed the model composition, and nothing else." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "search = af.Nautilus(\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_4\"),\n", + " name=\"tutorial_2_multi_galaxy\",\n", + " unique_tag=dataset_name,\n", + " n_live=100,\n", + " n_batch=50, # GPU batching and VRAM use explained in chapter 2 tutorial 2.\n", + ")\n", + "\n", + "analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", + "\n", + "print(\n", + " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_4/tutorial_2_multi_galaxy\"\n", + " \" folder for live output of the results, images and model.\"\n", + " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + ")\n", + "\n", + "result = search.fit(model=model, analysis=analysis)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Result__\n", + "\n", + "The result's `info` shows the inferred model, with each galaxy's parameters listed under its own `galaxy_0` /\n", + "`galaxy_1` entry." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(result.info)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The fit subplot shows the joint model reproducing the full blend \u2014 both galaxies and the bridge of overlapping\n", + "light between them \u2014 with residuals at the noise level." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_fit_imaging(fit=result.max_log_likelihood_fit)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The core deliverable of a multi-galaxy fit is the **decomposition**: `subplot_fit_imaging_of_galaxy` shows,\n", + "for each galaxy in turn, its modeled light on its own and the data with the *other* galaxy's model subtracted.\n", + "This is the closest thing to an image of each galaxy as it would appear without its companion \u2014 something no\n", + "mask could ever have given us." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "for i in range(len(galaxy_centres)):\n", + " aplt.subplot_fit_imaging_of_galaxy(fit=result.max_log_likelihood_fit, galaxy_index=i)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With the blend decomposed, per-galaxy photometry is direct: each galaxy's model image contains only its own\n", + "light, so summing it gives that galaxy's flux uncontaminated by its neighbour." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxies = result.max_log_likelihood_galaxies\n", + "\n", + "for i, galaxy in enumerate(galaxies):\n", + " image = galaxy.image_2d_from(grid=dataset.grids.lp)\n", + " print(f\"galaxy_{i}: total model flux = {float(image.array.sum()):.3f}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Light Decomposition Degeneracy__\n", + "\n", + "The photometry above comes with an important caveat, and it is the deepest lesson of the multi-galaxy regime.\n", + "\n", + "In the pixels where the two galaxies overlap, the data constrains only the **sum** of their light. A model\n", + "that brightens galaxy 0's outer envelope and dims galaxy 1's by the same amount produces a nearly identical\n", + "image, and therefore a nearly identical likelihood. The *total* flux of the blend is pinned down superbly \u2014\n", + "every photon is accounted for \u2014 but the *split* of that flux between the two galaxies is constrained only by\n", + "the shapes of the profiles, which is a much weaker lever. The closer the pair and the larger their overlap,\n", + "the weaker it gets.\n", + "\n", + "You can see this degeneracy directly in the posterior. When you run this tutorial for real (rather than\n", + "skimming the generated output), open the search's corner plot \u2014 the `.png` in the output folder's `image`\n", + "directory, or plot it yourself with `aplt.corner_anesthetic(samples=result.samples)` \u2014 and focus on the panels\n", + "pairing `galaxy_0`'s parameters against `galaxy_1`'s. You will see tilted, anti-correlated contours: when one\n", + "galaxy's model claims more of the shared light, the other's gives it up. Each galaxy's flux, effective radius\n", + "and shape measurements inherit the full width of this degeneracy, so their error bars are larger \u2014 sometimes\n", + "much larger \u2014 than an equivalent isolated galaxy's would be. Honest science on blended systems propagates\n", + "these widened uncertainties rather than quoting the best-fit split as if it were certain, and this is\n", + "precisely why we fit blends with a sampler that maps the full posterior rather than an optimizer that returns\n", + "a single point: the best-fit sits somewhere on the degeneracy ridge and tells you nothing about the ridge's\n", + "length.\n", + "\n", + "This is also the subtler benefit of linear light profiles promised earlier. The flux ratio between the two\n", + "galaxies \u2014 the quantity most degenerate in a blend \u2014 lives in the `intensity` values, and the inversion solves\n", + "those exactly at every likelihood evaluation. The non-linear search never has to explore the intensity\n", + "degeneracy stochastically; it is handled by linear algebra, leaving the sampler to map only the (much milder)\n", + "degeneracies between the galaxies' shapes. Fitting a blend with standard (non-linear) light profiles forfeits\n", + "this, and is markedly slower and less reliable as a result.\n", + "\n", + "__Wrap Up__\n", + "\n", + "In this tutorial, we modeled a system of two blended galaxies simultaneously. Let's summarise what we've\n", + "learnt:\n", + "\n", + "- **Co-dominant galaxies**: When two or more galaxies of comparable brightness overlap on the sky, every one\n", + " of them needs its own free light model \u2014 unlike the nuisance neighbours of the previous tutorial, none can\n", + " be masked away or simplified. Such blends arise from interacting pairs and mergers at one redshift, or\n", + " chance projections of unrelated galaxies along the line of sight.\n", + "\n", + "- **Separate fits are biased**: Masking one galaxy out and fitting the other contaminates the fit with the\n", + " neighbour's flux while amputating the target's own overlapping light \u2014 both measurements come out wrong.\n", + " The joint fit, in which every pixel is modeled as the sum of both galaxies, is the only principled\n", + " decomposition.\n", + "\n", + "- **Parameter accounting**: Two full Sersics would cost 14 free parameters; linear profiles cut this to 12,\n", + " the MGE to 8, and fixing each galaxy's centre to its observed light centre to just 4 \u2014 with the MGE's\n", + " intensities solved by linear algebra rather than sampled. Cheap, flexible light models are what make the\n", + " multi-galaxy regime tractable, because every additional galaxy multiplies the cost.\n", + "\n", + "- **Fixed centres**: Blended light makes free centres degenerate \u2014 one galaxy's centre shift can be absorbed\n", + " by changes to the other's model. Fixing each centre to the observed light peak, the one parameter we can\n", + " measure robustly before fitting, is the standard trick for taming this.\n", + "\n", + "- **Light decomposition degeneracy**: The data constrains the total flux of the blend far better than its\n", + " split between the galaxies, producing anti-correlated posteriors between the two galaxies' parameters and\n", + " inflating the uncertainties on all per-galaxy measurements. Map it with a sampler, and propagate it.\n", + "\n", + "Everything here scaled comfortably to two galaxies, and the loop-based composition would carry us to three or\n", + "four. But the ladder keeps climbing: in the next tutorial we reach **cluster fields**, where a brightest\n", + "cluster galaxy sits among tens to hundreds of member galaxies. Giving every member its own free model \u2014 even\n", + "a 2-parameter one \u2014 cannot scale that far, and modeling them one blend at a time would reintroduce every bias\n", + "we just eliminated. The answer is catalogue-driven composition: the member population is built automatically\n", + "from a catalogue of measured positions and photometry, and modeled collectively. That is where we go next." + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/chapter_4_scaling_up_galaxies/tutorial_3_cluster.ipynb b/notebooks/chapter_4_scaling_up_galaxies/tutorial_3_cluster.ipynb new file mode 100644 index 0000000..e99b6fb --- /dev/null +++ b/notebooks/chapter_4_scaling_up_galaxies/tutorial_3_cluster.ipynb @@ -0,0 +1,650 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Tutorial 3: Cluster\n", + "===================\n", + "\n", + "Throughout this chapter we have been scaling up: from a single galaxy with extra galaxies nearby, to blended\n", + "multi-galaxy systems where every galaxy received its own free light model.\n", + "\n", + "This tutorial climbs the final rung: **cluster fields**, the richest environments in which galaxies live. An\n", + "image of a galaxy cluster contains:\n", + "\n", + "- **A brightest cluster galaxy (BCG)**: the giant elliptical that sits at the cluster's centre. BCGs are the\n", + " most massive galaxies in the Universe, built up over billions of years by swallowing their neighbours, and\n", + " their extended envelopes hold a record of that assembly history.\n", + "\n", + "- **Tens to hundreds of member galaxies**: the cluster's population of (mostly elliptical) galaxies,\n", + " scattered across a field many times wider than the single-galaxy images of earlier chapters.\n", + "\n", + "- **Intra-cluster light (ICL)**: a faint, diffuse glow of stars stripped from member galaxies, filling the\n", + " space between them. It blends smoothly into the BCG's outer envelope, which is why careful modeling of the\n", + " BCG's light is the starting point of every ICL measurement.\n", + "\n", + "Why model all of this light? Three science cases drive the cluster regime:\n", + "\n", + "- **BCG growth**: the size and shape of a BCG's outer envelope constrains how much of its mass was accreted\n", + " from merging satellites \u2014 but measuring that envelope requires subtracting every member galaxy whose light\n", + " overlaps it.\n", + "\n", + "- **Intra-cluster light**: the ICL is only measurable once the BCG and members are modeled and removed;\n", + " what remains is the diffuse component.\n", + "\n", + "- **Member photometry**: the luminosity of each member, uncontaminated by its neighbours and the BCG's\n", + " envelope, is what luminosity functions and scaling relations are built from.\n", + "\n", + "(A quick note for readers heading towards gravitational lensing: galaxy clusters are also the Universe's\n", + "most powerful strong lenses, and the lensing version of this scale \u2014 where a catalogue like the one below\n", + "drives member *masses* and the fit is to point-source image positions rather than galaxy light \u2014 lives in\n", + "chapter 4 of the **HowToLens** lectures.)\n", + "\n", + "__The Scaling Problem__\n", + "\n", + "The previous tutorials composed models the way we have all chapter: one `af.Model(ag.Galaxy)` per galaxy,\n", + "each with its own free light profile. A blend of two or three galaxies is fine. A cluster is not:\n", + "\n", + "- **Parameters**: ten members with a free spherical Sersic each (5 parameters per galaxy) is 50 free\n", + " parameters before we even touch the BCG; a hundred members is 500. No non-linear search can sample such a\n", + " space reliably.\n", + "\n", + "- **Information**: the faint members do not contain enough signal to constrain four free parameters each \u2014\n", + " most of those dimensions would be unconstrained noise.\n", + "\n", + "- **Practicality**: hand-writing Python model code for hundreds of galaxies is not sensible, and no\n", + " astronomer works that way. What an observer actually has is a **catalogue**: a table listing where each\n", + " member is and how bright it is.\n", + "\n", + "The cluster regime therefore changes how the model is composed. The BCG \u2014 the galaxy whose structure we\n", + "care most about \u2014 is modeled individually and richly, with a free MGE, exactly as a single galaxy would be.\n", + "The member population is driven by the catalogue: each member's centre is fixed to its catalogue position,\n", + "its shape is fixed to sensible values, and its intensity is TIED to its catalogue luminosity through a\n", + "single shared free normalization. Adding a member is a row append; the model's dimensionality does not grow\n", + "with the population.\n", + "\n", + "__Contents__\n", + "\n", + "- **The Scaling Problem:** Why one-free-model-per-galaxy cannot scale to a cluster's member population.\n", + "- **Dataset:** Load the simulated cluster field (1 BCG + 10 members).\n", + "- **Dataset Auto-Simulation:** Automatically simulate the dataset if it does not already exist.\n", + "- **Member Catalogue:** Load the member centres and luminosities from `scaling_galaxies.csv`.\n", + "- **Masking:** Mask the wide cluster field and over-sample every galaxy's centre.\n", + "- **Model:** Compose the two-tier cluster model \u2014 free BCG MGE + catalogue-driven member tier.\n", + "- **Search + Analysis:** Configure the Nautilus non-linear search and the analysis.\n", + "- **Model Fit:** Run the fit.\n", + "- **Result:** Inspect the fit, including the member-subtracted BCG decomposition.\n", + "- **Per-Member Results:** Access each member's fitted light \u2014 catalogue-scale photometry.\n", + "- **Refinements:** Where the composition goes next \u2014 freeing tier shapes, promoting bright members.\n", + "- **Wrap Up:** The chapter and the **HowToGalaxy** lectures conclude." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Google Colab Setup__\n", + "\n", + "This cell sets up the environment when the notebook is run on Google Colab: it installs the\n", + "required PyAuto packages, clones the workspace (configuration files and example datasets) and\n", + "points the configuration at it. If you are running the notebook elsewhere (e.g. locally via\n", + "your own installation) it does nothing, and you can run it safely.\n", + "\n", + "Colab tip: model-fits run much faster on a GPU \u2014 enable one via \"Runtime\" -> \"Change runtime\n", + "type\" -> \"Hardware accelerator\" before running the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import google.colab\n", + "except ImportError:\n", + " from autogalaxy import setup_colab as _setup_colab\n", + "else:\n", + " import importlib\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.check_call(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"autonerves\", \"--no-deps\"]\n", + " )\n", + " _setup_colab = importlib.import_module(\"autonerves.setup_colab\")\n", + "\n", + "_setup_colab.setup(\"howtogalaxy\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "\n", + "from autogalaxy import jax_wrapper # Sets JAX environment before other imports\n", + "\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", + "\n", + "from pathlib import Path\n", + "\n", + "import autofit as af\n", + "import autogalaxy as ag\n", + "import autogalaxy.plot as aplt" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Dataset__\n", + "\n", + "We fit a simulated cluster field kept deliberately small so it runs quickly \u2014 1 BCG and 10 member galaxies\n", + "in a 25\" x 25\" field \u2014 but containing every ingredient of the cluster regime. A real cluster simply has more\n", + "members, and we will see that scaling the model up to hundreds of them does not add a single free parameter." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset_name = \"simple\"\n", + "dataset_path = Path(\"dataset\") / \"cluster\" / dataset_name" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Dataset Auto-Simulation__\n", + "\n", + "If the dataset does not already exist on your system, it will be created by running the corresponding\n", + "simulator script. This ensures that all example scripts can be run without manually simulating data first." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "if ag.util.dataset.should_simulate(str(dataset_path)):\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.run(\n", + " [sys.executable, \"scripts/simulators/cluster.py\"],\n", + " check=True,\n", + " )\n", + "\n", + "\n", + "dataset = ag.Imaging.from_fits(\n", + " data_path=dataset_path / \"data.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " psf_path=dataset_path / \"psf.fits\",\n", + " pixel_scales=0.1,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plotting the dataset shows what a cluster field looks like in imaging: the bright, extended BCG dominating\n", + "the centre, with the fainter members scattered across the frame. In real data the diffuse intra-cluster\n", + "light would fill the space between them.\n", + "\n", + "Note the field of view compared to earlier chapters \u2014 the members sit up to ~10\" from the centre, so the\n", + "image is far larger than the ~6\" cutouts we fitted before." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Member Catalogue__\n", + "\n", + "The member population enters the model through a catalogue file, `scaling_galaxies.csv`, with one row per\n", + "member galaxy and three columns:\n", + "\n", + " - `y`, `x`: the member's centre in arc-seconds \u2014 where the galaxy is.\n", + " - `luminosity`: the member's luminosity \u2014 how bright it is. Any consistent luminosity units work, because\n", + " (as we will see below) only a shared normalization of the luminosities is fitted.\n", + "\n", + "This is exactly what an observer measures: a photometry catalogue. In a real analysis this CSV comes\n", + "straight from source-extraction software or a survey database \u2014 it is spreadsheet-editable, and adding a\n", + "member to the model means appending a row, not writing Python.\n", + "\n", + "We load it with `ag.galaxy_table_from_csv`, the catalogue-loading API." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "scaling_table = ag.galaxy_table_from_csv(\n", + " file_path=dataset_path / \"scaling_galaxies.csv\"\n", + ")\n", + "\n", + "member_centres = scaling_table.centres.in_list\n", + "member_luminosities = scaling_table.luminosities\n", + "\n", + "print(f\"Members in catalogue: {len(member_centres)}\")\n", + "print(f\"First member centre: {member_centres[0]}\")\n", + "print(f\"First member luminosity: {float(member_luminosities[0])}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The BCG's centre is recorded in its own file, `bcg_centres.json` \u2014 in a real cluster there may be more than\n", + "one dominant galaxy (e.g. a second bright elliptical) that deserves individual modeling, so their centres\n", + "are kept separate from the member catalogue." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "bcg_centres = ag.from_json(file_path=dataset_path / \"bcg_centres.json\")\n", + "\n", + "print(f\"BCG centre: {bcg_centres[0]}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Masking__\n", + "\n", + "We mask the field generously \u2014 the members span the frame, and the model must account for every galaxy\n", + "inside the mask (an unmodeled galaxy inside the mask would bias the fit, as the first tutorial of this\n", + "chapter showed).\n", + "\n", + "The over-sampling scheme is centred on every galaxy in the field \u2014 the BCG and all ten members \u2014 so each\n", + "galaxy's steep central light gradient is evaluated accurately." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "mask_radius = 11.0\n", + "\n", + "mask = ag.Mask2D.circular(\n", + " shape_native=dataset.shape_native,\n", + " pixel_scales=dataset.pixel_scales,\n", + " radius=mask_radius,\n", + ")\n", + "\n", + "dataset = dataset.apply_mask(mask=mask)\n", + "\n", + "over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from(\n", + " grid=dataset.grid,\n", + " sub_size_list=[4, 2, 2],\n", + " radial_list=[0.3, 0.6],\n", + " centre_list=list(bcg_centres) + list(member_centres),\n", + ")\n", + "\n", + "dataset = dataset.apply_over_sampling(over_sample_size_lp=over_sample_size)\n", + "\n", + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Model__\n", + "\n", + "The model has two tiers, and this two-tier composition is the cluster regime's signature:\n", + "\n", + "**Tier 1 \u2014 the BCG, modeled richly.** The BCG is the science target, so it gets the most flexible light\n", + "model we have: a free Multi-Gaussian Expansion (the MGE introduced in chapter 2), composed via\n", + "`ag.model_util.mge_model_from`. Its 20 Gaussians can capture the BCG's extended envelope, radially-varying\n", + "ellipticity and isophotal twists \u2014 exactly the structures that BCG growth and intra-cluster light studies\n", + "measure. This costs the usual handful of non-linear parameters (centre + elliptical components; the\n", + "Gaussian intensities are solved linearly).\n", + "\n", + "**Tier 2 \u2014 the members, modeled cheaply from the catalogue.** Each member gets a spherical Sersic\n", + "(`SersicSph`) generated in a loop over the catalogue rows:\n", + "\n", + " - its `centre` is FIXED to the catalogue (y, x) position;\n", + " - its shape (`effective_radius`, `sersic_index`) is FIXED to values typical of cluster ellipticals;\n", + " - its `intensity` is TIED to the catalogue luminosity through one shared free parameter:\n", + "\n", + " intensity_i = intensity_scale * luminosity_i\n", + "\n", + "The single `intensity_scale` prior is defined once, outside the loop, and every member's intensity is an\n", + "arithmetic expression of it \u2014 an extension of the prior linking we used in earlier chapters (pairing two\n", + "parameters with `=`, e.g. `bulge.centre = disk.centre`), now doing population-scale work. The whole 10-member tier therefore contributes ONE free parameter to the non-linear\n", + "search, and would still contribute one with 200 members.\n", + "\n", + "Physically, `intensity_scale` is the conversion between the catalogue's luminosity units and the image's\n", + "flux units: the catalogue fixes each member's brightness *relative* to the others (which photometry measures\n", + "well), and the fit solves for the one overall normalization." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# BCG:\n", + "\n", + "bulge = ag.model_util.mge_model_from(\n", + " mask_radius=3.0,\n", + " total_gaussians=20,\n", + " centre_prior_is_uniform=True,\n", + " centre=(bcg_centres[0][0], bcg_centres[0][1]),\n", + " sigma_min=dataset.pixel_scales[0] / 10.0,\n", + ")\n", + "\n", + "galaxy_dict = {\"bcg\": af.Model(ag.Galaxy, redshift=0.5, bulge=bulge)}\n", + "\n", + "# Members: one shared free normalization for the whole tier.\n", + "\n", + "intensity_scale = af.UniformPrior(lower_limit=0.0, upper_limit=10.0)\n", + "\n", + "for i, (centre, luminosity) in enumerate(zip(member_centres, member_luminosities)):\n", + "\n", + " bulge = af.Model(ag.lp.SersicSph)\n", + " bulge.centre = tuple(centre)\n", + " bulge.intensity = intensity_scale * float(luminosity) # tied to the catalogue\n", + " bulge.effective_radius = 0.6\n", + " bulge.sersic_index = 3.0\n", + "\n", + " galaxy_dict[f\"member_{i}\"] = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge)\n", + "\n", + "model = af.Collection(galaxies=af.Collection(**galaxy_dict))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Printing `model.info` confirms the composition: the BCG's MGE parameters are free, every member's intensity\n", + "shows as tied to the single shared `intensity_scale`, and the total dimensionality stays small despite the\n", + "model containing 11 galaxies. This is the punchline of the cluster workflow \u2014 catalogues decouple the\n", + "model's physical richness from the dimensionality of the search." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(model.info)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Search + Analysis__\n", + "\n", + "The analysis is the same `AnalysisImaging` used on every rung of the ladder \u2014 the cluster regime changed how\n", + "the model is *composed*, not how it is *fitted*. We fit with the Nautilus nested sampler, as throughout the\n", + "lectures, which returns the full posterior: the errors on the BCG's structural parameters and on\n", + "`intensity_scale` are what a cluster paper reports." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "search = af.Nautilus(\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_4\"),\n", + " name=\"tutorial_3_cluster\",\n", + " unique_tag=dataset_name,\n", + " n_live=150,\n", + " n_batch=50,\n", + ")\n", + "\n", + "analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Model Fit__\n", + "\n", + "Run the fit. Despite the model containing 11 galaxies, the parameter space is barely larger than a single\n", + "galaxy's \u2014 this is the catalogue tier doing its job." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(\n", + " \"The non-linear search has begun running - checkout the workspace/output/howtogalaxy/chapter_4/tutorial_3_cluster\"\n", + " \" folder for live output of the results, images and model.\"\n", + " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + ")\n", + "\n", + "result = search.fit(model=model, analysis=analysis)\n", + "\n", + "print(\"The search has finished run - you may now continue the notebook.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Result__\n", + "\n", + "The `info` attribute shows the resulting model \u2014 the BCG's MGE parameters with their errors, and the single\n", + "inferred `intensity_scale` shared by the member tier." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "print(result.info)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plotting the maximum likelihood fit shows the full cluster model reproducing the field: BCG and all ten\n", + "members fitted simultaneously." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_fit_imaging(fit=result.max_log_likelihood_fit)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The plot below is the one that motivates cluster light modeling: the fit decomposed by galaxy, showing the\n", + "BCG's light (galaxy index 0) in isolation \u2014 that is, with every member galaxy's light accounted for by its\n", + "own model component rather than contaminating the BCG's.\n", + "\n", + "This member-subtracted BCG is the measurement BCG-growth and intra-cluster light studies are built on: its\n", + "outer envelope can now be traced without the members' light biasing it, and whatever diffuse light the model\n", + "cannot attribute to any galaxy is the ICL candidate signal." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_fit_imaging_of_galaxy(fit=result.max_log_likelihood_fit, galaxy_index=0)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Per-Member Results__\n", + "\n", + "The member tier's single free parameter was `intensity_scale`, but the maximum likelihood instance contains\n", + "the fully-realised light profile of every member \u2014 the tie to the catalogue has been applied, so each member\n", + "carries the intensity its luminosity implies. Looping over the members tabulates the fitted photometry of\n", + "the whole population, exactly as a member-photometry analysis of a real cluster would." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "instance = result.max_log_likelihood_instance\n", + "\n", + "for i in range(len(member_centres)):\n", + " member = getattr(instance.galaxies, f\"member_{i}\")\n", + " print(\n", + " f\"Member {i}: centre={member.bulge.centre}, \"\n", + " f\"intensity={float(member.bulge.intensity):.3f}\"\n", + " )" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The simulator built every member with `intensity = luminosity` \u2014 a true shared normalization of 1.0. We can\n", + "check the fit recovered it by dividing any member's fitted intensity by its catalogue luminosity.\n", + "\n", + "(A real, full run recovers a value close to 1.0; if you are running this script in the fast test mode used\n", + "for automated checks, the search is bypassed and the printed value is a prior median, not meaningful.)" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "recovered_scale = float(instance.galaxies.member_0.bulge.intensity) / float(\n", + " member_luminosities[0]\n", + ")\n", + "\n", + "print(f\"Recovered member-tier intensity_scale = {recovered_scale:.3f} (truth: 1.0)\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Refinements__\n", + "\n", + "This tutorial used the simplest catalogue tier \u2014 fixed shapes, one shared normalization \u2014 which is the right\n", + "starting point. Real cluster analyses then reach for two refinements, both of which preserve the\n", + "row-append scaling:\n", + "\n", + "- **Free the tier's shape**: promote the shared `effective_radius` and `sersic_index` from fixed values to\n", + " parameters shared by the whole tier. The tier then costs 3 free parameters instead of 1 \u2014 still\n", + " independent of the member count.\n", + "\n", + "- **Promote bright members**: give the brightest few members their own free light models alongside the BCG,\n", + " since the data constrains them well. Promotion costs the full per-galaxy parameters, so promote sparingly\n", + " \u2014 brightest first, and only while the data keeps constraining them.\n", + "\n", + "Both are demonstrated in `autogalaxy_workspace/*/cluster/modeling.py`, which fits this same dataset.\n", + "\n", + "__Wrap Up__\n", + "\n", + "This tutorial completes the chapter's ladder of scale, and with it the **HowToGalaxy** lectures. Lets recap\n", + "the ladder one final time:\n", + "\n", + "- **One galaxy** (chapters 1-3): light profiles, non-linear searches, linear profiles and the MGE, and\n", + " pixelized reconstructions \u2014 the core toolkit, applied to a single galaxy at the centre of its image.\n", + "\n", + "- **Extra galaxies**: nearby galaxies whose light contaminates the target's, noise-scaled out of the fit or\n", + " included in the model so the target's photometry is unbiased.\n", + "\n", + "- **Multi-galaxy blends**: systems of overlapping galaxies modeled simultaneously, each with its own free\n", + " light model.\n", + "\n", + "- **Cluster fields** (this tutorial): a BCG modeled richly plus a member population too numerous for free\n", + " models, driven instead by a catalogue \u2014 fixed centres, luminosity-tied intensities, one shared free\n", + " normalization. Model complexity decoupled from population size.\n", + "\n", + "The theme of the chapter has been that scaling up is about *composition*, not new fitting machinery: the\n", + "same `af.Model` / `af.Collection` API, the same `AnalysisImaging`, the same searches \u2014 arranged so that the\n", + "information in the data decides which parameters are free.\n", + "\n", + "And that theme closes the series: you now know how to simulate and fit galaxy imaging, compose models from\n", + "simple Sersics to MGEs and pixelizations, run and interpret non-linear searches, and scale all of it from\n", + "one galaxy to a cluster field. Where to go next:\n", + "\n", + "- `autogalaxy_workspace`: the destination for real science \u2014 the `imaging`, `multi_galaxy` and `cluster`\n", + " packages mirror this chapter's ladder with production-ready scripts (including interferometer data,\n", + " multi-wavelength fitting and ellipse fitting, which the lectures did not cover), and its `guides` cover\n", + " every API in depth.\n", + "\n", + "- **HowToLens**: the companion lecture series for strong gravitational lensing, where galaxies like these\n", + " become lenses \u2014 its chapter 4 climbs this same ladder on the mass side, with the cluster rung fitting\n", + " point-source image positions driven by the very same catalogue machinery.\n", + "\n", + "Congratulations on finishing the **HowToGalaxy** lectures \u2014 now go model some galaxies!" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/chapter_optional/tutorial_searches.ipynb b/notebooks/chapter_optional/tutorial_searches.ipynb index ae73c21..7885d6e 100644 --- a/notebooks/chapter_optional/tutorial_searches.ipynb +++ b/notebooks/chapter_optional/tutorial_searches.ipynb @@ -11,8 +11,8 @@ "sampling. In this tutorial, we'll consider how we can change these setting to balance finding the global maxima\n", "solution with fast run time.\n", "\n", - "We will also discuss other types of non-linear searches, such as MCMC and optimizers, which we can use to perform lens\n", - "modeling. So far, we have no found any of these alternatives to give anywhere near as robust and efficient results as\n", + "We will also discuss other types of non-linear searches, such as MCMC and optimizers, which we can use to perform\n", + "modeling. So far, we have not found any of these alternatives to give anywhere near as robust and efficient results as\n", "Nautilus, and we recommend users use Nautilus unless they are particularly interested in investigating different\n", "model-fitting techniques.\n", "\n", @@ -66,7 +66,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -80,11 +80,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "we'll use new galaxying data, where:\n", + "we'll use the `simple__sersic` imaging data of a galaxy, where:\n", "\n", - " - The galaxy's light is an `Sersic`.\n", - " - The galaxy's total mass distribution is an `Isothermal` and `ExternalShear`.\n", - " - The source galaxy's `LightProfile` is an `Sersic`." + " - The galaxy's light is a `Sersic`." ] }, { @@ -135,7 +133,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "we'll create and use a smaller 2.0\" `Mask2D` again." + "we'll create and use a 2.6\" `Mask2D`." ] }, { @@ -178,11 +176,11 @@ "\n", "f_live:\n", "\n", - "A nested sampling algorithm estimates the *Bayesian Evidence* of the model-fit, which is quantity the non-linear \n", + "A nested sampling algorithm estimates the *Bayesian Evidence* of the model-fit, which is a quantity the non-linear \n", "search algorithms we introduce later do not. The Bayesian evidence quantifies how well the model as a whole fits\n", "the data, following a principle called Occam's Razor (`https://simple.wikipedia.org/wiki/Occam%27s_razor`). This \n", "penalizes models for being more complex (e.g. more parameters) and requires that their additional complexity improve \n", - "their overall fit to the data compared to a simpler model. By computing the comparing the Bayesian evidence of \n", + "their overall fit to the data compared to a simpler model. By comparing the Bayesian evidence of \n", "different models one can objectively choose the model that best fits the data.\n", "\n", "A nested sampling algorithm stops sampling when it estimates that continuing sampling will not increase the Bayesian \n", @@ -193,7 +191,7 @@ "A high `f_live` will make the errors estimated on every parameter unreliable and its value must be kept \n", "below 0.8 for reliable error estimates. However, when chaining searches, we typically *do not care* about the errors \n", "in the first search, therefore setting a high evidence tolerance can be an effective means to make Nautilus converge\n", - "faster (we'll estimate reliable errors in the second search when the `f_live is 0.8 or less). \n", + "faster (we'll estimate reliable errors in the second search when the `f_live` is 0.8 or less). \n", "\n", "\n", "Lets perform two fits, where:\n", @@ -211,10 +209,7 @@ "source": [ "model = af.Collection(\n", " galaxies=af.Collection(\n", - " lens=af.Model(\n", - " ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic, mass=ag.mp.Isothermal\n", - " ),\n", - " source=af.Model(ag.Galaxy, redshift=1.0, bulge=ag.lp.Sersic),\n", + " galaxy=af.Model(ag.Galaxy, redshift=0.5, bulge=ag.lp.Sersic),\n", " )\n", ")\n", "\n", @@ -223,7 +218,7 @@ " name=\"tutorial_searches_slow\",\n", " unique_tag=dataset_name,\n", " n_live=400,\n", - " n_batch=50, # GPU lens model fits are batched and run simultaneously, see VRAM section below.\n", + " n_batch=50, # GPU model fits are batched and run simultaneously (see chapter 2 tutorial 2).\n", ")\n", "\n", "analysis = ag.AnalysisImaging(dataset=dataset, use_jax=True)\n", @@ -231,7 +226,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the workspace/output\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_slow = search.fit(model=model, analysis=analysis)" @@ -284,17 +279,17 @@ "metadata": {}, "source": [ "search = af.Nautilus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_2\"),\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_optional\"),\n", " name=\"tutorial_searches_fast\",\n", " unique_tag=dataset_name,\n", " n_live=75,\n", - " n_batch=50, # GPU lens model fits are batched and run simultaneously, see VRAM section below.\n", + " n_batch=50, # GPU model fits are batched and run simultaneously (see chapter 2 tutorial 2).\n", ")\n", "\n", "print(\n", " \"The non-linear search has begun running - checkout the workspace/output\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_fast = search.fit(model=model, analysis=analysis)\n", @@ -379,43 +374,21 @@ "cell_type": "code", "metadata": {}, "source": [ - "lens_bulge = af.Model(ag.lp.Sersic)\n", - "lens_bulge.centre.centre_0 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1)\n", - "lens_bulge.centre.centre_1 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1)\n", - "lens_bulge.ell_comps.ell_comps_0 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3)\n", - "lens_bulge.ell_comps.ell_comps_1 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3)\n", - "lens_bulge.intensity = af.UniformPrior(lower_limit=0.5, upper_limit=1.5)\n", - "lens_bulge.effective_radius = af.UniformPrior(lower_limit=0.2, upper_limit=1.6)\n", - "lens_bulge.sersic_index = af.UniformPrior(lower_limit=3.0, upper_limit=5.0)\n", - "\n", - "\n", - "mass = af.Model(ag.mp.Isothermal)\n", - "mass.centre.centre_0 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1)\n", - "mass.centre.centre_1 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1)\n", - "mass.ell_comps.ell_comps_0 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3)\n", - "mass.ell_comps.ell_comps_1 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3)\n", - "mass.einstein_radius = af.UniformPrior(lower_limit=1.0, upper_limit=2.0)\n", - "\n", - "shear = af.Model(ag.mp.ExternalShear)\n", - "shear.gamma_1 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1)\n", - "shear.gamma_2 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1)\n", - "\n", "bulge = af.Model(ag.lp.Sersic)\n", "bulge.centre.centre_0 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1)\n", "bulge.centre.centre_1 = af.UniformPrior(lower_limit=-0.1, upper_limit=0.1)\n", "bulge.ell_comps.ell_comps_0 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3)\n", "bulge.ell_comps.ell_comps_1 = af.UniformPrior(lower_limit=-0.3, upper_limit=0.3)\n", - "bulge.intensity = af.UniformPrior(lower_limit=0.1, upper_limit=0.5)\n", - "bulge.effective_radius = af.UniformPrior(lower_limit=0.0, upper_limit=0.4)\n", - "bulge.sersic_index = af.UniformPrior(lower_limit=0.5, upper_limit=2.0)\n", + "bulge.intensity = af.UniformPrior(lower_limit=0.5, upper_limit=1.5)\n", + "bulge.effective_radius = af.UniformPrior(lower_limit=0.2, upper_limit=1.6)\n", + "bulge.sersic_index = af.UniformPrior(lower_limit=3.0, upper_limit=5.0)\n", "\n", - "lens = af.Model(ag.Galaxy, redshift=0.5, mass=mass, shear=shear)\n", - "source = af.Model(ag.Galaxy, redshift=1.0, bulge=bulge)\n", + "galaxy = af.Model(ag.Galaxy, redshift=0.5, bulge=bulge)\n", "\n", - "model = af.Collection(galaxies=af.Collection(lens=lens, source=source))\n", + "model = af.Collection(galaxies=af.Collection(galaxy=galaxy))\n", "\n", "search = af.Zeus(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_2\"),\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_optional\"),\n", " name=\"tutorial_searches_zeus\",\n", " unique_tag=dataset_name,\n", " nwalkers=50,\n", @@ -425,7 +398,7 @@ "print(\n", " \"Zeus has begun running - checkout the workspace/output\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_zeus = search.fit(model=model, analysis=analysis)\n", @@ -436,7 +409,7 @@ "\n", "\n", "search = af.Emcee(\n", - " path_prefix=Path(\"howtogalaxy\", \"chapter_2\"),\n", + " path_prefix=Path(\"howtogalaxy\", \"chapter_optional\"),\n", " name=\"tutorial_searches_emcee\",\n", " unique_tag=dataset_name,\n", " nwalkers=50,\n", @@ -446,7 +419,7 @@ "print(\n", " \"The non-linear search has begun running - checkout the workspace/output\"\n", " \" folder for live output of the results, images and model.\"\n", - " \" This Jupyter notebook cell with progress once search has completed - this could take some time!\"\n", + " \" This Jupyter notebook cell will progress once search has completed - this could take some time!\"\n", ")\n", "\n", "result_emcee = search.fit(model=model, analysis=analysis)\n", @@ -464,8 +437,8 @@ "source": [ "__Wrap Up__\n", "\n", - "This tutorial showed how the same model can be fitted using different non-linear searches (e.g. `Nautilus`, `Emcee`,\n", - "`PySwarms`). Each search explores parameter space in a different way, and the best choice depends on the\n", + "This tutorial showed how the same model can be fitted using different non-linear searches (e.g. `Nautilus`, `Zeus`,\n", + "`Emcee`). Each search explores parameter space in a different way, and the best choice depends on the\n", "dimensionality and complexity of the model you are fitting.\n", "\n", "For the vast majority of models in **PyAutoGalaxy**, the default nested sampling search `Nautilus` is recommended, as\n", diff --git a/notebooks/simulators/cluster.ipynb b/notebooks/simulators/cluster.ipynb new file mode 100644 index 0000000..732e329 --- /dev/null +++ b/notebooks/simulators/cluster.ipynb @@ -0,0 +1,462 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Simulator: Cluster\n", + "==================\n", + "\n", + "This script simulates `Imaging` of a cluster field: a brightest cluster galaxy (BCG) surrounded by ten\n", + "lower-luminosity member galaxies. It is used in chapter 4 of the **HowToGalaxy** lectures, where the member\n", + "population is modeled via a **catalogue**: a CSV of member centres and luminosities whose photometry pins\n", + "the faint galaxies while only a shared normalization stays free.\n", + "\n", + "This script simulates `Imaging` of a cluster field where:\n", + "\n", + " - The BCG's bulge is an elliptical `Sersic` (de Vaucouleurs-like).\n", + " - Ten member galaxies have `SersicSph` light profiles whose intensities follow their catalogue\n", + " luminosities.\n", + "\n", + "__Contents__\n", + "\n", + "- **Dataset Paths:** Set the output path for the simulated dataset.\n", + "- **Grid:** Create a 2D grid with adaptive over-sampling for simulation.\n", + "- **Galaxies:** Define the BCG and member galaxy light profiles used for simulation.\n", + "- **Output:** Save the simulated dataset to FITS files.\n", + "- **Member Catalogue CSV:** Write `scaling_galaxies.csv` (y, x, luminosity) \u2014 the tutorial's input.\n", + "- **Visualize:** Output subplot and image PNGs of the simulated dataset.\n", + "- **Plane Output:** Save the Galaxies object and BCG centre as JSON files.\n", + "\n", + "__Start Here Notebook__\n", + "\n", + "If any code in this script is unclear, refer to the `simulators/simple.ipynb` notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Google Colab Setup__\n", + "\n", + "This cell sets up the environment when the notebook is run on Google Colab: it installs the\n", + "required PyAuto packages, clones the workspace (configuration files and example datasets) and\n", + "points the configuration at it. If you are running the notebook elsewhere (e.g. locally via\n", + "your own installation) it does nothing, and you can run it safely.\n", + "\n", + "Colab tip: model-fits run much faster on a GPU \u2014 enable one via \"Runtime\" -> \"Change runtime\n", + "type\" -> \"Hardware accelerator\" before running the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import google.colab\n", + "except ImportError:\n", + " from autogalaxy import setup_colab as _setup_colab\n", + "else:\n", + " import importlib\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.check_call(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"autonerves\", \"--no-deps\"]\n", + " )\n", + " _setup_colab = importlib.import_module(\"autonerves.setup_colab\")\n", + "\n", + "_setup_colab.setup(\"howtogalaxy\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", + "\n", + "import csv\n", + "from pathlib import Path\n", + "import autogalaxy as ag\n", + "import autogalaxy.plot as aplt" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Dataset Paths__\n", + "\n", + "The `dataset_type` describes the type of data being simulated and `dataset_name` gives it a descriptive name." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset_type = \"cluster\"\n", + "dataset_name = \"simple\"\n", + "\n", + "dataset_path = Path(\"dataset\", dataset_type, dataset_name)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Grid__\n", + "\n", + "The cluster field spans a much wider area than the single-galaxy datasets of earlier chapters, so the grid\n", + "is 250 x 250 pixels (25\" x 25\" at 0.1\" per pixel).\n", + "\n", + "The BCG sits at the centre of the field and the ten member galaxies are scattered across it. Their centres\n", + "and luminosities are defined here because both the light profiles and the member catalogue CSV are built\n", + "from them below." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "grid = ag.Grid2D.uniform(\n", + " shape_native=(250, 250),\n", + " pixel_scales=0.1,\n", + ")\n", + "\n", + "bcg_centre = (0.0, 0.0)\n", + "\n", + "member_centres = [\n", + " (5.5, -6.5),\n", + " (-7.5, 3.0),\n", + " (3.0, 8.0),\n", + " (8.0, 5.0),\n", + " (-6.5, -8.0),\n", + " (-2.5, 6.5),\n", + " (7.0, -2.0),\n", + " (-7.0, 7.5),\n", + " (2.0, -8.5),\n", + " (-4.0, -3.5),\n", + "]\n", + "\n", + "member_luminosities = [0.40, 0.32, 0.25, 0.20, 0.16, 0.13, 0.10, 0.08, 0.06, 0.05]" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Simulate the image using a (y,x) grid with the adaptive over sampling scheme, centred on every galaxy in\n", + "the field (the BCG and all ten members)." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from(\n", + " grid=grid,\n", + " sub_size_list=[32, 8, 2],\n", + " radial_list=[0.3, 0.6],\n", + " centre_list=[bcg_centre] + member_centres,\n", + ")\n", + "\n", + "grid = grid.apply_over_sampling(over_sample_size=over_sample_size)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Simulate a simple Gaussian PSF for the image." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "psf = ag.Convolver.from_gaussian(\n", + " shape_native=(11, 11), sigma=0.1, pixel_scales=grid.pixel_scales\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the simulator for the imaging data, which defines the exposure time, background sky, noise levels and psf." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "simulator = ag.SimulatorImaging(\n", + " exposure_time=300.0,\n", + " psf=psf,\n", + " background_sky_level=0.1,\n", + " add_poisson_noise_to_data=True,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Galaxies__\n", + "\n", + "Setup the BCG: a bright, extended de Vaucouleurs-like Sersic at the cluster centre. In the chapter 4\n", + "cluster tutorial it is the one galaxy modeled individually, with a free MGE." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "bcg = ag.Galaxy(\n", + " redshift=0.5,\n", + " bulge=ag.lp.Sersic(\n", + " centre=bcg_centre,\n", + " ell_comps=ag.convert.ell_comps_from(axis_ratio=0.8, angle=45.0),\n", + " intensity=1.5,\n", + " effective_radius=2.5,\n", + " sersic_index=4.0,\n", + " ),\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setup the ten member galaxies, whose central intensities equal their catalogue luminosities \u2014 so the\n", + "rendered image visibly traces the catalogue, and the tutorial's shared-normalization tier\n", + "(intensity = scale * luminosity) can recover the truth with `scale = 1`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "members = []\n", + "for centre, luminosity in zip(member_centres, member_luminosities):\n", + " members.append(\n", + " ag.Galaxy(\n", + " redshift=0.5,\n", + " bulge=ag.lp.SersicSph(\n", + " centre=centre,\n", + " intensity=luminosity,\n", + " effective_radius=0.6,\n", + " sersic_index=3.0,\n", + " ),\n", + " )\n", + " )" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use these galaxies to generate the image for the simulated `Imaging` dataset." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxies = ag.Galaxies(galaxies=[bcg] + members)\n", + "aplt.plot_array(array=galaxies.image_2d_from(grid=grid), title=\"Image\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pass the simulator galaxies, which creates the image which is simulated as an imaging dataset." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset = simulator.via_galaxies_from(galaxies=galaxies, grid=grid)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot the simulated `Imaging` dataset before outputting it to fits." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Output__\n", + "\n", + "Output the simulated dataset to the dataset path as .fits files." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.fits_imaging(\n", + " dataset=dataset,\n", + " data_path=dataset_path / \"data.fits\",\n", + " psf_path=dataset_path / \"psf.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " overwrite=True,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Member Catalogue CSV__\n", + "\n", + "Write the member catalogue to `scaling_galaxies.csv` with the three-column `y, x, luminosity` schema used\n", + "by the `autogalaxy_workspace` cluster package (and shared with the lensing workspace's cluster package,\n", + "where the same catalogue drives member MASSES via a scaling relation; here it drives member LIGHT). The\n", + "tutorial loads it with `ag.galaxy_table_from_csv` \u2014 the catalogue-loading API that makes the member\n", + "population a row-append away from scaling up." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "with open(dataset_path / \"scaling_galaxies.csv\", \"w\", newline=\"\") as f:\n", + " writer = csv.writer(f, lineterminator=\"\\n\")\n", + " writer.writerow([\"y\", \"x\", \"luminosity\"])\n", + " for centre, luminosity in zip(member_centres, member_luminosities):\n", + " writer.writerow([centre[0], centre[1], luminosity])" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Visualize__\n", + "\n", + "Output a subplot of the simulated dataset, the image and the galaxies quantities to the dataset path as .png files." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(\n", + " dataset=dataset, output_path=dataset_path, output_format=\"png\"\n", + ")\n", + "aplt.plot_array(\n", + " array=dataset.data, title=\"Data\", output_path=dataset_path, output_format=\"png\"\n", + ")\n", + "\n", + "aplt.subplot_galaxies(\n", + " galaxies=galaxies, grid=grid, output_path=dataset_path, output_format=\"png\"\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Plane Output__\n", + "\n", + "Save the `Galaxies` in the dataset folder as a .json file, ensuring the true light profiles and galaxies\n", + "are safely stored and available to check how the dataset was simulated in the future.\n", + "\n", + "This can be loaded via the method `galaxies = ag.from_json()`.\n", + "\n", + "The BCG centre is also saved as a .json file, mirroring the real-cluster workflow where the BCG centre(s)\n", + "are recorded separately from the member catalogue." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "ag.output_to_json(\n", + " obj=galaxies,\n", + " file_path=Path(dataset_path, \"galaxies.json\"),\n", + ")\n", + "\n", + "ag.output_to_json(\n", + " obj=ag.Grid2DIrregular([bcg_centre]),\n", + " file_path=Path(dataset_path, \"bcg_centres.json\"),\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The dataset can be viewed in the folder `dataset/cluster/simple`." + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/simulators/extra_galaxy.ipynb b/notebooks/simulators/extra_galaxy.ipynb new file mode 100644 index 0000000..5e6b0db --- /dev/null +++ b/notebooks/simulators/extra_galaxy.ipynb @@ -0,0 +1,471 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Simulator: Extra Galaxy\n", + "=======================\n", + "\n", + "This script simulates `Imaging` of a galaxy using light profiles where:\n", + "\n", + " - The main galaxy's bulge is an `Sersic`.\n", + " - There is one extra galaxy nearby, whose light is an `ExponentialSph`, located a few arc-seconds\n", + " from the main galaxy.\n", + "\n", + "This dataset is used in chapter 4 tutorial 1, which shows the two approaches to dealing with an\n", + "extra galaxy whose light contaminates the fit: noise scaling its emission out of the data, or\n", + "including it in the model via the extra galaxies API.\n", + "\n", + "__Contents__\n", + "\n", + "- **Dataset Paths:** Set the output path for the simulated dataset.\n", + "- **Grid:** Create a 2D grid with adaptive over-sampling for simulation.\n", + "- **Galaxies:** Define the main galaxy and the nearby extra galaxy used for simulation.\n", + "- **Output:** Save the simulated dataset to FITS files.\n", + "- **Visualize:** Output subplot and image PNGs of the simulated dataset.\n", + "- **Mask Extra Galaxies:** Build and save `mask_extra_galaxies.fits` covering the extra galaxy's light.\n", + "- **Plane Output:** Save the Galaxies object as a JSON file.\n", + "- **Extra Galaxies Centres:** Save the extra galaxy's (y,x) centre as a JSON file.\n", + "\n", + "__Start Here Notebook__\n", + "\n", + "If any code in this script is unclear, refer to the `simulators/simple.ipynb` notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Google Colab Setup__\n", + "\n", + "This cell sets up the environment when the notebook is run on Google Colab: it installs the\n", + "required PyAuto packages, clones the workspace (configuration files and example datasets) and\n", + "points the configuration at it. If you are running the notebook elsewhere (e.g. locally via\n", + "your own installation) it does nothing, and you can run it safely.\n", + "\n", + "Colab tip: model-fits run much faster on a GPU \u2014 enable one via \"Runtime\" -> \"Change runtime\n", + "type\" -> \"Hardware accelerator\" before running the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import google.colab\n", + "except ImportError:\n", + " from autogalaxy import setup_colab as _setup_colab\n", + "else:\n", + " import importlib\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.check_call(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"autonerves\", \"--no-deps\"]\n", + " )\n", + " _setup_colab = importlib.import_module(\"autonerves.setup_colab\")\n", + "\n", + "_setup_colab.setup(\"howtogalaxy\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", + "\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "\n", + "import autogalaxy as ag\n", + "import autogalaxy.plot as aplt" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Dataset Paths__\n", + "\n", + "The `dataset_type` describes the type of data being simulated and `dataset_name` gives it a descriptive name." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset_type = \"imaging\"\n", + "dataset_name = \"extra_galaxy\"\n", + "\n", + "dataset_path = Path(\"dataset\", dataset_type, dataset_name)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Grid__\n", + "\n", + "Simulate the image using a (y,x) grid with the adaptive over sampling scheme.\n", + "\n", + "The grid is larger than the 100 x 100 pixels used by other simulators in this folder, so that the extra galaxy,\n", + "which is offset from the main galaxy centre of (0.0\", 0.0\"), is comfortably contained in the image.\n", + "\n", + "The adaptive over sampling scheme is centred on both the main galaxy and the extra galaxy, ensuring the light of\n", + "both is over sampled accurately." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "grid = ag.Grid2D.uniform(\n", + " shape_native=(150, 150),\n", + " pixel_scales=0.1,\n", + ")\n", + "\n", + "extra_galaxy_centre = (1.0, 3.5)\n", + "\n", + "over_sample_size = ag.util.over_sample.over_sample_size_via_radial_bins_from(\n", + " grid=grid,\n", + " sub_size_list=[32, 8, 2],\n", + " radial_list=[0.3, 0.6],\n", + " centre_list=[(0.0, 0.0), extra_galaxy_centre],\n", + ")\n", + "\n", + "grid = grid.apply_over_sampling(over_sample_size=over_sample_size)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Simulate a simple Gaussian PSF for the image." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "psf = ag.Convolver.from_gaussian(\n", + " shape_native=(11, 11), sigma=0.1, pixel_scales=grid.pixel_scales\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the simulator for the imaging data, which defines the exposure time, background sky, noise levels and psf." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "simulator = ag.SimulatorImaging(\n", + " exposure_time=300.0,\n", + " psf=psf,\n", + " background_sky_level=0.1,\n", + " add_poisson_noise_to_data=True,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Galaxies__\n", + "\n", + "Setup the main galaxy with a bulge (elliptical Sersic) for this simulation." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxy = ag.Galaxy(\n", + " redshift=0.5,\n", + " bulge=ag.lp.Sersic(\n", + " centre=(0.0, 0.0),\n", + " ell_comps=ag.convert.ell_comps_from(axis_ratio=0.9, angle=45.0),\n", + " intensity=1.0,\n", + " effective_radius=0.8,\n", + " sersic_index=4.0,\n", + " ),\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setup the extra galaxy, a smaller galaxy whose light is a spherical Exponential profile, offset a few arc-seconds\n", + "from the main galaxy.\n", + "\n", + "Note that its redshift is the same as the main galaxy, which is not necessarily the case in real observations.\n", + "If it is at a different redshift, the tools for masking or modeling extra galaxies are equipped to handle this." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "extra_galaxy = ag.Galaxy(\n", + " redshift=0.5,\n", + " light=ag.lp.ExponentialSph(\n", + " centre=extra_galaxy_centre, intensity=2.0, effective_radius=0.5\n", + " ),\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use these galaxies to generate the image for the simulated `Imaging` dataset." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxies = ag.Galaxies(galaxies=[galaxy, extra_galaxy])\n", + "aplt.plot_array(array=galaxies.image_2d_from(grid=grid), title=\"Image\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pass the simulator galaxies, which creates the image which is simulated as an imaging dataset." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset = simulator.via_galaxies_from(galaxies=galaxies, grid=grid)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot the simulated `Imaging` dataset before outputting it to fits." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Output__\n", + "\n", + "Output the simulated dataset to the dataset path as .fits files." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.fits_imaging(\n", + " dataset=dataset,\n", + " data_path=dataset_path / \"data.fits\",\n", + " psf_path=dataset_path / \"psf.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " overwrite=True,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Visualize__\n", + "\n", + "Output a subplot of the simulated dataset, the image and the galaxies quantities to the dataset path as .png files." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_imaging_dataset(\n", + " dataset=dataset, output_path=dataset_path, output_format=\"png\"\n", + ")\n", + "aplt.plot_array(\n", + " array=dataset.data, title=\"Data\", output_path=dataset_path, output_format=\"png\"\n", + ")\n", + "\n", + "aplt.subplot_galaxies(\n", + " galaxies=galaxies, grid=grid, output_path=dataset_path, output_format=\"png\"\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Mask Extra Galaxies__\n", + "\n", + "Build and output a `mask_extra_galaxies.fits` covering the extra galaxy's emission, so that the chapter 4\n", + "tutorial which uses this dataset can load the mask directly without a separate data-preparation step. For real\n", + "data, a user would create this mask themselves by inspecting the image (the `autogalaxy_workspace`'s\n", + "`data_preparation` package includes a GUI for drawing it).\n", + "\n", + "The circle is sized to ~3x the extra galaxy's `effective_radius`, which comfortably covers the light extent of\n", + "the `ExponentialSph` profile used above. The geometry is derived from the same centre + radius defined for the\n", + "extra galaxy in this script, so it stays in sync with any future tweak to those values.\n", + "\n", + "`Mask2D.circular` honours the `PYAUTO_SMALL_DATASETS=1` env var, so the mask automatically shrinks alongside the\n", + "small-dataset image and never raises an out-of-bounds error." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "extra_galaxies_mask = np.zeros(dataset.shape_native, dtype=bool)\n", + "\n", + "for centre, radius in [\n", + " (extra_galaxy_centre, 3.0 * 0.5),\n", + "]:\n", + " circle = ag.Mask2D.circular(\n", + " shape_native=dataset.shape_native,\n", + " pixel_scales=dataset.pixel_scales,\n", + " centre=centre,\n", + " radius=radius,\n", + " invert=True, # True inside the circle (i.e. masked region)\n", + " )\n", + " extra_galaxies_mask = np.logical_or(extra_galaxies_mask, circle.native)\n", + "\n", + "mask_extra_galaxies = ag.Mask2D(\n", + " mask=extra_galaxies_mask,\n", + " pixel_scales=dataset.pixel_scales,\n", + ")\n", + "\n", + "aplt.fits_array(\n", + " array=mask_extra_galaxies,\n", + " file_path=dataset_path / \"mask_extra_galaxies.fits\",\n", + " overwrite=True,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Plane Output__\n", + "\n", + "Save the `Galaxies` in the dataset folder as a .json file, ensuring the true light profiles and galaxies\n", + "are safely stored and available to check how the dataset was simulated in the future.\n", + "\n", + "This can be loaded via the method `galaxies = ag.from_json()`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "ag.output_to_json(\n", + " obj=galaxies,\n", + " file_path=Path(dataset_path, \"galaxies.json\"),\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Extra Galaxies Centres__\n", + "\n", + "Save the (y,x) centre of the extra galaxy as a `Grid2DIrregular` JSON file. The chapter 4 tutorial loads this\n", + "file to fix the extra galaxy's light profile centre when composing the model. For real data, a user would mark\n", + "these centres themselves on the image (the `autogalaxy_workspace`'s `data_preparation` package shows how)." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "extra_galaxies_centres = ag.Grid2DIrregular(values=[extra_galaxy_centre])\n", + "\n", + "ag.output_to_json(\n", + " obj=extra_galaxies_centres,\n", + " file_path=Path(dataset_path, \"extra_galaxies_centres.json\"),\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The dataset can be viewed in the folder `dataset/imaging/extra_galaxy`." + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/simulators/interferometer.ipynb b/notebooks/simulators/interferometer.ipynb new file mode 100644 index 0000000..5f06ed4 --- /dev/null +++ b/notebooks/simulators/interferometer.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Simulator: Interferometer\n", + "=========================\n", + "\n", + "This script simulates `Interferometer` data of a galaxy, as would be observed by a radio or sub-mm\n", + "interferometer like ALMA or the JVLA, where:\n", + "\n", + " - The galaxy's bulge is an `Sersic`.\n", + "\n", + "Unlike CCD imaging, an interferometer does not observe an image of the galaxy. It measures \"visibilities\", which\n", + "are the Fourier transform of the sky brightness sampled at a set of points in the \"uv-plane\", where each point\n", + "corresponds to the separation of a pair of antennas in the array.\n", + "\n", + "It is used to illustrate interferometer data in the HowToGalaxy lecture series. HowToGalaxy does not cover\n", + "interferometry beyond this glimpse; the `autogalaxy_workspace/scripts/interferometer` package is the dedicated\n", + "resource for uv-plane galaxy modeling.\n", + "\n", + "__Contents__\n", + "\n", + "- **Dataset Paths:** Set the output path for the simulated dataset.\n", + "- **Simulate:** Simulate the image using a (y,x) real-space grid and a synthetic set of uv-plane baselines.\n", + "- **Galaxies:** Define the galaxy Sersic light profile used for simulation.\n", + "- **Output:** Save the simulated dataset to FITS files.\n", + "- **Visualize:** Output subplot and dirty-image PNGs of the simulated dataset.\n", + "- **Plane Output:** Save the Galaxies object as a JSON file.\n", + "\n", + "__Start Here Notebook__\n", + "\n", + "If any code in this script is unclear, refer to the `autogalaxy_workspace/*/interferometer/simulator.ipynb`\n", + "notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Google Colab Setup__\n", + "\n", + "This cell sets up the environment when the notebook is run on Google Colab: it installs the\n", + "required PyAuto packages, clones the workspace (configuration files and example datasets) and\n", + "points the configuration at it. If you are running the notebook elsewhere (e.g. locally via\n", + "your own installation) it does nothing, and you can run it safely.\n", + "\n", + "Colab tip: model-fits run much faster on a GPU \u2014 enable one via \"Runtime\" -> \"Change runtime\n", + "type\" -> \"Hardware accelerator\" before running the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " import google.colab\n", + "except ImportError:\n", + " from autogalaxy import setup_colab as _setup_colab\n", + "else:\n", + " import importlib\n", + " import subprocess\n", + " import sys\n", + "\n", + " subprocess.check_call(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"autonerves\", \"--no-deps\"]\n", + " )\n", + " _setup_colab = importlib.import_module(\"autonerves.setup_colab\")\n", + "\n", + "_setup_colab.setup(\"howtogalaxy\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", + "\n", + "from pathlib import Path\n", + "import numpy as np\n", + "import autogalaxy as ag\n", + "import autogalaxy.plot as aplt" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Dataset Paths__\n", + "\n", + "The `dataset_type` describes the type of data being simulated and `dataset_name` gives it a descriptive name." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset_type = \"interferometer\"\n", + "dataset_name = \"simple\"\n", + "\n", + "dataset_path = Path(\"dataset\", dataset_type, dataset_name)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Simulate__\n", + "\n", + "For interferometer data, the galaxy's image is evaluated in real space on a (y,x) grid and then Fourier\n", + "transformed to the uv-plane, where it is compared to the observed visibilities.\n", + "\n", + "Interferometers do not observe galaxies in a way where over sampling is necessary, therefore the grid uses no\n", + "over sampling." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "grid = ag.Grid2D.uniform(\n", + " shape_native=(100, 100),\n", + " pixel_scales=0.1,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To perform the Fourier transform we need the wavelengths of the baselines, which describe where in the uv-plane\n", + "each visibility samples the sky's Fourier transform.\n", + "\n", + "For real data these are determined by the layout of the interferometer's antennas. The `autogalaxy_workspace`\n", + "bundles baselines of real instruments (e.g. the Square Mile Array (SMA) and ALMA). For this tutorial dataset we\n", + "instead draw a small synthetic set of baselines from a Gaussian distribution in the uv-plane, with a scale\n", + "comparable to the SMA's (a few hundred visibilities), keeping the simulation fast and self-contained." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "rng = np.random.default_rng(1)\n", + "\n", + "total_visibilities = 200\n", + "\n", + "uv_wavelengths = rng.normal(loc=0.0, scale=1.0e5, size=(total_visibilities, 2))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To simulate the interferometer dataset we first create a simulator, which defines the exposure time, noise levels\n", + "and Fourier transform method used in the simulation.\n", + "\n", + "We use the `TransformerDFT`, an exact Discrete Fourier Transform which is fast for datasets with a low number of\n", + "visibilities like this one." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "simulator = ag.SimulatorInterferometer(\n", + " uv_wavelengths=uv_wavelengths,\n", + " exposure_time=300.0,\n", + " noise_sigma=1000.0,\n", + " transformer_class=ag.TransformerDFT,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Galaxies__\n", + "\n", + "Setup the galaxy with a bulge (elliptical Sersic) for this simulation." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxy = ag.Galaxy(\n", + " redshift=0.5,\n", + " bulge=ag.lp.Sersic(\n", + " centre=(0.0, 0.0),\n", + " ell_comps=ag.convert.ell_comps_from(axis_ratio=0.9, angle=45.0),\n", + " intensity=1.0,\n", + " effective_radius=0.8,\n", + " sersic_index=4.0,\n", + " ),\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use these galaxies to generate the image for the simulated `Interferometer` dataset." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "galaxies = ag.Galaxies(galaxies=[galaxy])\n", + "aplt.plot_array(array=galaxies.image_2d_from(grid=grid), title=\"Image\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pass the simulator galaxies, which creates the image plotted above and simulates it as an interferometer dataset." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "dataset = simulator.via_galaxies_from(galaxies=galaxies, grid=grid)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot the simulated interferometer dataset's dirty images before outputting it to fits." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_interferometer_dirty_images(dataset=dataset)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Output__\n", + "\n", + "Output the simulated dataset to the dataset path as .fits files." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.fits_interferometer(\n", + " dataset=dataset,\n", + " data_path=dataset_path / \"data.fits\",\n", + " noise_map_path=dataset_path / \"noise_map.fits\",\n", + " uv_wavelengths_path=dataset_path / \"uv_wavelengths.fits\",\n", + " overwrite=True,\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Visualize__\n", + "\n", + "Output a subplot of the simulated dataset's dirty images and the galaxies quantities to the dataset path as\n", + ".png files." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "aplt.subplot_interferometer_dirty_images(\n", + " dataset=dataset, output_path=dataset_path, output_format=\"png\"\n", + ")\n", + "\n", + "aplt.subplot_galaxies(\n", + " galaxies=galaxies, grid=grid, output_path=dataset_path, output_format=\"png\"\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__Plane Output__\n", + "\n", + "Save the `Galaxies` in the dataset folder as a .json file, ensuring the true light profiles and galaxies\n", + "are safely stored and available to check how the dataset was simulated in the future.\n", + "\n", + "This can be loaded via the method `galaxies = ag.from_json()`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "ag.output_to_json(\n", + " obj=galaxies,\n", + " file_path=Path(dataset_path, \"galaxies.json\"),\n", + ")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The dataset can be viewed in the folder `dataset/interferometer/simple`." + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/simulators/sersic.ipynb b/notebooks/simulators/sersic.ipynb index 019cdf6..a6a4b37 100644 --- a/notebooks/simulators/sersic.ipynb +++ b/notebooks/simulators/sersic.ipynb @@ -68,7 +68,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -322,7 +322,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The dataset can be viewed in the folder `autogalaxy_workspace/dataset/imaging/simple__sersic`." + "The dataset can be viewed in the folder `dataset/imaging/simple__sersic`." ] } ], diff --git a/notebooks/simulators/sersic_x2.ipynb b/notebooks/simulators/sersic_x2.ipynb index 6c347dc..b77aa77 100644 --- a/notebooks/simulators/sersic_x2.ipynb +++ b/notebooks/simulators/sersic_x2.ipynb @@ -12,7 +12,7 @@ " - The first galaxy's bulge is an `Sersic`.\n", " - The second galaxy's bulge is an `Sersic`.\n", "\n", - "This dataset is used in chapter 3 of the **HowToGalaxy** lectures.\n", + "This dataset is used in chapter 4 of the **HowToGalaxy** lectures.\n", "\n", "__Contents__\n", "\n", @@ -23,11 +23,6 @@ "- **Visualize:** Output subplot and image PNGs of the simulated dataset.\n", "- **Plane Output:** Save the Galaxies object as a JSON file.\n", "\n", - "__Advanced__\n", - "\n", - "This is an advanced simulator script, meaning that detailed explanations of certain code are omitted. Refer to\n", - "simulators not in the `advanced` folder for more detailed comments.\n", - "\n", "__Start Here Notebook__\n", "\n", "If any code in this script is unclear, refer to the `simulators/simple.ipynb` notebook." @@ -76,7 +71,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -330,7 +325,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The dataset can be viewed in the folder `autogalaxy_workspace/dataset/imaging/sersic_x2`." + "The dataset can be viewed in the folder `dataset/imaging/sersic_x2`." ] } ], diff --git a/notebooks/simulators/simple.ipynb b/notebooks/simulators/simple.ipynb index 1e5c6b6..b9886a2 100644 --- a/notebooks/simulators/simple.ipynb +++ b/notebooks/simulators/simple.ipynb @@ -66,7 +66,7 @@ "metadata": {}, "source": [ "\n", - "# from autogalaxy import setup_notebook; setup_notebook()\n", + "from autogalaxy import setup_notebook; setup_notebook()\n", "\n", "from pathlib import Path\n", "import autogalaxy as ag\n", @@ -83,9 +83,9 @@ "\n", "The `dataset_type` describes the type of data being simulated and `dataset_name` gives it a descriptive name. They define the folder the dataset is output to on your hard-disk:\n", "\n", - " - The image will be output to `/autogalaxy_workspace/dataset/dataset_type/dataset_name/image.fits`.\n", - " - The noise-map will be output to `/autogalaxy_workspace/dataset/dataset_type/dataset_name/noise_map.fits`.\n", - " - The psf will be output to `/autogalaxy_workspace/dataset/dataset_type/dataset_name/psf.fits`." + " - The image will be output to `/dataset/dataset_type/dataset_name/image.fits`.\n", + " - The noise-map will be output to `/dataset/dataset_type/dataset_name/noise_map.fits`.\n", + " - The psf will be output to `/dataset/dataset_type/dataset_name/psf.fits`." ] }, { @@ -104,7 +104,7 @@ "source": [ "The path where the dataset will be output.\n", "\n", - "In this example, this is: `/autogalaxy_workspace/dataset/imaging/simple`" + "In this example, this is: `/dataset/imaging/simple`" ] }, { @@ -382,7 +382,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The dataset can be viewed in the folder `autogalaxy_workspace/dataset/imaging/simple`.\n", + "The dataset can be viewed in the folder `dataset/imaging/simple`.\n", "\n", "__JAX Variant__\n", "\n", diff --git a/workspace_index.json b/workspace_index.json index c9006e1..24542e6 100644 --- a/workspace_index.json +++ b/workspace_index.json @@ -19,8 +19,8 @@ "Grids", "Geometry", "Light Profiles", - "Galaxies", "One Dimension Projection", + "Galaxies", "Unit Conversion", "Wrap Up" ], @@ -40,9 +40,12 @@ "Background Sky", "Simulator", "Output", + "Interferometer Data", "Wrap Up" ], - "cross_refs": [], + "cross_refs": [ + "scripts/simulators/interferometer.py" + ], "notebook": "notebooks/chapter_1_introduction/tutorial_2_data.ipynb", "path": "scripts/chapter_1_introduction/tutorial_2_data.py", "summary": "In the previous tutorial, we used light profiles to create images of galaxies. However, those images don't accurately represent what we would observe through a telescope.", @@ -89,7 +92,28 @@ "notebook": "notebooks/chapter_1_introduction/tutorial_5_summary.ipynb", "path": "scripts/chapter_1_introduction/tutorial_5_summary.py", "summary": "In this chapter, we have learnt that:", - "title": "Tutorial 9: Summary" + "title": "Tutorial 5: Summary" + }, + { + "contents": [ + "Initial Setup", + "Dataset Auto-Simulation", + "Model", + "Search", + "Result (Search 1)", + "Prior Passing", + "Result", + "Wrap Up", + "Detailed Explanation Of Prior Passing", + "EXAMPLE" + ], + "cross_refs": [ + "autogalaxy_workspace/scripts/guides/modeling/chaining.py" + ], + "notebook": "notebooks/chapter_2_modeling/tutorial_10_prior_passing.ipynb", + "path": "scripts/chapter_2_modeling/tutorial_10_prior_passing.py", + "summary": "In the previous tutorial, we used non-linear search chaining to break the model-fitting procedure down into two non-linear searches. This used an initial search to fit a simple model, whose results were used to tune and initialize the priors of a more complex model that was fitted by the second search.", + "title": "Tutorial 10: Prior Passing" }, { "contents": [ @@ -199,7 +223,7 @@ ], "notebook": "notebooks/chapter_2_modeling/tutorial_6_masking.ipynb", "path": "scripts/chapter_2_modeling/tutorial_6_masking.py", - "summary": "We have learnt everything we need to know about non-linear searches to model a galaxy and infer a good lens model solution. Now, lets consider masking in more detail, something we have not given much consideration previously. We'll also learn a neat trick to improve the speed and accuracy of a non-linear search.", + "summary": "We have learnt everything we need to know about non-linear searches to model a galaxy and infer a good model solution. Now, lets consider masking in more detail, something we have not given much consideration previously. We'll also learn a neat trick to improve the speed and accuracy of a non-linear search.", "title": "Tutorial 6: Masking" }, { @@ -214,19 +238,20 @@ "cross_refs": [], "notebook": "notebooks/chapter_2_modeling/tutorial_7_results.ipynb", "path": "scripts/chapter_2_modeling/tutorial_7_results.py", - "summary": "In the previous tutorials, each search returned a `Result` object, which we used to plot the maximum log likelihood fit each model-fit. In this tutorial, we'll take a look at the result object in a little more detail.", - "title": "Tutorial 5: Results" + "summary": "In the previous tutorials, each search returned a `Result` object, which we used to plot the maximum log likelihood fit of each model-fit. In this tutorial, we'll take a look at the result object in a little more detail.", + "title": "Tutorial 7: Results" }, { "contents": [ "Searching Non-linear Parameter Space", - "Algorithmic Optimization", + "Cost Per Evaluation", + "JAX", + "Run Time Estimation", "Data Quantity", "Wrap Up" ], "cross_refs": [ - "//numba.py", - "chapter_optional/tutorial_searches.ipynb" + "chapter_optional/tutorial_searches.py" ], "notebook": "notebooks/chapter_2_modeling/tutorial_8_need_for_speed.ipynb", "path": "scripts/chapter_2_modeling/tutorial_8_need_for_speed.py", @@ -236,6 +261,7 @@ { "contents": [ "Initial Setup", + "Dataset Auto-Simulation", "Model", "Search + Analysis", "Result", @@ -244,48 +270,10 @@ "Wrap Up" ], "cross_refs": [], - "notebook": "notebooks/chapter_3_search_chaining/tutorial_1_search_chaining.ipynb", - "path": "scripts/chapter_3_search_chaining/tutorial_1_search_chaining.py", - "summary": "In chapter 2, we learnt how to perform modeling using a non-linear search. In all of the tutorials, we fitted the data using just one non-linear search. In this chapter, we introduce a technique called 'non-linear search chaining', which fits a model using a sequence of non-linear searches. The initial searches fit simpler models whose parameter spaces can be more accurately and efficiently sampled. The results of this search are then passed to later searches which fit models of gradually increasing complexity.", - "title": "Tutorial 1: Search Chaining" - }, - { - "contents": [ - "Initial Setup", - "Model", - "Search", - "Result (Search 1)", - "Prior Passing", - "Result", - "Wrap Up", - "Detailed Explanation Of Prior Passing", - "EXAMPLE" - ], - "cross_refs": [], - "notebook": "notebooks/chapter_3_search_chaining/tutorial_2_prior_passing.ipynb", - "path": "scripts/chapter_3_search_chaining/tutorial_2_prior_passing.py", - "summary": "In the previous tutorial, we used non-linear search chaining to break the model-fitting procedure down into two non-linear searches. This used an initial search to fit a simple model, whose results were used to tune and initialize the priors of a more complex model that was fitted by the second search.", - "title": "Tutorial 2: Prior Passing" - }, - { - "contents": [ - "Initial Setup", - "Paths", - "Search Chaining Approach", - "Model + Search + Analysis + Model-Fit (Search 1)", - "Result (Search 1)", - "Model (Search 2)", - "Search + Analysis + Model-Fit (Search 2)", - "Result (Search 2)", - "Model + Search + Analysis + Model-Fit (Search 4)", - "Result (Search 3)", - "Wrap Up" - ], - "cross_refs": [], - "notebook": "notebooks/chapter_3_search_chaining/tutorial_3_x2_galaxies.ipynb", - "path": "scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py", - "summary": "Up to now, all the images we've fitted had one galaxy. However, we saw in chapter 1 that our galaxies object can consist of multiple galaxies which each contribute to the overall emission. Multi-galaxy systems are challenging to model, because they add an extra 5-10 parameters to the non-linear search per galaxy and, more problematically, the degeneracies between the parameters of the light profiles of the galaxies can be severe.", - "title": "Tutorial 3: Two Galaxies" + "notebook": "notebooks/chapter_2_modeling/tutorial_9_search_chaining.ipynb", + "path": "scripts/chapter_2_modeling/tutorial_9_search_chaining.py", + "summary": "Throughout this chapter, we have fitted the data using just one non-linear search. The previous tutorial discussed the run-time cost of doing this: with a single search, the dimensionality of the model, the priors on its parameters and the search settings must all be juggled at once, leaving us little flexibility to trade them off against one another.", + "title": "Tutorial 9: Search Chaining" }, { "contents": [ @@ -294,59 +282,101 @@ "Wrap Up" ], "cross_refs": [], - "notebook": "notebooks/chapter_4_pixelizations/tutorial_1_pixelizations.ipynb", - "path": "scripts/chapter_4_pixelizations/tutorial_1_pixelizations.py", + "notebook": "notebooks/chapter_3_pixelizations/tutorial_1_pixelizations.ipynb", + "path": "scripts/chapter_3_pixelizations/tutorial_1_pixelizations.py", "summary": "In the previous chapters, we used light profiles to model the light of a galaxy, where the light profile was an analytic description of how the luminosity varies as a function of radius.", - "title": "Tutorial 1: pixelizations" + "title": "Tutorial 1: Pixelizations" }, { "contents": [ "Initial Setup", + "Dataset Auto-Simulation", "Mappers", "Mask", "Wrap Up" ], "cross_refs": [], - "notebook": "notebooks/chapter_4_pixelizations/tutorial_2_mappers.ipynb", - "path": "scripts/chapter_4_pixelizations/tutorial_2_mappers.py", - "summary": "In the previous tutorial, we used a pixelization to create made a `Mapper`. However, it was not clear what a `Mapper` does, why it was called a mapper and whether it was mapping anything at all!", + "notebook": "notebooks/chapter_3_pixelizations/tutorial_2_mappers.ipynb", + "path": "scripts/chapter_3_pixelizations/tutorial_2_mappers.py", + "summary": "In the previous tutorial, we used a pixelization to create a `Mapper`. However, it was not clear what a `Mapper` does, why it was called a mapper and whether it was mapping anything at all!", "title": "Tutorial 2: Mappers" }, { "contents": [ "Initial Setup", + "Dataset Auto-Simulation", "Pixelization", "Positive Only Solver", - "Wrap Up", - "Detailed Explanation" + "Detailed Explanation", + "Wrap Up" ], "cross_refs": [ "/imaging/features/pixelization/likelihood_function.ipynb" ], - "notebook": "notebooks/chapter_4_pixelizations/tutorial_3_inversions.ipynb", - "path": "scripts/chapter_4_pixelizations/tutorial_3_inversions.py", + "notebook": "notebooks/chapter_3_pixelizations/tutorial_3_inversions.ipynb", + "path": "scripts/chapter_3_pixelizations/tutorial_3_inversions.py", "summary": "In the previous two tutorials, we introduced:", "title": "Tutorial 3: Inversions" }, { "contents": [ "Initial Setup", + "Dataset Auto-Simulation", "Convenience Function", "Pixelization", "Regularization", "Bayesian Evidence", "Non-Linear and Linear", - "Detailed Description" + "Detailed Description", + "Wrap Up" ], "cross_refs": [], - "notebook": "notebooks/chapter_4_pixelizations/tutorial_4_bayesian_regularization.ipynb", - "path": "scripts/chapter_4_pixelizations/tutorial_4_bayesian_regularization.py", + "notebook": "notebooks/chapter_3_pixelizations/tutorial_4_bayesian_regularization.ipynb", + "path": "scripts/chapter_3_pixelizations/tutorial_4_bayesian_regularization.py", "summary": "So far, we have:", "title": "Tutorial 4: Bayesian Regularization" }, { "contents": [ "Initial Setup", + "Dataset Auto-Simulation", + "Mask", + "Over Sampling", + "Mesh Shape", + "Galaxy", + "Image Grid", + "Mesh Pixel Centres", + "Interpolation", + "Mapper", + "Mapping Matrix", + "Blurred Mapping Matrix", + "Data Vector (D)", + "Curvature Matrix (F)", + "Unregularized Solve", + "Regularization Matrix (H)", + "Galaxy Reconstruction (s)", + "Image Reconstruction", + "Likelihood Function", + "Chi Squared", + "Regularization Term", + "Complexity Terms", + "Noise Normalization Term", + "Log Evidence", + "Fit", + "Wrap Up" + ], + "cross_refs": [ + "/imaging/features/pixelization/likelihood_function.ipynb" + ], + "notebook": "notebooks/chapter_3_pixelizations/tutorial_5_bayesian_formalism.ipynb", + "path": "scripts/chapter_3_pixelizations/tutorial_5_bayesian_formalism.py", + "summary": "In tutorials 1 to 4, we built an intuition for how pixelized galaxy reconstruction works: pixelizations place a pixel-grid over the galaxy's image, mappers pair pixelization pixels with image-pixels, inversions solve for the pixel fluxes that best fit the data, and regularization smooths the solution within a Bayesian framework.", + "title": "Tutorial 5: Bayesian Formalism" + }, + { + "contents": [ + "Initial Setup", + "Dataset Auto-Simulation", "Model + Search + Analysis + Model-Fit (Search 1)", "Mesh Shape", "Model + Search + Analysis + Model-Fit (Search 2)", @@ -354,10 +384,76 @@ "Wrap Up" ], "cross_refs": [], - "notebook": "notebooks/chapter_4_pixelizations/tutorial_5_model_fit.ipynb", - "path": "scripts/chapter_4_pixelizations/tutorial_5_model_fit.py", - "summary": "In the previous tutorials we used an inversion to reconstruct a complex galaxy. However, from the perspective of a scientific analysis, it is not clear how useful this was. When I fit a galaxy with light profiles, I learn about its brightness (`intensity`), size (`effective_radius`), compactness (`sersic_index`), etc.", - "title": "Tutorial 5: Model-Fit" + "notebook": "notebooks/chapter_3_pixelizations/tutorial_6_model_fit.ipynb", + "path": "scripts/chapter_3_pixelizations/tutorial_6_model_fit.py", + "summary": "In the previous tutorials we used an inversion to reconstruct a complex galaxy. However, from the perspective of a scientific analysis, it is not clear how useful this was. When we fit a galaxy with light profiles, we learn about its brightness (`intensity`), size (`effective_radius`), compactness (`sersic_index`), etc.", + "title": "Tutorial 6: Model Fit" + }, + { + "contents": [ + "Initial Setup", + "Dataset Auto-Simulation", + "The Decision", + "Mask", + "Approach 1 Noise Scaling", + "Noise Scaling Fit", + "Approach 2 Extra Galaxies Model", + "Extra Galaxy Centres", + "Extra Galaxies Model Composition", + "Extra Galaxies Fit", + "Which Approach When", + "Wrap Up" + ], + "cross_refs": [ + "/imaging/features/extra_galaxies/modeling.py" + ], + "notebook": "notebooks/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.ipynb", + "path": "scripts/chapter_4_scaling_up_galaxies/tutorial_1_extra_galaxies.py", + "summary": "Welcome to chapter 4 of **HowToGalaxy**, where we scale up galaxy modeling beyond a single galaxy.", + "title": "Tutorial 1: Extra Galaxies" + }, + { + "contents": [ + "Initial Setup", + "Dataset Auto-Simulation", + "Mask", + "Over Sampling", + "Why Not Fit Them Separately?", + "Model", + "Fixing the Centres", + "Model Fit", + "Result", + "Light Decomposition Degeneracy", + "Wrap Up" + ], + "cross_refs": [], + "notebook": "notebooks/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.ipynb", + "path": "scripts/chapter_4_scaling_up_galaxies/tutorial_2_multi_galaxy.py", + "summary": "In the previous tutorial, we learned how to deal with extra galaxies near the galaxy we care about \u2014 nuisance objects whose light contaminates the data but which are not themselves the subject of our study. We removed their emission from the analysis, or gave them a heavily restricted model, and the single galaxy we were studying remained the star of the show.", + "title": "Tutorial 2: Multi-Galaxy Blends" + }, + { + "contents": [ + "The Scaling Problem", + "Dataset", + "Dataset Auto-Simulation", + "Member Catalogue", + "Masking", + "Model", + "Search + Analysis", + "Model Fit", + "Result", + "Per-Member Results", + "Refinements", + "Wrap Up" + ], + "cross_refs": [ + "/cluster/modeling.py" + ], + "notebook": "notebooks/chapter_4_scaling_up_galaxies/tutorial_3_cluster.ipynb", + "path": "scripts/chapter_4_scaling_up_galaxies/tutorial_3_cluster.py", + "summary": "Throughout this chapter we have been scaling up: from a single galaxy with extra galaxies nearby, to blended multi-galaxy systems where every galaxy received its own free light model.", + "title": "Tutorial 3: Cluster" }, { "contents": [ @@ -371,6 +467,60 @@ "summary": "Up to now, we've always used the non-linear search Nautilus and not considered the input parameters that control its sampling. In this tutorial, we'll consider how we can change these setting to balance finding the global maxima solution with fast run time.", "title": "Tutorial: Alternative Searches" }, + { + "contents": [ + "Dataset Paths", + "Grid", + "Galaxies", + "Output", + "Member Catalogue CSV", + "Visualize", + "Plane Output" + ], + "cross_refs": [ + "simulators/simple.ipynb" + ], + "notebook": "notebooks/simulators/cluster.ipynb", + "path": "scripts/simulators/cluster.py", + "summary": "This script simulates `Imaging` of a cluster field: a brightest cluster galaxy (BCG) surrounded by ten lower-luminosity member galaxies. It is used in chapter 4 of the **HowToGalaxy** lectures, where the member population is modeled via a **catalogue**: a CSV of member centres and luminosities whose photometry pins the faint galaxies while only a shared normalization stays free.", + "title": "Simulator: Cluster" + }, + { + "contents": [ + "Dataset Paths", + "Grid", + "Galaxies", + "Output", + "Visualize", + "Mask Extra Galaxies", + "Plane Output", + "Extra Galaxies Centres" + ], + "cross_refs": [ + "simulators/simple.ipynb" + ], + "notebook": "notebooks/simulators/extra_galaxy.ipynb", + "path": "scripts/simulators/extra_galaxy.py", + "summary": "This script simulates `Imaging` of a galaxy using light profiles where:", + "title": "Simulator: Extra Galaxy" + }, + { + "contents": [ + "Dataset Paths", + "Simulate", + "Galaxies", + "Output", + "Visualize", + "Plane Output" + ], + "cross_refs": [ + "/interferometer/simulator.ipynb" + ], + "notebook": "notebooks/simulators/interferometer.ipynb", + "path": "scripts/simulators/interferometer.py", + "summary": "This script simulates `Interferometer` data of a galaxy, as would be observed by a radio or sub-mm interferometer like ALMA or the JVLA, where:", + "title": "Simulator: Interferometer" + }, { "contents": [ "Dataset Paths",