From e09983937650c1aa3f38b6e7bf49f83fd8b6854d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 03:08:28 +0000 Subject: [PATCH] Extract workspace/ to the autoreduce_workspace repository The staged tutorial workspace now lives at https://github.com/PyAutoLabs/autoreduce_workspace, extracted with full history via git filter-repo --subdirectory-filter workspace. Remove the embedded copy, move the offline workspace-smoke CI job to that repo's own workflow, and update AGENTS.md to point at the sibling repository. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NEporMREot87nV51fedoZH --- .github/workflows/main.yml | 38 +- AGENTS.md | 21 +- workspace/.gitignore | 18 - workspace/.script_sizes.json | 40 -- workspace/AGENTS.md | 114 ---- workspace/CLAUDE.md | 7 - workspace/README.md | 146 ----- workspace/config/README.md | 16 - workspace/config/build/no_run.yaml | 23 - workspace/config/build/profile_smoke.yaml | 16 - workspace/dataset/README.md | 32 - workspace/llms.txt | 98 ---- workspace/requirements.txt | 15 - workspace/runtime.txt | 1 - workspace/scripts/alma/README.md | 22 - workspace/scripts/alma/__init__.py | 0 workspace/scripts/alma/simulator.py | 354 ------------ workspace/scripts/alma/start_here.py | 517 ----------------- workspace/scripts/alma/step_by_step.py | 526 ----------------- workspace/scripts/check_sizes.sh | 63 -- workspace/scripts/guides/README.md | 15 - workspace/scripts/guides/__init__.py | 0 workspace/scripts/guides/noise_maps.py | 253 -------- workspace/scripts/guides/output_contract.py | 304 ---------- workspace/scripts/guides/target_spec.py | 271 --------- workspace/scripts/hst_acs/README.md | 20 - workspace/scripts/hst_acs/__init__.py | 0 workspace/scripts/hst_acs/dials.py | 331 ----------- workspace/scripts/hst_acs/individual.py | 403 ------------- workspace/scripts/hst_acs/psf.py | 402 ------------- workspace/scripts/hst_acs/simulator.py | 360 ------------ workspace/scripts/hst_acs/start_here.py | 398 ------------- workspace/scripts/hst_acs/step_by_step.py | 546 ------------------ workspace/scripts/hst_wfc3_ir/README.md | 22 - workspace/scripts/hst_wfc3_ir/__init__.py | 0 workspace/scripts/hst_wfc3_ir/start_here.py | 339 ----------- workspace/scripts/hst_wfc3_ir/step_by_step.py | 303 ---------- workspace/scripts/hst_wfc3_uvis/README.md | 20 - workspace/scripts/hst_wfc3_uvis/__init__.py | 0 workspace/scripts/hst_wfc3_uvis/psf.py | 290 ---------- workspace/scripts/hst_wfc3_uvis/simulator.py | 257 --------- workspace/scripts/hst_wfc3_uvis/start_here.py | 333 ----------- .../scripts/hst_wfc3_uvis/step_by_step.py | 307 ---------- workspace/scripts/jwst_nircam/README.md | 18 - workspace/scripts/jwst_nircam/__init__.py | 0 workspace/scripts/jwst_nircam/individual.py | 352 ----------- workspace/scripts/jwst_nircam/multi_band.py | 294 ---------- workspace/scripts/jwst_nircam/psf.py | 352 ----------- workspace/scripts/jwst_nircam/simulator.py | 333 ----------- workspace/scripts/jwst_nircam/start_here.py | 437 -------------- workspace/scripts/jwst_nircam/step_by_step.py | 377 ------------ workspace/scripts/keck_nirc2/README.md | 19 - workspace/scripts/keck_nirc2/__init__.py | 0 workspace/scripts/keck_nirc2/psf.py | 281 --------- workspace/scripts/keck_nirc2/simulator.py | 376 ------------ workspace/scripts/keck_nirc2/start_here.py | 464 --------------- workspace/scripts/keck_nirc2/step_by_step.py | 408 ------------- workspace/scripts/start_here.py | 446 -------------- workspace/scripts/surveys/README.md | 17 - workspace/scripts/surveys/__init__.py | 0 workspace/scripts/surveys/start_here.py | 304 ---------- workspace/smoke_tests.txt | 62 -- 62 files changed, 13 insertions(+), 11768 deletions(-) delete mode 100644 workspace/.gitignore delete mode 100644 workspace/.script_sizes.json delete mode 100644 workspace/AGENTS.md delete mode 100644 workspace/CLAUDE.md delete mode 100644 workspace/README.md delete mode 100644 workspace/config/README.md delete mode 100644 workspace/config/build/no_run.yaml delete mode 100644 workspace/config/build/profile_smoke.yaml delete mode 100644 workspace/dataset/README.md delete mode 100644 workspace/llms.txt delete mode 100644 workspace/requirements.txt delete mode 100644 workspace/runtime.txt delete mode 100644 workspace/scripts/alma/README.md delete mode 100644 workspace/scripts/alma/__init__.py delete mode 100644 workspace/scripts/alma/simulator.py delete mode 100644 workspace/scripts/alma/start_here.py delete mode 100644 workspace/scripts/alma/step_by_step.py delete mode 100755 workspace/scripts/check_sizes.sh delete mode 100644 workspace/scripts/guides/README.md delete mode 100644 workspace/scripts/guides/__init__.py delete mode 100644 workspace/scripts/guides/noise_maps.py delete mode 100644 workspace/scripts/guides/output_contract.py delete mode 100644 workspace/scripts/guides/target_spec.py delete mode 100644 workspace/scripts/hst_acs/README.md delete mode 100644 workspace/scripts/hst_acs/__init__.py delete mode 100644 workspace/scripts/hst_acs/dials.py delete mode 100644 workspace/scripts/hst_acs/individual.py delete mode 100644 workspace/scripts/hst_acs/psf.py delete mode 100644 workspace/scripts/hst_acs/simulator.py delete mode 100644 workspace/scripts/hst_acs/start_here.py delete mode 100644 workspace/scripts/hst_acs/step_by_step.py delete mode 100644 workspace/scripts/hst_wfc3_ir/README.md delete mode 100644 workspace/scripts/hst_wfc3_ir/__init__.py delete mode 100644 workspace/scripts/hst_wfc3_ir/start_here.py delete mode 100644 workspace/scripts/hst_wfc3_ir/step_by_step.py delete mode 100644 workspace/scripts/hst_wfc3_uvis/README.md delete mode 100644 workspace/scripts/hst_wfc3_uvis/__init__.py delete mode 100644 workspace/scripts/hst_wfc3_uvis/psf.py delete mode 100644 workspace/scripts/hst_wfc3_uvis/simulator.py delete mode 100644 workspace/scripts/hst_wfc3_uvis/start_here.py delete mode 100644 workspace/scripts/hst_wfc3_uvis/step_by_step.py delete mode 100644 workspace/scripts/jwst_nircam/README.md delete mode 100644 workspace/scripts/jwst_nircam/__init__.py delete mode 100644 workspace/scripts/jwst_nircam/individual.py delete mode 100644 workspace/scripts/jwst_nircam/multi_band.py delete mode 100644 workspace/scripts/jwst_nircam/psf.py delete mode 100644 workspace/scripts/jwst_nircam/simulator.py delete mode 100644 workspace/scripts/jwst_nircam/start_here.py delete mode 100644 workspace/scripts/jwst_nircam/step_by_step.py delete mode 100644 workspace/scripts/keck_nirc2/README.md delete mode 100644 workspace/scripts/keck_nirc2/__init__.py delete mode 100644 workspace/scripts/keck_nirc2/psf.py delete mode 100644 workspace/scripts/keck_nirc2/simulator.py delete mode 100644 workspace/scripts/keck_nirc2/start_here.py delete mode 100644 workspace/scripts/keck_nirc2/step_by_step.py delete mode 100644 workspace/scripts/start_here.py delete mode 100644 workspace/scripts/surveys/README.md delete mode 100644 workspace/scripts/surveys/__init__.py delete mode 100644 workspace/scripts/surveys/start_here.py delete mode 100644 workspace/smoke_tests.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ba578e4..62d16b9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,39 +23,5 @@ jobs: with: package: autoreduce secrets: inherit - - # The staged workspace's offline-runnable tutorial scripts double as an - # integration smoke of the public API (TargetSpec dials + guard rails, the - # Casertano noise mathematics, the drizzle-dial trade study). Everything - # network-dependent in workspace/ is excluded — archive access does not - # belong in CI. This job moves out with workspace/ when it is extracted to - # the autoreduce_workspace repository. - workspace_smoke: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.12", "3.13"] - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: pip - - - name: Install autoreduce + plotting deps - run: | - pip install --upgrade pip setuptools wheel - pip install -e . matplotlib scipy - - - name: Run offline workspace scripts - run: | - set -e - cd workspace - export MPLCONFIGDIR=/tmp/matplotlib - for s in scripts/guides/target_spec.py scripts/guides/noise_maps.py scripts/hst_acs/dials.py; do - echo "::group::$s" - python "$s" - echo "::endgroup::" - done + # (The offline workspace-smoke job moved to the extracted + # autoreduce_workspace repository's own smoke_tests.yml.) diff --git a/AGENTS.md b/AGENTS.md index 765a5b2..b539462 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,16 +34,17 @@ applies here. - FITS files are never committed (`.gitignore` enforces this); `prototypes/` writes to `prototypes/output/` and `prototypes/cache/`. -## The embedded workspace (staged for extraction) - -`workspace/` is the complete **autoreduce_workspace** — the user-facing tutorial -workspace (per-instrument `start_here.py` / `step_by_step.py` / `psf.py` / -`simulator.py` examples, guides, README, llms.txt) — staged here because the -`PyAutoLabs/autoreduce_workspace` repository does not exist yet. It is written -as a self-contained repo root: once the repo is created, extract it with -`git filter-repo --subdirectory-filter workspace` (or copy the tree) and it -works unchanged. Its own `AGENTS.md` governs work inside it; do not entangle it -with library code (scripts consume only the public `autoreduce` API). +## The workspace + +The user-facing tutorial workspace (per-instrument `start_here.py` / +`step_by_step.py` / `psf.py` / `simulator.py` examples, guides, README, +llms.txt) lives in its own repository: +https://github.com/PyAutoLabs/autoreduce_workspace (typically cloned as a +sibling, `../autoreduce_workspace`). It was staged under `workspace/` in this +repo until 2026-08-07 and extracted with history via +`git filter-repo --subdirectory-filter workspace`. Workspace scripts consume +only the public `autoreduce` API — a public-API change here means the +workspace examples may need updating; flag it in your PR. ## Never rewrite history diff --git a/workspace/.gitignore b/workspace/.gitignore deleted file mode 100644 index cd6fc7a..0000000 --- a/workspace/.gitignore +++ /dev/null @@ -1,18 +0,0 @@ -__pycache__/ -*.pyc -.ipynb_checkpoints/ - -# Reduction runtime trees — created by the scripts, never committed. -output/ -cache/ - -# FITS files are never committed anywhere in this repo: reduced products land -# in output//, downloaded exposures in cache/. This deny + allow-list -# pattern keeps dataset/ to per-target YAML spec files (and READMEs) only — -# to ship a new non-FITS ancillary file, re-include it explicitly below. -*.fits -dataset/** -!dataset/README.md -!dataset/**/*.yaml - -*.log diff --git a/workspace/.script_sizes.json b/workspace/.script_sizes.json deleted file mode 100644 index 4b003de..0000000 --- a/workspace/.script_sizes.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "scripts/alma/__init__.py": 0, - "scripts/alma/simulator.py": 16291, - "scripts/alma/start_here.py": 26066, - "scripts/alma/step_by_step.py": 25668, - "scripts/guides/__init__.py": 0, - "scripts/guides/noise_maps.py": 13825, - "scripts/guides/output_contract.py": 15552, - "scripts/guides/target_spec.py": 14916, - "scripts/hst_acs/__init__.py": 0, - "scripts/hst_acs/dials.py": 17822, - "scripts/hst_acs/individual.py": 19265, - "scripts/hst_acs/psf.py": 20797, - "scripts/hst_acs/simulator.py": 16907, - "scripts/hst_acs/start_here.py": 19414, - "scripts/hst_acs/step_by_step.py": 27733, - "scripts/hst_wfc3_ir/__init__.py": 0, - "scripts/hst_wfc3_ir/start_here.py": 16687, - "scripts/hst_wfc3_ir/step_by_step.py": 15687, - "scripts/hst_wfc3_uvis/__init__.py": 0, - "scripts/hst_wfc3_uvis/psf.py": 13840, - "scripts/hst_wfc3_uvis/simulator.py": 11721, - "scripts/hst_wfc3_uvis/start_here.py": 16867, - "scripts/hst_wfc3_uvis/step_by_step.py": 15826, - "scripts/jwst_nircam/__init__.py": 0, - "scripts/jwst_nircam/individual.py": 17097, - "scripts/jwst_nircam/multi_band.py": 13495, - "scripts/jwst_nircam/psf.py": 17739, - "scripts/jwst_nircam/simulator.py": 15754, - "scripts/jwst_nircam/start_here.py": 22483, - "scripts/jwst_nircam/step_by_step.py": 19041, - "scripts/keck_nirc2/__init__.py": 0, - "scripts/keck_nirc2/psf.py": 14243, - "scripts/keck_nirc2/simulator.py": 17456, - "scripts/keck_nirc2/start_here.py": 23681, - "scripts/keck_nirc2/step_by_step.py": 20592, - "scripts/start_here.py": 23230, - "scripts/surveys/__init__.py": 0, - "scripts/surveys/start_here.py": 14173 -} diff --git a/workspace/AGENTS.md b/workspace/AGENTS.md deleted file mode 100644 index bf09a7e..0000000 --- a/workspace/AGENTS.md +++ /dev/null @@ -1,114 +0,0 @@ -# PyAutoReduce Workspace — Agent Instructions - -This is the tutorial and example workspace for **PyAutoReduce**, a Python library that reduces raw -archival telescope data (HST, JWST, Keck-AO, ALMA, survey cutout services) into modeling-ready -datasets for **PyAutoLens** and **PyAutoGalaxy**. These are the canonical, agent-agnostic -instructions for this repo; `CLAUDE.md` imports this file. - -## Repository Structure - -- `scripts/` — Runnable Python tutorial scripts, organised by instrument: - - `start_here.py` — the top-level overview: why reduction quality matters, the declarative - `TargetSpec` philosophy, a complete HST/ACS reduction of a SLACS lens, routing to every folder. - - `guides/` — cross-instrument guides: `output_contract.py` (the four-file + `reduction.json` - contract), `noise_maps.py` (noise recipes + the Casertano correlated-noise factor), - `target_spec.py` (every `TargetSpec` dial; runs offline). - - `hst_acs/`, `hst_wfc3_uvis/`, `hst_wfc3_ir/` — HST reductions (AstroDrizzle path). - - `jwst_nircam/` — JWST NIRCam reductions (calwebb_image3 path; MJy/sr units). - - `keck_nirc2/` — Keck NIRC2 LGS-AO reductions (ground-based calibrate/sky stages, native combine). - - `alma/` — ALMA visibility extraction (uv-plane datasets for `al.Interferometer`). - - `surveys/` — survey cutout fetching (colour context only, never modeling data). -- `notebooks/` — Jupyter notebook versions, generated from `scripts/` at release time (do not - edit directly). -- `config/` — build/CI configuration only; **PyAutoReduce** itself needs no config YAML. -- `dataset/` — per-target `TargetSpec` YAML files only; FITS is never committed. -- `output/`, `cache/` — reduction products and downloaded exposures (gitignored, created at runtime). - -## Running Scripts - -Scripts are run **from the workspace root**: - -```bash -python scripts/hst_acs/start_here.py -``` - -Path logic inside scripts does not depend on the cwd — every script anchors paths to the -workspace root via `Path(__file__).resolve().parents[...]`, because **PyAutoReduce**'s drizzle -step changes the working directory internally and requires absolute paths. - -**Almost every script needs network access and a heavy instrument stack** (MAST/KOA/ALMA archive -downloads, drizzlepac / jwst / casatools). First runs of a target download exposures into -`cache/` (re-used afterwards) and write products into `output//`. Both directories are -gitignored. The exceptions that run offline are `scripts/guides/target_spec.py` and most of -`scripts/guides/noise_maps.py`. - -### Standard imports - -```python -from autoreduce import TargetSpec, reduce_target -``` - -The public surface is exactly these two names plus the documented helpers -(`autoreduce.instruments`, `autoreduce.validation`, `autoreduce.noise.rms`). The canonical idiom -— build a frozen `TargetSpec`, call `reduce_target`, read the returned provenance dict, load the -FITS off disk — is shown in `scripts/start_here.py`; read that rather than relying on a recipe -here. - -## Style Contract - -Every script is a long-form narrated tutorial in the PyAuto workspace house style: - -- Module docstring with the title underlined with `=`, framing paragraphs, and a `__Contents__` - bullet list. -- Every prose block is a bare triple-quoted `"""__Section__"""` docstring at module level. **No - `# ----` banner comments.** Inline `#` is reserved for short line-level notes and trailing - per-kwarg explanations in constructor calls. -- Second person, present tense; **PyAutoReduce** / **PyAutoLens** bolded on every mention; papers - cited inline with arXiv IDs; honest caveats are first-class sections. -- Scripts close with `__Wrap Up__` and, where the script needs network, a final - `__Env__ (Developer Only)` section containing the bare line `ENV: network`. - -## Testing - -`smoke_tests.txt` is the curated smoke list. It is currently **all commented out**: every script -needs archive network access plus a heavy dependency stack, so none run in CI smoke tests yet. -Each commented entry cites its reason. `config/build/profile_smoke.yaml` carries only writable -cache-dir defaults; `config/build/no_run.yaml` is empty apart from its conventions. - -## Bulk-edit safety - -When editing the same region across many scripts in one pass (adding a section, renaming a -symbol, updating an import block), only rewrite the targeted region. **Never produce a whole-file -write unless you have read the entire current contents of that file** — a whole-file write based -on a header skim silently deletes every section below the header. Prefer targeted edits over -whole-file writes. - -## Hard Boundaries - -- Workspace scripts consume the **public** `autoreduce` API (`TargetSpec`, `reduce_target`, the - documented helpers) and optionally `autolens` for loading the finished products. They **never - import private `_`-prefixed autoreduce functions** — the pipeline stages are internal, and the - step-by-step scripts teach them by reading the evidence out of `reduction.json` and the `work/` - directory, not by calling them. -- `autolens` imports are always guarded with `try/except ImportError` — the reduction itself - never depends on the modeling stack (**PyAutoReduce** never imports it either). -- No FITS is ever committed; `output/` and `cache/` stay out of git. - -## Related Repos - -The PyAutoReduce stack (all on the `PyAutoLabs` GitHub org): - -- https://github.com/PyAutoLabs/PyAutoReduce — the library this workspace demonstrates. -- https://github.com/PyAutoLabs/PyAutoLens — strong-lens modeling; loads these products via - `al.Imaging.from_fits` / `al.Interferometer.from_fits`. -- https://github.com/PyAutoLabs/PyAutoGalaxy — galaxy-morphology modeling; same input format. -- https://github.com/PyAutoLabs/autolens_workspace — the modeling workspace these datasets feed; - its `data_preparation` examples state the standards every product here satisfies. -- https://github.com/PyAutoLabs/PyAutoHands — notebook generation + CI tooling. - -## Never rewrite history - -Never rewrite pushed history on any repo with a remote — no `git init` over a -tracked repo, no force-push to `main`, no fresh-start "Initial commit", no -`filter-repo` / `filter-branch` / `rebase -i` on pushed branches. To get a -clean tree: `git fetch origin && git reset --hard origin/main && git clean -fd`. diff --git a/workspace/CLAUDE.md b/workspace/CLAUDE.md deleted file mode 100644 index d67979f..0000000 --- a/workspace/CLAUDE.md +++ /dev/null @@ -1,7 +0,0 @@ -# autoreduce_workspace — agent instructions - -The canonical, agent-agnostic instructions live in `AGENTS.md`. Claude Code loads them via the -import below; if your tool does not process `@`-imports, open `AGENTS.md` in this directory and -read it directly. - -@AGENTS.md diff --git a/workspace/README.md b/workspace/README.md deleted file mode 100644 index 8d77d49..0000000 --- a/workspace/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# PyAutoReduce Workspace - -[PyAutoReduce](https://github.com/PyAutoLabs/PyAutoReduce) | -[PyAutoLens](https://github.com/PyAutoLabs/PyAutoLens) | -[autolens_workspace](https://github.com/PyAutoLabs/autolens_workspace) | -[PyAutoGalaxy](https://github.com/PyAutoLabs/PyAutoGalaxy) - -Welcome to the **PyAutoReduce** Workspace! - -**PyAutoReduce** reduces raw archival telescope data into **modeling-ready datasets** for -[PyAutoLens](https://github.com/PyAutoLabs/PyAutoLens) and -[PyAutoGalaxy](https://github.com/PyAutoLabs/PyAutoGalaxy). Given a target (a strong lens, a -galaxy), it downloads the archive exposures, reduces them with the instrument's standard pipeline -tooling, and emits the exact products the modeling stack loads via `al.Imaging.from_fits`: - -| Product | Description | -|---------|-------------| -| `data.fits` | Science cutout at the modeling pixel scale | -| `noise_map.fits` | Per-pixel RMS noise, correlated-noise corrected | -| `psf.fits` / `psf_full.fits` | Reduction-consistent PSF estimate (compact + extended) | -| `reduction.json` | Full provenance: exposures, pipeline dials, diagnostics, software versions | - -A reduction is *declared*, not scripted: you build a `TargetSpec`, call `reduce_target`, and the -pipeline is a pure function of that spec plus the archive. Every script in this workspace is a -narrated, runnable walk through one instrument's reduction. - -## Instrument Coverage - -Each instrument folder is anchored to a real target reduced end-to-end and validated against a -published literature reduction — the workspace's quality bar: - -| Folder | Instrument | Validation anchor | Literature quality bar | -|--------|-----------|-------------------|------------------------| -| `scripts/hst_acs` | HST ACS/WFC | SLACS J0008-0004 (prop. 10886) | SLACS, Bolton et al. 2008 | -| `scripts/hst_wfc3_uvis` | HST WFC3/UVIS | SDSS J0252+0039 F390W | Bayer et al. (arXiv:1803.05952) | -| `scripts/hst_wfc3_ir` | HST WFC3/IR | J0252+0039 IR snapshot | WFC3 Data Handbook conventions | -| `scripts/jwst_nircam` | JWST NIRCam (SW + LW) | COSMOS-Web ring (prop. 1727) | Mercier et al. 2024 (arXiv:2309.15986) | -| `scripts/keck_nirc2` | Keck NIRC2 LGS-AO | B1938+666 | SHARP, Lagattuta et al. 2012; Chen et al. 2019 | -| `scripts/alma` | ALMA (visibilities) | G09v1.40 (2016.1.00282.S) | uv-plane practice, e.g. Hezaveh et al. 2016 | -| `scripts/surveys` | Legacy Surveys / SDSS / Pan-STARRS cutouts | SLACS J0008-0004 field | Colour context only — **not** modeling data | - -## Getting Started - -Install the core package, then the extras for the instrument(s) you reduce: - -```bash -pip install autoreduce # core (numpy, astropy, astroquery, photutils, PyYAML) -``` - -| Extra | Installs | Needed for | -|-------|----------|------------| -| `autoreduce[hst]` | drizzlepac, drizzle | HST ACS + WFC3 (the AstroDrizzle combine) | -| `autoreduce[keck]` | pykoa, drizzle | Keck NIRC2 (KOA acquisition + native combine) | -| `autoreduce[psf]` | psfr, stpsf | High-fidelity PSF back-ends | -| `autoreduce[starred]` | starred-astro | The STARRED super-sampled ePSF back-end (GPL + JAX) | -| `autoreduce[frames]` | deepCR | Per-frame cosmic-ray masking (`cr_method="deepcr"`, frame products) | - -Two stacks are deliberately **not** pip extras and are installed separately: - -- **JWST**: `pip install jwst==1.14.0` (the pipeline is pinned to this version) plus `crds`, - with `CRDS_PATH` / `CRDS_SERVER_URL` set for lazy reference syncing. -- **ALMA**: modular CASA — `pip install casatools casatasks` — for the visibility branch. - -**Be honest with yourself about what running these scripts involves**: every reduction downloads -real exposures from an archive (MAST, KOA, the ALMA archive) and runs the instrument's heavy -reduction stack. Scripts need network access, several GB of cache space, and minutes-to-hours of -runtime on the first run of a target. They are written to *read* as documentation even when you -cannot run them. - -## New Users - -New users should read `scripts/start_here.py`. It explains why reduction quality matters for lens -modeling, walks a complete HST/ACS reduction of a SLACS lens end-to-end, and routes you to the -right instrument folder for your data. - -The `scripts/guides` folder then covers the pieces every instrument shares: the output contract -(`output_contract.py`), noise-map construction (`noise_maps.py`) and the `TargetSpec` declaration -(`target_spec.py`) — the last two run offline. - -## Workspace Structure - -The workspace includes the following main directories: - -- `scripts`: **PyAutoReduce** examples written as Python scripts, one folder per instrument. -- `notebooks`: Jupyter notebook versions, generated from `scripts` at release time. -- `config`: Build/CI configuration only — **PyAutoReduce** itself needs no config files. -- `dataset`: Per-target `TargetSpec` YAML files you may add (no FITS is ever committed). -- `output`: Where reduced datasets land, one folder per target (not committed). -- `cache`: Downloaded exposures + CRDS references, re-used across runs (not committed). - -The instrument packages include the following types of example: - -- `start_here`: The default pipeline end-to-end on the instrument's validation anchor. -- `step_by_step`: Every reduction step in as much granularity as the public API allows, taught - with the instrument handbook and the literature. -- `individual`: Per-exposure frame products — modeling native frames instead of a mosaic. -- `psf`: The PSF story for that instrument — star selection, tiers, back-ends, diagnostics. -- `simulator`: Synthetic-source injection into real frames, and flux-recovery closure tests. - -The `README.md` files distributed throughout the workspace describe what is in each folder. - -## Community & Support - -Support for **PyAutoReduce** is available via the PyAuto Slack workspace, where the community -shares updates and helps troubleshoot problems. Slack is invitation-only: if you'd like to join, -please send an email requesting an invite. - -For installation issues, bug reports, or feature requests, please raise an issue on the -[GitHub issues page](https://github.com/PyAutoLabs/PyAutoReduce/issues). - -## Contribution - -To make changes to the examples, edit the corresponding Python files (`.py`) in the `scripts` -folder — never the notebooks, which are generated from the scripts at release time. The marker -`# %%` alternates between code cells and markdown cells in the generated notebooks. - -## The Reduction Domain Ladder: Mosaics, Frames, Visibilities and Cutouts - -**PyAutoReduce** organises reductions into a ladder of four data domains. What changes as you -climb is not the instrument but the *form* the modeling-ready dataset takes: - -- An **imaging mosaic** (`hst_acs`, `hst_wfc3_uvis`, `hst_wfc3_ir`, `jwst_nircam`, `keck_nirc2`) - is the default rung: dithered exposures combined onto a single resampled grid, with a matching - noise map and a PSF built through the identical resampling. Resampling correlates the noise — - which is why the noise map carries the Casertano correlated-noise correction. - -- **Per-exposure frames** (`frame_products=True`; the `individual.py` scripts) sidestep the - mosaic: every calibrated exposure ships as its own native-pixel dataset with per-frame PSFs and - a manifest. Modeling the frames jointly costs more bookkeeping but the noise in each frame is - *uncorrelated* — no drizzle, no correction factor. - -- **Visibilities** (`alma`) skip images entirely: the dataset is the calibrated interferometer - visibilities themselves — `data.fits` / `uv_wavelengths.fits` / `noise_map.fits`, each an - `(N_vis, 2)` array loaded via `al.Interferometer.from_fits` — because fitting in the uv-plane - keeps the noise independent and the likelihood well-defined. - -- **Survey cutouts** (`surveys`) are the context rung: pre-reduced coadd cutouts fetched from - Legacy Surveys / SDSS / Pan-STARRS services for colour context around a target. They ship no - PSF and are **not** modeling data. - -## Build Configuration - -The `config/build/` directory contains files used by the automated build and test system (CI, -smoke tests). These are not relevant to normal workspace usage — see `config/README.md`. Because -every script here needs network access and a heavy instrument stack, no script currently runs in -CI smoke tests (see `smoke_tests.txt` for the per-script reasons). diff --git a/workspace/config/README.md b/workspace/config/README.md deleted file mode 100644 index 87e9350..0000000 --- a/workspace/config/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# config/ - -**PyAutoReduce needs no configuration YAML.** Unlike the PyAutoLens / PyAutoGalaxy workspaces — -whose `config/` folders customise priors, visualization and search behaviour — a reduction has no -tunable global state: everything a run needs is declared on the `TargetSpec` itself (see -`scripts/guides/target_spec.py`), and the pipeline is a pure function of that spec plus the -archive. There is deliberately nothing here for a user to edit. - -The only contents are `config/build/`, which is **CI-only** — files consumed by the automated -build and test system, not by **PyAutoReduce**: - -- `config/build/profile_smoke.yaml` — per-script environment for automated runs. Since no script - currently runs in CI smoke tests (they all need archive network access + heavy instrument - stacks; see `smoke_tests.txt`), this holds only writable cache-dir defaults. -- `config/build/no_run.yaml` — scripts to skip during automated runs, with the repo's - `# SLOW` / `# NEEDS_FIX` tagging conventions. Currently empty. diff --git a/workspace/config/build/no_run.yaml b/workspace/config/build/no_run.yaml deleted file mode 100644 index eb3d771..0000000 --- a/workspace/config/build/no_run.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Scripts to skip during automated runs (smoke tests, pre-release checks, CI). -# Each entry is matched against script paths: -# - Entries with '/' do a substring match against the file path -# - Entries without '/' match the file stem exactly -# Add an inline # comment to document the reason for skipping. -# -# THIS LIST IS REPO-LOCAL. Every entry must match a file in THIS repo; a -# pattern that matches nothing is silently inert, not a skip. -# -# SLOW-skip convention: -# Entries tagged `# SLOW - ` mark scripts skipped -# because they exceed the per-script timeout cap. These are NOT permanent -# skips — fix the performance issue and remove the SLOW marker. -# -# NEEDS_FIX convention: -# Entries tagged `# NEEDS_FIX - ` mark scripts that are -# broken and parked as a to-do list. NOT permanent — investigate, fix the -# underlying bug, and remove the NEEDS_FIX marker. -# -# No entries yet: nothing in this workspace runs in automated smoke tests -# (every script is network + heavy-stack gated in smoke_tests.txt instead). - -[] diff --git a/workspace/config/build/profile_smoke.yaml b/workspace/config/build/profile_smoke.yaml deleted file mode 100644 index f319355..0000000 --- a/workspace/config/build/profile_smoke.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Per-script environment variable configuration for automated runs -# (smoke tests, pre-release checks, CI). -# -# "defaults" are applied to every script on top of the inherited environment. -# "overrides" (none yet) selectively unset or replace vars for matching path -# patterns — patterns containing '/' substring-match the file path, patterns -# without '/' match the file stem exactly. -# -# This workspace's scripts perform real archive reductions, so none run in CI -# smoke tests yet (see smoke_tests.txt). The defaults below exist so that any -# script which does run in a sandboxed environment inherits writable cache -# directories for numba and matplotlib. - -defaults: - NUMBA_CACHE_DIR: "/tmp/numba_cache" # Writable cache dir for numba - MPLCONFIGDIR: "/tmp/matplotlib" # Writable config dir for matplotlib diff --git a/workspace/dataset/README.md b/workspace/dataset/README.md deleted file mode 100644 index 60a3167..0000000 --- a/workspace/dataset/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# dataset/ - -**No FITS is ever committed to this repository** — the `.gitignore` enforces it. Reduced -products (`data.fits`, `noise_map.fits`, `psf.fits`, `psf_full.fits`, `reduction.json`) land in -`output//` at runtime, and downloaded exposures live in `cache/`; both directories are -gitignored and created by the scripts. - -This folder holds only **per-target `TargetSpec` YAML files** that you may add for your own -targets. A spec file is the declarative input of a reduction — load it with -`TargetSpec.from_yaml(path)` and pass it to `reduce_target` (see -`scripts/guides/target_spec.py` for every dial). An example: - -```yaml -# dataset/slacs0008-0004.yaml -name: slacs0008-0004 -ra: 2.012333 -dec: -0.068944 -proposal_ids: [10886] -cutout_shape: [281, 281] -final_pixfrac: 0.6 -``` - -```python -from autoreduce import TargetSpec, reduce_target - -spec = TargetSpec.from_yaml("dataset/slacs0008-0004.yaml") -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) -``` - -Keeping one YAML per target makes a sample reproducible: the pipeline is a pure function of the -spec plus the archive, so committing the spec (never the FITS) is enough to reproduce the -dataset. diff --git a/workspace/llms.txt b/workspace/llms.txt deleted file mode 100644 index 0cf2f04..0000000 --- a/workspace/llms.txt +++ /dev/null @@ -1,98 +0,0 @@ -If your AI can't browse GitHub, paste this entire file into the chat as context. - -# PyAutoReduce Workspace - -> Example and tutorial scripts (and generated notebooks) for reducing raw archival telescope data -> (HST, JWST, Keck-AO, ALMA, survey cutouts) into modeling-ready datasets for **PyAutoLens** / -> **PyAutoGalaxy** with **PyAutoReduce**. This file is a routing layer: given a user's task, point -> them to the RIGHT existing script/notebook/guide in this workspace instead of inventing code. -> Every path below is a real file in this repository; scripts are run from the repo root -> (e.g. `python scripts/hst_acs/start_here.py`) and — unlike a modeling workspace — almost every -> script downloads real archive data and needs the instrument's heavy reduction stack installed. - -## New to data reduction? - -- [scripts/start_here.py](scripts/start_here.py): Why reduction quality (PSF accuracy, honest - noise maps) decides lens-modeling quality, the declarative `TargetSpec` philosophy, and a - complete HST/ACS reduction of a SLACS lens — the single best first read. -- [autolens_workspace](https://github.com/PyAutoLabs/autolens_workspace): The modeling workspace - these datasets feed; its `imaging/data_preparation` examples state the standards every product - here satisfies. - -## Start here - -- [scripts/start_here.py](scripts/start_here.py): Top-level overview — philosophy, output contract, a full SLACS J0008-0004 ACS reduction, provenance walk, loading into PyAutoLens, and routing to every instrument folder. -- [scripts/hst_acs/start_here.py](scripts/hst_acs/start_here.py): HST ACS/WFC entry point — the default AstroDrizzle pipeline end-to-end on SLACS J0008-0004 (the SLACS quality bar). -- [scripts/hst_wfc3_uvis/start_here.py](scripts/hst_wfc3_uvis/start_here.py): HST WFC3/UVIS entry point — SDSS J0252+0039 F390W at native 0.0396"/pix, pixfrac 1.0 (the Bayer et al. dials). -- [scripts/hst_wfc3_ir/start_here.py](scripts/hst_wfc3_ir/start_here.py): HST WFC3/IR entry point — undersampled 0.128" IR pixels, ramp-fitted CR rejection, and the few-dither pixfrac rule. -- [scripts/jwst_nircam/start_here.py](scripts/jwst_nircam/start_here.py): JWST NIRCam entry point — the COSMOS-Web ring through calwebb_image3; surface-brightness MJy/sr units. -- [scripts/keck_nirc2/start_here.py](scripts/keck_nirc2/start_here.py): Keck NIRC2 LGS-AO entry point — B1938+666 from raw KOA frames: darks/flats, running sky, distortion + native combine. -- [scripts/alma/start_here.py](scripts/alma/start_here.py): ALMA entry point — calibrated measurement set to the uv-plane visibility triplet `al.Interferometer` loads. -- [scripts/surveys/start_here.py](scripts/surveys/start_here.py): Survey-cutout entry point — Legacy Surveys / SDSS / Pan-STARRS colour context around a target (never modeling data). - -## I want to… - -- **Understand the output products (`data.fits` / `noise_map.fits` / `psf.fits` / `psf_full.fits` / `reduction.json`)** → [scripts/guides/output_contract.py](scripts/guides/output_contract.py): The contract in depth — headers/WCS/units, the masked-by-noise convention, psf vs psf_full, frame products, the visibility triplet, block-by-block `reduction.json`. -- **Understand noise maps / correlated noise / the Casertano R factor** → [scripts/guides/noise_maps.py](scripts/guides/noise_maps.py): The per-domain noise recipes, `casertano_r` across pixfrac values, blank-sky closure diagnostics, and why chi-squared needs honest uncorrelated noise (mostly runs offline). -- **See every `TargetSpec` dial / write a spec YAML** → [scripts/guides/target_spec.py](scripts/guides/target_spec.py): Every dial annotated, `from_yaml` round-trip, `dataclasses.replace` variants, validation guard rails (runs fully offline). -- **Learn what each HST/ACS pipeline stage does to the data** → [scripts/hst_acs/step_by_step.py](scripts/hst_acs/step_by_step.py): Acquisition, calibration levels, CR handling, alignment, drizzle, noise, PSF, packaging — with the STScI handbooks and literature. -- **Tune the drizzle dials (scale / pixfrac / kernel / CR method)** → [scripts/hst_acs/dials.py](scripts/hst_acs/dials.py): The drizzle trade study — R vs pixfrac, weight-uniformity limits, the driz_cr flux-loss story and the deepCR opt-in, what the literature disagrees on. -- **Get per-exposure frames instead of a mosaic (uncorrelated noise)** → [scripts/hst_acs/individual.py](scripts/hst_acs/individual.py): `frame_products=True` — native-frame cutouts, the manifest, per-frame PSFs, when to model frames jointly; JWST equivalent in [scripts/jwst_nircam/individual.py](scripts/jwst_nircam/individual.py). -- **Understand HST PSF estimation (stars, tiers, star passes)** → [scripts/hst_acs/psf.py](scripts/hst_acs/psf.py): Why PSF accuracy matters for lensing, star selection, the `psf_star_pass` dial, psf vs psf_full, the drizzled-PSF invariant; STARRED vs photutils regimes in [scripts/hst_wfc3_uvis/psf.py](scripts/hst_wfc3_uvis/psf.py). -- **Validate a reduction with synthetic-source injection** → [scripts/hst_acs/simulator.py](scripts/hst_acs/simulator.py): Inject an analytic lensed-arc image into the real frames, re-reduce, difference and check flux recovery; also [scripts/hst_wfc3_uvis/simulator.py](scripts/hst_wfc3_uvis/simulator.py), [scripts/jwst_nircam/simulator.py](scripts/jwst_nircam/simulator.py), [scripts/keck_nirc2/simulator.py](scripts/keck_nirc2/simulator.py) and the simobserve route in [scripts/alma/simulator.py](scripts/alma/simulator.py). -- **Reduce WFC3/UVIS data step by step** → [scripts/hst_wfc3_uvis/step_by_step.py](scripts/hst_wfc3_uvis/step_by_step.py): The UVIS deltas — CTE-corrected `_flc`, post-flash backgrounds, native-scale drizzling. -- **Reduce WFC3/IR data step by step** → [scripts/hst_wfc3_ir/step_by_step.py](scripts/hst_wfc3_ir/step_by_step.py): Up-the-ramp CR rejection, no CTE, undersampling, and the zero-weight-speckle guard on few-dither data. -- **Reduce JWST/NIRCam step by step** → [scripts/jwst_nircam/step_by_step.py](scripts/jwst_nircam/step_by_step.py): Level-2 `_cal` products, calwebb_image3, read-don't-construct ERR noise, MJy/sr units. -- **Build a multi-wavelength JWST dataset** → [scripts/jwst_nircam/multi_band.py](scripts/jwst_nircam/multi_band.py): Loop the four COSMOS-Web bands (SW vs LW adapters and shapes) into a multi-band set for joint modeling. -- **Understand the JWST PSF options** → [scripts/jwst_nircam/psf.py](scripts/jwst_nircam/psf.py): Empirical ePSF vs STARRED vs the STPSF model fallback, SW undersampling, diagnostics in `reduction.json`. -- **Reduce Keck AO data step by step / handle the AO PSF** → [scripts/keck_nirc2/step_by_step.py](scripts/keck_nirc2/step_by_step.py) and [scripts/keck_nirc2/psf.py](scripts/keck_nirc2/psf.py): Ground-based calibration, running sky, distortion epochs, and the provisional PSF-candidate contract (final selection belongs to modeling). -- **Extract ALMA visibilities step by step** → [scripts/alma/step_by_step.py](scripts/alma/step_by_step.py): Calibrated MS → split → extract → assemble; weights, Stokes-I combination, smearing limits. -- **Fetch colour-context cutouts** → [scripts/surveys/start_here.py](scripts/surveys/start_here.py): Legacy Surveys / SDSS / Pan-STARRS cutouts around a target — context only, never modeling inputs. - -## How to answer (for the assistant) - -When routing a user, reply in this shape: - -- **Start here** — the single best existing file for their task (from the lists above). -- **Then see** — the deeper/detailed companion file (e.g. `start_here.py` → `step_by_step.py`). -- **Related guide** — the relevant `scripts/guides/` page (output contract, noise maps, target spec). -- **Why this is the right example** — one line tying the file to their specific need. -- **What to modify** — the few lines they'd change for their case (target name/RA/Dec, proposal - IDs, instrument/filter, drizzle dials, cutout shape). -- **What needs local execution** — flag that a reduction downloads archive data and runs the - instrument's heavy stack on their machine; nothing here runs in a chat. - -### Code style (match the workspace, not banner comments) - -Drafted code must match this workspace's style: use triple-quoted `"""__Section__"""` docstrings -with a line of prose explaining each step, and give a full script a module docstring whose title is -underlined with `=`. Do NOT use `# -----` banner comments to mark sections. Reserve inline `#` for -short line-level notes only. - -Write this: - -```python -"""__Target Spec__ - -Declare the reduction: SLACS J0008-0004 through the default ACS/WFC pipeline. -""" -spec = TargetSpec(name="slacs0008-0004", ra=2.012333, dec=-0.068944, proposal_ids=("10886",)) -``` - -Not this: - -```python -# ----- Target Spec ----- -spec = TargetSpec(name="slacs0008-0004", ra=2.012333, dec=-0.068944, proposal_ids=("10886",)) -``` - -## Capability boundary (chat without local execution) - -A chat assistant can route to the right file, explain reduction concepts (drizzle, correlated -noise, ePSFs, visibility weights), review pasted scripts / tracebacks / `reduction.json` records, -and draft `TargetSpec` declarations. It CANNOT run a reduction: every reduction needs network -access to the archives (MAST, KOA, ALMA), the instrument's heavy dependency stack (drizzlepac, -jwst, casatools), gigabytes of cache space and real runtime — all on the user's machine. The two -offline exceptions are `scripts/guides/target_spec.py` and most of `scripts/guides/noise_maps.py`. -For sustained editing and execution, point the user to a local coding agent (Claude Code / Codex) -run from the workspace root. diff --git a/workspace/requirements.txt b/workspace/requirements.txt deleted file mode 100644 index 24cf3ba..0000000 --- a/workspace/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -autoreduce[hst] - -# Per-instrument extras — install what your scripts need (see README.md): -# -# autoreduce[hst] drizzlepac + drizzle hst_acs / hst_wfc3_uvis / hst_wfc3_ir -# autoreduce[keck] pykoa + drizzle keck_nirc2 -# autoreduce[psf] psfr + stpsf high-fidelity PSF back-ends -# autoreduce[starred] starred-astro STARRED ePSF back-end (GPL + JAX) -# autoreduce[frames] deepCR cr_method="deepcr" / frame products -# -# Installed separately (not pip extras of autoreduce): -# -# jwst==1.14.0 + crds jwst_nircam (the pipeline is pinned to 1.14.0; -# set CRDS_PATH / CRDS_SERVER_URL) -# casatools + casatasks alma (modular CASA for the visibility branch) diff --git a/workspace/runtime.txt b/workspace/runtime.txt deleted file mode 100644 index 1781745..0000000 --- a/workspace/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.12.1 diff --git a/workspace/scripts/alma/README.md b/workspace/scripts/alma/README.md deleted file mode 100644 index 738896b..0000000 --- a/workspace/scripts/alma/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# ALMA — uv-plane visibility reduction - -The visibility domain: calibrated ALMA measurement sets in, the -`al.Interferometer.from_fits` triplet (`data.fits` / `uv_wavelengths.fits` / -`noise_map.fits`, each `(Nvis, 2)`) out. No image is ever made — lens modeling -fits the visibilities directly. - -Read in this order: - -1. `start_here.py` — the full branch end-to-end on the validation anchor - G09v1.40 (project 2016.1.00282.S): why visibilities, the archive/restore - reality, the split → extract → assemble → package chain, and the - **PyAutoLens** dirty-image round trip. -2. `step_by_step.py` — every stage by hand via the public modules: MS anatomy, - headless modular CASA, the bandwidth-smearing budget for channel averaging, - and the WEIGHT-column audit against the visibility scatter. -3. `simulator.py` — CASA `simobserve` as the acquire-alternative: a synthetic - Jy/pixel source through the identical chain, with a flux-recovery closure test. - -All three need modular CASA (`pip install casatools casatasks`); the first two -need a calibrated MS directory on disk (they print acquisition guidance and exit -cleanly without one). Continuum only — cube/line extraction is deferred. diff --git a/workspace/scripts/alma/__init__.py b/workspace/scripts/alma/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/workspace/scripts/alma/simulator.py b/workspace/scripts/alma/simulator.py deleted file mode 100644 index 93f8c79..0000000 --- a/workspace/scripts/alma/simulator.py +++ /dev/null @@ -1,354 +0,0 @@ -""" -ALMA Simulator: simobserve -========================== - -Every reduction pipeline needs a ground truth it can be tested against, and for ALMA the -observatory itself provides the canonical route: CASA's `simobserve` task turns a model -sky image into a measurement set of simulated visibilities for a chosen antenna -configuration, observing time and atmosphere — including realistic thermal noise -(https://casadocs.readthedocs.io/en/stable/notebooks/simulation.html). - -This script builds a lensed-source-like image in pure numpy (Jy per pixel — no lensing -library needed), hands it to the **PyAutoReduce** visibility branch's simobserve -acquire-alternative, and pushes the simulated measurement set through the *identical* -split / extract / assemble / package chain the real G09v1.40 data traverses in -`start_here.py`. Because you set the input flux, you can close the loop: the recovered -short-baseline flux must match what you injected. Budget ~15 minutes; the simulation -itself takes a couple of minutes of CASA time. - -__Contents__ - -- **The Simulation Route:** Why simobserve is the canonical realistic-ALMA simulation. -- **Imports:** Import **PyAutoReduce** and check for modular CASA. -- **Paths:** Anchor every path to the workspace root. -- **The Source Image:** A ring + core in Jy/pixel, from a formula, in pure numpy. -- **Sky Model:** The 4-axis FITS sky model simobserve consumes. -- **Simulation Dials:** The `TargetSpec` with every `alma_sim_*` dial explained. -- **Run The Simulation:** simobserve, then the identical visibility chain. -- **The Inject Block:** The provenance that marks this dataset as synthetic. -- **Flux Recovery:** Close the loop — shortest-baseline flux vs the injected total. -- **UV Coverage:** The simulated array's sampling pattern. -- **Load In PyAutoLens:** The dirty image of your own simulated source. -- **Real-MS Injection:** What is deliberately deferred, and the literature route it maps to. -- **Wrap Up:** Summary and good places to check out next. - -__The Simulation Route__ - -For the imaging instruments, **PyAutoReduce**'s simulation strategy is *injection into -real frames*: real exposures carry cosmic rays, sky, bad pixels and PSF wings for free, -and no maintained raw-frame simulator exists for HST/Keck imaging anyway. ALMA is the one -instrument where the calculus flips: `simobserve` is observatory-supported, actively -maintained, and models the things that matter — the array configuration's uv sampling, -the integration cadence, and atmospheric + system-temperature thermal noise via a pwv -(precipitable water vapour) parameter. So for the visibility branch, fully-synthetic -simulation *is* the canonical route, and `inject_image` on an ALMA `TargetSpec` switches -acquisition to simobserve entirely — no archive, no uids, no calibrated MS required. - -The downstream stages neither know nor care: a simulated MS is split, extracted, -assembled and packaged by exactly the code paths as real data, which is what makes this a -test of the pipeline and not just of the simulator. - -__Imports__ - -`skymodel_fits` and `simobserve_kwargs` are public helpers of the simulate stage — we -demonstrate both standalone below before letting `reduce_target` drive them for real. -""" -import json -import sys -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np - -from autoreduce import TargetSpec, reduce_target -from autoreduce.visibilities.simulate import simobserve_kwargs, skymodel_fits - -try: - import casatools # noqa: F401 - import casatasks # noqa: F401 (provides simobserve and split) -except ImportError: - print( - "Modular CASA is not installed. The simulator needs the pip-installable " - "CASA packages:\n\n pip install casatools casatasks\n\nExiting cleanly." - ) - sys.exit(0) - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: simobserve -writes its project directory relative to the working directory, and the pipeline manages -that by changing directory into scratch internally — relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" -OUTPUT_ROOT = WORKSPACE / "output" - -NAME = "alma_sim_ring" -RA, DEC = 137.0, 2.1 # an arbitrary field visible from the ALMA site - -out_dir = OUTPUT_ROOT / NAME -out_dir.mkdir(parents=True, exist_ok=True) - -""" -__The Source Image__ - -The input is a plain 2-D numpy array in **Jy per pixel** — the same flux contract as JWST -injection, and deliberately free of any lensing-library dependency. To evoke what ALMA -actually sees toward a lensed DSFG — an Einstein ring with a compact bright region — we -compose two analytic pieces on a radial grid r (arcsec from centre): - -- a thin ring: I_ring(r) = exp( -(r - r_ring)^2 / (2 w^2) ), with r_ring = 0.45", - w = 0.06" — the smeared image of a compact source near the caustic; -- a compact core: I_core(r) = exp( -1.678 r / r_eff ), an exponential (Sersic n=1) - profile with r_eff = 0.08", where 1.678 is the n=1 Sersic constant that makes r_eff the - half-light radius. - -The sum is normalised so the array total equals `TOTAL_FLUX_JY` exactly — the number the -recovery check at the end must reproduce. At 0.02"/pixel the 129x129 field spans 2.6", -comfortably containing the ring. -""" -TOTAL_FLUX_JY = 0.02 # total source flux (typical of a bright lensed DSFG continuum) -PIXEL_SCALE = 0.02 # arcsec / pixel of the input image -SHAPE = (129, 129) - -yy, xx = np.mgrid[0 : SHAPE[0], 0 : SHAPE[1]] -cy, cx = SHAPE[0] // 2, SHAPE[1] // 2 -r = np.hypot(yy - cy, xx - cx) * PIXEL_SCALE - -ring = np.exp(-((r - 0.45) ** 2) / (2.0 * 0.06**2)) # thin Einstein-ring-like annulus -core = np.exp(-1.678 * r / 0.08) # compact exponential core -image = ring + 0.5 * core -image = TOTAL_FLUX_JY * image / image.sum() # normalise: sum(image) == TOTAL_FLUX_JY - -from astropy.io import fits - -input_path = out_dir / "input_ring_jy.fits" -fits.PrimaryHDU(image.astype(np.float32)).writeto(input_path, overwrite=True) -print(f"Wrote input source image ({image.sum():.4f} Jy total) to {input_path}") - -plt.figure(figsize=(5, 5)) -plt.imshow(np.arcsinh(image / image.max() * 100.0), origin="lower", cmap="magma") -plt.title(f"input source ({TOTAL_FLUX_JY} Jy, Jy/pixel)") -input_png = out_dir / "plots" -input_png.mkdir(exist_ok=True) -input_png = input_png / "input_source.png" -plt.savefig(input_png, dpi=150, bbox_inches="tight") -plt.close() -print(f"Saved input-source plot to {input_png.resolve()}") - -""" -__Sky Model__ - -simobserve does not consume a bare 2-D image: it wants a FITS cube with four world axes — -RA---SIN, DEC--SIN, STOKES, FREQ — carrying the pixel scale, sky position and observing -frequency in its WCS, with BUNIT = Jy/pixel. The public `skymodel_fits` helper performs -exactly that wrapping (the pipeline calls it internally); we demonstrate it standalone so -you can inspect the header it builds. Likewise `simobserve_kwargs` shows you the exact -headless simobserve call the pipeline will make, as a pure, inspectable dict — no -surprises hidden behind the one-call API. -""" -demo_skymodel = skymodel_fits( - input_image=image, # the Jy/pixel array above - pixel_scale=PIXEL_SCALE, # arcsec / pixel, written into the WCS - ra=RA, # sky position of the model centre (deg) - dec=DEC, - freq_ghz=230.0, # observing frequency for the FREQ axis - out_path=out_dir / "skymodel_demo.fits", -) -header = fits.getheader(demo_skymodel) -print( - f"skymodel axes: {[header[f'CTYPE{i}'] for i in (1, 2, 3, 4)]}, " - f"BUNIT={header['BUNIT']!r}" -) - -""" -__Simulation Dials__ - -The `alma_sim_*` dials on `TargetSpec` map one-to-one onto simobserve's own parameters. -Setting `inject_image` (plus its mandatory `inject_pixel_scale`) on an ALMA spec is the -switch that replaces archive acquisition with simulation — `alma_uids` / `alma_field` / -`alma_spws` are not required in this mode, because the simulated MS has exactly one field -and one spectral window. - -The `alma_sim_pwv_mm` dial deserves the highlight: it sets the precipitable water vapour -of the simulated atmosphere, which (with the system temperature model) sets the thermal -noise level. **`alma_sim_pwv_mm=0` disables noise entirely** — a noiseless simulation, -invaluable when you want to test the chain's arithmetic (flux recovery below becomes -exact to numerical precision) rather than its statistics. -""" -spec = TargetSpec( - name=NAME, # products land at output// - ra=RA, # phase centre of the simulated observation (deg) - dec=DEC, - instrument="alma", # the visibility-domain adapter - inject_image=str(input_path), # the Jy/pixel source image -> switches acquire to simobserve - inject_pixel_scale=PIXEL_SCALE, # arcsec / pixel of that image (required with inject_image) - alma_sim_antennalist="alma.cycle8.3.cfg", # antenna configuration file (a mid-compact 12-m config) - alma_sim_totaltime_s=600.0, # total on-source observing time (s) - alma_sim_integration_s=10.0, # correlator integration (dump) time (s) - alma_sim_freq_ghz=230.0, # observing frequency (Band 6 continuum) - alma_sim_pwv_mm=0.5, # atmosphere: 0.5 mm pwv thermal noise (0 = noiseless) -) - -""" -__Run The Simulation__ - -One call. Under the hood: `skymodel_fits` wraps the image, simobserve synthesises the -observation into a `_sim` project directory in the work area, and then — this is -the point — the *same* split, extract, assemble and package stages as `start_here.py` -turn the simulated MS into the product triplet. The simulated dataset gets uid "sim", -field "0", spw "0" in provenance and sidecar names. -""" -print( - "\nRunning simobserve + the visibility chain (a couple of minutes of CASA " - "time for the simulation; the chain itself takes seconds)..." -) - -record = reduce_target( - spec, - cache_root=CACHE_ROOT, # unused in simulation mode (nothing is downloaded) - output_root=OUTPUT_ROOT, # products land at output/alma_sim_ring/ -) - -print(f"\nSimulation + reduction complete. Products in {out_dir}") -print(f"packaged {record['package']['n_visibilities']} visibilities") - -""" -__The Inject Block__ - -A synthetic dataset must never masquerade as real data. The provenance record therefore -carries an `inject` block stating the source ("simobserve"), the sky model, the injected -total flux, and every simulation dial — including whether thermal noise was on. Any -consumer of `reduction.json` can (and should) check for this block before treating a -dataset as an observation. -""" -print("\ninject block:") -print(json.dumps(record["inject"], indent=2)) - -""" -__Flux Recovery__ - -The closure test. For a compact source, the visibility amplitude at the shortest -baselines approaches the total flux: a baseline much shorter than 1/theta_source barely -resolves the source, so its visibility's real part is (nearly) the full flux density. -We therefore average the real part over the shortest 5% of baselines and compare it to -the injected total. With thermal noise at pwv 0.5 mm and 10 minutes of integration, -agreement to within a few percent is expected; re-run with `alma_sim_pwv_mm=0.0` and the -ratio snaps to 1 at numerical precision. -""" -visibilities = fits.getdata(out_dir / "data.fits") -uv_wavelengths = fits.getdata(out_dir / "uv_wavelengths.fits") -noise_map = fits.getdata(out_dir / "noise_map.fits") - -uv_dist = np.hypot(uv_wavelengths[:, 0], uv_wavelengths[:, 1]) -short = uv_dist < np.percentile(uv_dist, 5.0) -recovered_jy = float(np.mean(visibilities[short, 0])) - -print(f"\ninjected total flux : {TOTAL_FLUX_JY:.4f} Jy") -print(f"short-baseline : {recovered_jy:.4f} Jy") -print(f"recovery ratio : {recovered_jy / TOTAL_FLUX_JY:.3f}") - -""" -__UV Coverage__ - -The uv coverage here is not the G09v1.40 coverage — it is whatever the chosen antenna -configuration and hour-angle range produced, which is precisely why simulation is useful: -change `alma_sim_antennalist` to a more extended configuration and watch the coverage -(and the resolution of the dirty image below) transform, before you ever propose for the -real thing. -""" -plot_dir = out_dir / "plots" -u_klambda, v_klambda = uv_wavelengths[:, 0] / 1e3, uv_wavelengths[:, 1] / 1e3 -plt.figure(figsize=(6, 6)) -plt.scatter(u_klambda, v_klambda, s=0.5, lw=0, alpha=0.5) -plt.scatter(-u_klambda, -v_klambda, s=0.5, lw=0, alpha=0.5) -plt.xlabel(r"u [k$\lambda$]") -plt.ylabel(r"v [k$\lambda$]") -plt.gca().set_aspect("equal") -plt.title(f"simulated uv coverage ({visibilities.shape[0]} visibilities)") -uv_png = plot_dir / "uv_coverage.png" -plt.savefig(uv_png, dpi=150, bbox_inches="tight") -plt.close() -print(f"\nSaved uv-coverage plot to {uv_png.resolve()}") - -""" -__Load In PyAutoLens__ - -The consumer-side round trip, exactly as for real data — which is the point: a simulated -dataset that loads and images identically to a real one is a simulated dataset you can -use to rehearse an entire modeling workflow. The dirty image should show your ring. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "\nPyAutoLens is not installed, so the dirty-image check is skipped " - "(pip install autolens). The simulated dataset itself is complete." - ) - -if al is not None: - real_space_mask = al.Mask2D.circular( - shape_native=(128, 128), # matches the 2.6" input field at 0.02"/pixel - pixel_scales=0.02, - radius=1.2, # arcsec — contains the 0.45"-radius ring - ) - dataset = al.Interferometer.from_fits( - data_path=out_dir / "data.fits", - noise_map_path=out_dir / "noise_map.fits", - uv_wavelengths_path=out_dir / "uv_wavelengths.fits", - real_space_mask=real_space_mask, - transformer_class=al.TransformerNUFFT, - ) - dirty = np.asarray(dataset.dirty_image.native) - peak_over_rms = float(np.max(np.abs(dirty)) / np.std(dirty)) - print(f"\ndirty-image peak/rms = {peak_over_rms:.1f}") - - plt.figure(figsize=(6, 6)) - plt.imshow(dirty, origin="lower", cmap="magma") - plt.colorbar(label="dirty-image intensity") - plt.title(f"simulated ring: dirty image (peak/rms = {peak_over_rms:.1f})") - dirty_png = plot_dir / "dirty_image.png" - plt.savefig(dirty_png, dpi=150, bbox_inches="tight") - plt.close() - print(f"Saved dirty-image plot to {dirty_png.resolve()}") - -""" -__Real-MS Injection__ - -Honesty section. There is a second simulation mode in the literature: predicting model -visibilities at the (u, v) points of a **real** measurement set and adding them to (or -replacing) its data — the interferometric analogue of source injection into real imaging -frames, with the real observation's calibration systematics carried along for free. CASA -supports the ingredients (the guide "Fit an arbitrary sky model to an existing MS" -documents predicting a model into MODEL_DATA and manipulating the data columns), and -published weight checks are built on similar machinery. - -**PyAutoReduce** defers this mode deliberately: it needs Fourier prediction at arbitrary -uv points and phase-centre bookkeeping that simobserve mode does not, and simobserve -already answers the pipeline-validation question this script exists for. Only the -simobserve route exists today — do not look for a real-MS injection dial, there isn't -one. - -__Wrap Up__ - -You simulated an ALMA observation of a ring + core source you specified to the microjansky, -watched the identical visibility chain reduce it, verified the injected flux came back at -the shortest baselines, and imaged the result through **PyAutoLens**. - -Good places to checkout next: - -- `scripts/alma/start_here.py` — the same chain on real G09v1.40 data. -- `scripts/alma/step_by_step.py` — each stage by hand, including the smearing budget and - the weight audit (both equally applicable to simulated data — try them on this MS). -- `autolens_workspace/scripts/interferometer/` — fit a lens model to the visibilities you - just simulated. -""" - -""" -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/alma/start_here.py b/workspace/scripts/alma/start_here.py deleted file mode 100644 index 823b7eb..0000000 --- a/workspace/scripts/alma/start_here.py +++ /dev/null @@ -1,517 +0,0 @@ -""" -Start Here: ALMA Visibilities -============================= - -ALMA is the premier instrument for observing strongly lensed dusty star-forming galaxies -(DSFGs): submillimetre-bright sources whose rest-frame far-infrared emission ALMA resolves -into Einstein rings and arcs at resolutions down to tens of milliarcseconds. The canonical -example is SDP.81, whose ~30 mas Einstein ring from the ALMA Long Baseline Campaign -(ALMA Partnership 2015, https://arxiv.org/abs/1503.02652) became the community's benchmark -lens dataset, and samples of tens of lensed DSFGs have been modeled from Herschel and SPT -selections (Bussmann et al. 2015, https://arxiv.org/abs/1504.05256; Spilker et al. 2016, -https://arxiv.org/abs/1604.05723). - -This script runs the **PyAutoReduce** visibility branch end-to-end on the validation anchor -G09v1.40 (ALMA project 2016.1.00282.S), producing the `al.Interferometer.from_fits` product -triplet that **PyAutoLens** models directly in the uv plane. In about 20 minutes of reading -(and a few minutes of compute, given a calibrated measurement set on disk) you will have a -modeling-ready interferometric dataset and understand every choice that produced it. - -Unlike the imaging instruments (HST, JWST, Keck), the ALMA reduction's product is not an -image: it is the visibilities themselves. The first two sections explain why that is the -right product for lens modeling, and why obtaining the input data involves one manual step -that **PyAutoReduce** is deliberately honest about. - -__Contents__ - -- **Why Visibilities:** Why lens modeling fits visibilities rather than CLEAN images. -- **The Archive Reality:** The ALMA archive does not serve calibrated visibilities — what you actually do. -- **Imports:** Import **PyAutoReduce** and the other libraries the script needs. -- **Paths:** Anchor every path to the workspace root. -- **The Target G09v1.40:** The Herschel-selected lensed DSFG this reduction is validated on. -- **Calibrated Measurement Set:** Locate the calibrated MS directory (or print acquisition guidance and exit). -- **Target Spec:** Build the `TargetSpec` with every ALMA dial explained. -- **The Visibility Branch:** The four stages: split, extract, assemble, package. -- **Run The Reduction:** Call `reduce_target` and let the branch run. -- **Provenance:** Walk the returned record: acquisition, splits, assembly counts, products. -- **Stokes I And The Noise Map:** The polarization combine formula and sigma = 1/sqrt(WEIGHT). -- **The Product Triplet:** The three (Nvis, 2) FITS files and their diagnostic sidecars. -- **UV Coverage:** Plot the uv-plane sampling of the combined dataset. -- **Load In PyAutoLens:** `al.Interferometer.from_fits` plus a dirty-image sanity check. -- **Wrap Up:** Summary and good places to check out next. - -__Why Visibilities__ - -An interferometer never measures an image. Each antenna pair measures a visibility — the -complex cross-correlation of the incoming wavefront — which, by the van Cittert-Zernike -theorem, samples one Fourier component of the sky brightness at the spatial frequency -(u, v) set by the projected baseline in wavelengths. Earth rotation sweeps each baseline -through an elliptical uv track, but the sampling is always incomplete: the "image" you see -in an ALMA press release is the sky convolved with the dirty beam, deconvolved by the -non-linear CLEAN algorithm (the textbook treatment is Thompson, Moran & Swenson 2017). - -For lens modeling that matters in two ways: - -- **CLEAN image noise is correlated between pixels.** The gridding and deconvolution that - produce a CLEAN map couple neighbouring pixels, so a pixel-by-pixel chi-squared against a - CLEAN image uses a wrong (and practically intractable) likelihood. - -- **CLEAN is non-linear.** Its artefacts depend on the source structure itself, so there is - no clean way to forward-model them. - -The visibilities have neither problem: each is an independent measurement with (near) -Gaussian noise, so the likelihood of a lens model — Fourier transform the model image, -compare to the data at the measured (u, v) points — is exact and well defined. This is why -the field moved to fitting visibilities directly: Rybak et al. 2015 -(https://arxiv.org/abs/1503.02025) reconstructed SDP.81's source at sub-50 pc resolution in -visibility space, Dye et al. 2018 (https://arxiv.org/abs/1705.05413) compared image-plane -and visibility-plane fits of the same data explicitly, and Powell et al. 2021 -(https://arxiv.org/abs/2005.03609) showed the full ~10^8-10^9 visibility problem is -tractable with NUFFT methods — the same transform **PyAutoLens** uses. The flagship science -this enables includes the 6.9-sigma detection of a ~10^9 solar-mass subhalo in SDP.81 from -its visibilities alone (Hezaveh et al. 2016, https://arxiv.org/abs/1601.01388). - -**PyAutoReduce** therefore never makes an image from ALMA data. Its product is the -visibility triplet **PyAutoLens** fits directly; CLEAN/tclean maps remain useful as -*diagnostics* (you will make a dirty image at the end of this script to sanity-check the -data), never as the science product. - -__The Archive Reality__ - -Here is the honest part. The ALMA Science Archive does **not** serve calibrated -visibilities as a plain download. What you can download anonymously are the raw data -(ASDM format) and the calibration/QA2 product tarballs. Turning those into a calibrated -measurement set (MS) requires running the observatory's `scriptForPI.py` restore script -*inside the CASA version that originally processed the data* — the tarball names record -which version — because the calibration tables are not portable across CASA releases -(see https://almascience.eso.org/processing/science-pipeline and the ALMA Knowledgebase -article "How do I obtain a file of calibrated visibilities"). Version-matching the restore -is the classic ALMA footgun, and automating it is deliberately out of scope for -**PyAutoReduce** today. - -So the canonical pipeline input is a **calibrated MS directory**, however obtained: - -- an ARC delivery (the EU ARC "CalMS" service, EA/NA helpdesk requests, or NRAO's SRDP - service for Cycle 5+ pipeline-calibrated data), or -- your own local `scriptForPI.py` restore of the downloaded tarballs, at the matching - CASA version. - -**PyAutoReduce** automates the archive *download* (`query_project`, -`download_product_tarballs`) and, if a downloaded tarball happens to contain a calibrated -MS (some ARC deliveries do), extracts it. When the tarballs contain only restore inputs — -the standard case — the pipeline raises loudly with step-by-step restore guidance rather -than pretending it produced calibrated data. You will see that guidance printed below if -no MS is on disk. - -__Imports__ - -We import **PyAutoReduce**'s two-name public API (`TargetSpec`, `reduce_target`) plus the -ALMA acquisition helpers used for the guidance path. The heavy CASA dependencies -(`casatools`, `casatasks` — the pip-installable modular CASA) are imported by the pipeline -inside its stage functions, but this script checks for them up front so it can fail with a -friendly message instead of a mid-reduction traceback. -""" -import os -import sys -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np - -from autoreduce import TargetSpec, reduce_target -from autoreduce.acquire import alma as alma_acquire - -try: - import casatools # noqa: F401 (modular CASA: the `table` tool the extract stage uses) - import casatasks # noqa: F401 (modular CASA: the `split` task the split stage uses) -except ImportError: - print( - "Modular CASA is not installed. The ALMA visibility branch needs the " - "pip-installable CASA packages:\n\n" - " pip install casatools casatasks\n\n" - "No monolithic CASA installation is required — see scripts/alma/step_by_step.py " - "for why the modular route is the right one. Exiting cleanly." - ) - sys.exit(0) - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: CASA tasks write -scratch directories and logs relative to the working directory, and the pipeline changes -directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # archive tarball downloads (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -""" -__The Target G09v1.40__ - -G09v1.40 (H-ATLAS J085358.9+015537, z ~ 2.09) is a strongly lensed dusty star-forming -galaxy from the Herschel-ATLAS survey. It was found by the submillimetre flux-selection -technique of Negrello et al. 2010 (Science 330, 800): at 500 micron, essentially every -extragalactic source brighter than ~100 mJy is either a blazar or a gravitational lens, -because the steep DSFG number counts make unlensed sources that bright vanishingly rare. -H-ATLAS turned this into candidate lists of ~80 lensed DSFGs (Negrello et al. 2017, -MNRAS 465, 3558), with uv-plane lens models following (see e.g. Enia et al. 2018, -MNRAS 475, 3467, from SMA visibilities). - -G09v1.40 itself was observed in ALMA Band 6 under project 2016.1.00282.S and modeled with -those data by Butler et al. 2021 (https://arxiv.org/abs/2104.10077), including OH+ and -CO(9-8) lines alongside the continuum this script reduces. - -One caution when reading the literature on this source: Yang et al. 2019 (A&A 624, A138) -present sub-kpc CO and H2O lens modeling of **G09v1.97** — a *different* H-ATLAS lens with -an easily-confused name. Do not carry numbers between the two. -""" -NAME = "alma_g09v140" -PROJECT = "2016.1.00282.S" # the ALMA project code -FIELD = "G09v1.40" # the science field name inside the MS -UIDS = ("A002_Xb9b1b9_X3046", "A002_Xb99cbd_X2456") # the two execution blocks (one MS each) -SPWS = ("1", "2") # the line-free continuum spectral windows -WIDTH = 240 # channels averaged per output channel -RA, DEC = 133.49542, 1.59367 # H-ATLAS J085358.9+015537 - -""" -__Calibrated Measurement Set__ - -The reduction consumes a directory of calibrated measurement sets, one per execution -block, in the standard delivered layout `uid___.ms.split.cal`. Point the path below -at yours (or set the `AUTOREDUCE_ALMA_MS_DIR` environment variable). - -If no MS directory is present, we print the acquisition guidance — including the exact -archive-download idiom and the restore instructions — and exit cleanly rather than crash. -The download idiom, for reference (network + ~tens of GB; run it deliberately): - - from autoreduce.acquire.alma import query_project, download_product_tarballs - - query_project("2016.1.00282.S") # one archive row per member OUS product - download_product_tarballs( # the scriptForPI restore inputs - "2016.1.00282.S", CACHE_ROOT / "alma_g09v140" / "tarballs" - ) - -`reduce_target` itself runs the same download automatically when you give it -`alma_project_code` instead of `alma_ms_dir` — and raises with the restore guidance if the -tarballs contain no calibrated MS, which for standard product tarballs they will not. -""" -ALMA_MS_DIR = Path( - os.environ.get( - "AUTOREDUCE_ALMA_MS_DIR", WORKSPACE / "dataset" / "alma" / "g09v140_calibrated" - ) -) - -if not ALMA_MS_DIR.is_dir(): - print( - f"No calibrated measurement-set directory found at:\n\n {ALMA_MS_DIR}\n\n" - "The ALMA archive does not serve calibrated visibilities directly (see the\n" - "'Archive Reality' section above). To run this script for real, obtain the\n" - "calibrated MS for project 2016.1.00282.S via an ARC delivery (EU CalMS /\n" - "NRAO SRDP) or a local scriptForPI.py restore at the matching CASA version,\n" - "then place (or symlink) the uid___.ms.split.cal directories at the path\n" - "above, or set AUTOREDUCE_ALMA_MS_DIR to their location.\n" - ) - print("The pipeline's own guidance message for this project reads:\n") - print(alma_acquire.restore_guidance(PROJECT, CACHE_ROOT / NAME / "tarballs")) - print("\nExiting cleanly — nothing was downloaded or written.") - sys.exit(0) - -try: - ms_paths = alma_acquire.resolve_calibrated_ms(ALMA_MS_DIR, UIDS) -except FileNotFoundError as error: - print(f"{error}\n\nExiting cleanly — fix the MS directory layout and re-run.") - sys.exit(0) - -print(f"Calibrated measurement sets resolved in {ALMA_MS_DIR}:") -for path in ms_paths: - print(f" {path.name}") - -""" -__Target Spec__ - -A **PyAutoReduce** reduction is declared, not scripted: you build one frozen `TargetSpec` -and hand it to one function. For the visibility domain the imaging dials (cutout shape, -drizzle parameters, PSF shapes) are ignored; the ALMA dials below are the whole interface. - -The `alma_width=240` choice deserves a word: each of these spectral windows has 240 -channels, so averaging 240 channels per output channel collapses each spw to a single -continuum channel. That is the continuum default (`alma_width=0` would auto-collapse by -reading each spw's channel count), and it is what cuts the visibility count to something a -lens-modeling likelihood evaluates quickly — the classic averaging step of published -practice (Hezaveh et al. 2016, https://arxiv.org/abs/1601.01388, section 2). Averaging is -not free: it smears the response to off-centre emission. `step_by_step.py` computes the -bandwidth-smearing limit for this dataset explicitly. -""" -spec = TargetSpec( - name=NAME, # products land at output// - ra=RA, # target RA (deg) — provenance for the visibility branch - dec=DEC, # target Dec (deg) - instrument="alma", # selects the visibility-domain adapter and branch - alma_uids=UIDS, # execution blocks to reduce (one calibrated MS each) - alma_field=FIELD, # science field to isolate from each MS (they also hold calibrators) - alma_spws=SPWS, # continuum spectral windows to extract - alma_width=WIDTH, # channel-averaging width (240 = collapse these 240-channel spws) - alma_ms_dir=str(ALMA_MS_DIR), # the calibrated-MS directory (the canonical input) - alma_project_code=PROJECT, # recorded in provenance; drives archive download if ms_dir unset -) - -""" -__The Visibility Branch__ - -`reduce_target` dispatches on the instrument adapter's domain. For `instrument="alma"` -that is the visibility branch — four stages, none shared with the imaging pipeline: - -- **split** — `casatasks.split`, run twice per execution block: first to isolate the - science field (each MS also contains the bandpass/phase/flux calibrators), then per - spectral window to average channels by `width`. Flagged rows are dropped at this point - (`keepflags=False`), so no zero-weight placeholder rows survive into the products. - -- **extract** — `casatools.table` reads of the split MS columns: DATA (the complex - visibilities), UVW (baselines in metres), WEIGHT (the noise bookkeeping), CHAN_FREQ, - and the ANTENNA1/ANTENNA2/TIME/SCAN_NUMBER diagnostics. - -- **assemble** — pure numpy: convert UVW from metres to wavelengths (u * frequency / c), - combine the two polarizations into Stokes I, form the noise map from the weights, and - concatenate all (execution block, spw) pieces into the final arrays. - -- **package** — write the `(Nvis, 2)` product triplet, the per-block diagnostic sidecars - and `reduction.json`. - -Both split steps are idempotent — an existing output MS is reused — so re-running this -script after the first reduction costs seconds, not minutes. - -__Run The Reduction__ - -The first run splits each execution block's MS twice with CASA (a few minutes per MS, -depending on its size and your disk), then extraction and assembly take seconds. -Intermediate measurement sets land under `output/alma_g09v140/work/` and are reused on -re-runs. -""" -print( - f"\nRunning the visibility branch on {len(UIDS)} execution blocks x " - f"{len(SPWS)} spws (first run: a few minutes of CASA split per MS; " - f"re-runs reuse the split MS)..." -) - -record = reduce_target( - spec, - cache_root=CACHE_ROOT, # archive tarballs would cache here (unused with a local ms_dir) - output_root=OUTPUT_ROOT, # products land at output/alma_g09v140/ -) - -out_dir = OUTPUT_ROOT / NAME -print(f"\nReduction complete. Products in {out_dir}") - -""" -__Provenance__ - -The returned record (also written to `reduction.json` alongside the products) documents -every stage. Walk the highlights: - -- `acquire`: which measurement sets were consumed and from where (`source: "local"` here; - an archive-download run records tarball checksums instead). -- `split`: one block per (execution block, spw) with the resolved channel-averaging width. -- `assemble`: the visibility counts — including, importantly, - `n_visibilities_dropped_invalid`: rows where *neither* polarization carried positive - weight and finite data. **PyAutoReduce** drops these loudly and counts them, rather than - silently zero-filling them (a zero-weight visibility written as data would poison the - likelihood downstream). -- `package`: the product filenames and the contract they satisfy. -""" -print("\nacquire:") -print(f" source : {record['acquire']['source']}") -print(f" measurement sets : {record['acquire']['measurement_sets']}") - -print("split:") -for block in record["split"]["blocks"]: - print( - f" uid {block['uid']} spw {block['spw']} width {block['width']}" - ) - -print("assemble:") -print(f" n_visibilities : {record['assemble']['n_visibilities']}") -for label, prov in record["assemble"]["blocks"].items(): - print( - f" {label}: {prov['n_visibilities']} visibilities " - f"({prov['n_rows']} rows x {prov['n_channels']} channel(s), " - f"{prov['n_visibilities_dropped_invalid']} dropped invalid, " - f"{prov['n_antennas']} antennas, {prov['n_scans']} scans)" - ) - -print("package:") -print(f" products : {record['package']['products']}") -print(f" contract : {record['package']['contract']}") - -""" -__Stokes I And The Noise Map__ - -The split MS carries two parallel-hand polarizations (XX, YY). Continuum lens modeling -fits total intensity, so the assemble stage forms the weighted Stokes-I average per -visibility, with the MS weights (weight = 1/sigma^2 per complex visibility): - - I = (w_xx * XX + w_yy * YY) / (w_xx + w_yy) - sigma_I = 1 / sqrt(w_xx + w_yy) - -The same sigma applies to the real and imaginary parts — the MS weight is per complex -visibility — which is exactly the `(Nvis, 2)` noise-map shape `al.Interferometer` expects. -Stacking the two polarizations as independent visibilities was considered and rejected: -it doubles the visibility count (and hence the NUFFT cost of every likelihood evaluation) -for zero information gain over the weighted average. - -Two things are deliberately *absent* from this noise map: - -- **No Casertano correlated-noise factor.** The imaging pipelines multiply their RMS maps - by the Casertano R factor because drizzling resamples pixels and correlates their noise. - Visibilities are not resampled pixels — each is an independent sample — so no such - correction exists or is applied here. - -- **No independent noise estimate.** sigma = 1/sqrt(WEIGHT) trusts the weight column of - the calibrated MS. Whether that trust is justified depends on the CASA version history - of your data and whether `statwt` was run — the single most important data-quality - question for uv-plane modeling, treated in depth (with a verification you can run on - your own data) in `step_by_step.py`. -""" - -""" -__The Product Triplet__ - -Three FITS files, each a float64 array of shape (Nvis, 2), matching what -`al.Interferometer.from_fits` loads: - -- `data.fits` — the visibilities: column 0 the real part, column 1 the imaginary part (Jy). -- `uv_wavelengths.fits` — the (u, v) coordinates of each visibility, in wavelengths. -- `noise_map.fits` — sigma on the real and imaginary parts (equal by convention, above). - -Alongside them, per-(execution block, spw) diagnostic sidecars — `antennas_*.fits`, -`scans_*.fits`, `times_*.fits`, `frequencies_*.fits` — record which baselines, scans, -timestamps and channel frequencies each block contributed. They are not modeling inputs; -they are the paper trail for debugging (e.g. tracing a bad scan or antenna back through -the assembly). -""" -from astropy.io import fits - -visibilities = fits.getdata(out_dir / "data.fits") -uv_wavelengths = fits.getdata(out_dir / "uv_wavelengths.fits") -noise_map = fits.getdata(out_dir / "noise_map.fits") - -print(f"\ndata.fits : {visibilities.shape} (real / imaginary, Jy)") -print(f"uv_wavelengths.fits : {uv_wavelengths.shape} (u / v, wavelengths)") -print(f"noise_map.fits : {noise_map.shape} (sigma real / sigma imaginary)") -print( - f"median |V| = {np.median(np.hypot(visibilities[:, 0], visibilities[:, 1])):.4f} Jy, " - f"median sigma = {np.median(noise_map[:, 0]):.4f} Jy" -) - -""" -__UV Coverage__ - -The uv-plane sampling determines what the dataset can constrain: the longest baselines -set the resolution, the shortest set the largest recoverable scale, and the gaps are where -the lens model is doing pure interpolation. Plotting it is the interferometric equivalent -of glancing at your image before fitting it. (Each visibility also implies its complex -conjugate at (-u, -v); we plot both to show the familiar symmetric coverage.) -""" -plot_dir = out_dir / "plots" -plot_dir.mkdir(exist_ok=True) - -u_klambda = uv_wavelengths[:, 0] / 1e3 -v_klambda = uv_wavelengths[:, 1] / 1e3 - -plt.figure(figsize=(6, 6)) -plt.scatter(u_klambda, v_klambda, s=0.5, lw=0, alpha=0.5, label="sampled") -plt.scatter(-u_klambda, -v_klambda, s=0.5, lw=0, alpha=0.5, label="conjugate") -plt.xlabel(r"u [k$\lambda$]") -plt.ylabel(r"v [k$\lambda$]") -plt.title(f"{FIELD}: uv coverage ({visibilities.shape[0]} visibilities)") -plt.gca().set_aspect("equal") -plt.legend(markerscale=10, loc="upper right") -uv_png = plot_dir / "uv_coverage.png" -plt.savefig(uv_png, dpi=150, bbox_inches="tight") -plt.close() -print(f"\nSaved uv-coverage plot to {uv_png.resolve()}") - -""" -__Load In PyAutoLens__ - -The consumer-side round trip. **PyAutoReduce** never imports **PyAutoLens** (a hard -boundary — the reducer stays releasable on its own), so this load is the workspace's job. -Two modeling-side choices enter here that are *not* reduction products: - -- `real_space_mask` defines the sky region the source and lens are reconstructed within — - its pixel scale and extent are yours to choose against the uv coverage above. -- `transformer_class` selects the Fourier transform: `al.TransformerNUFFT` scales to large - visibility counts (the Powell et al. 2021 lesson, https://arxiv.org/abs/2005.03609). - -The dirty image — the direct inverse transform of the visibilities, no deconvolution — is -the honest quick-look: if the lensed source is real and bright, its peak stands far above -the residual rms. This mirrors the pipeline's own validation check on this dataset. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "\nPyAutoLens is not installed, so the final loading check is skipped " - "(pip install autolens). The reduction itself is complete — the product " - "triplet above is ready for any uv-plane modeling tool." - ) - -if al is not None: - real_space_mask = al.Mask2D.circular( - shape_native=(256, 256), # real-space grid the source/lens are evaluated on - pixel_scales=0.05, # arcsec / pixel — chosen for the ~0.1" scales the uv coverage reaches - radius=4.0, # arcsec — comfortably contains the ~1.5" diameter ring - ) - - dataset = al.Interferometer.from_fits( - data_path=out_dir / "data.fits", - noise_map_path=out_dir / "noise_map.fits", - uv_wavelengths_path=out_dir / "uv_wavelengths.fits", - real_space_mask=real_space_mask, - transformer_class=al.TransformerNUFFT, - ) - - dirty = np.asarray(dataset.dirty_image.native) - peak_over_rms = float(np.max(np.abs(dirty)) / np.std(dirty)) - print( - f"\nLoaded {dataset.data.shape[0]} visibilities into al.Interferometer; " - f"dirty-image peak/rms = {peak_over_rms:.1f} (expect >> 1 for a detected ring)." - ) - - plt.figure(figsize=(6, 6)) - plt.imshow(dirty, origin="lower", cmap="magma") - plt.colorbar(label="dirty-image intensity") - plt.title(f"{FIELD}: dirty image (peak/rms = {peak_over_rms:.1f})") - dirty_png = plot_dir / "dirty_image.png" - plt.savefig(dirty_png, dpi=150, bbox_inches="tight") - plt.close() - print(f"Saved dirty-image plot to {dirty_png.resolve()}") - -""" -__Wrap Up__ - -You reduced ALMA Band 6 observations of the lensed DSFG G09v1.40 from a calibrated -measurement set to a modeling-ready visibility dataset: field-split and channel-averaged -with CASA, extracted to numpy, combined to Stokes I with weights-derived noise, packaged -as the `(Nvis, 2)` triplet, and verified by a dirty-image quick look through -**PyAutoLens**. - -Good places to checkout next: - -- `scripts/alma/step_by_step.py` — every stage of the chain by hand: MS anatomy, the - smearing limit on channel averaging, the weight-recalibration story and how to verify - your weights against the visibility scatter. -- `scripts/alma/simulator.py` — simulate a realistic ALMA observation of a synthetic - source with CASA's simobserve and push it through the identical chain. -- `scripts/surveys/start_here.py` — fetch optical survey cutouts for colour context on - lens fields like this one, whose modeling data has no optical counterpart. -- `autolens_workspace/scripts/interferometer/` — model this dataset in the uv plane with - **PyAutoLens**. -""" - -""" -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/alma/step_by_step.py b/workspace/scripts/alma/step_by_step.py deleted file mode 100644 index f31d5de..0000000 --- a/workspace/scripts/alma/step_by_step.py +++ /dev/null @@ -1,526 +0,0 @@ -""" -ALMA Step By Step: Measurement Set To Visibilities -================================================== - -`start_here.py` ran the **PyAutoReduce** visibility branch as one `reduce_target` call. -This script runs the same chain by hand, stage by stage, on the same G09v1.40 dataset — -because for ALMA, unlike the imaging instruments, every stage is public API: the split, -extract and assemble modules are importable functions you can compose yourself. - -Along the way you will learn what a measurement set actually is, why the pipeline runs on -headless "modular" CASA, how far you may average channels before smearing bites (with the -limit computed for this dataset), and — most importantly — what the WEIGHT column really -contains and how to verify it against the visibility scatter before you trust it as your -noise map. Budget ~30 minutes of reading; the compute re-uses `start_here.py`'s split -measurement sets where present. - -__Contents__ - -- **Measurement Set Anatomy:** The tables and columns inside an MS directory. -- **Headless CASA:** Why the pipeline uses pip-installed casatools/casatasks, not the CASA shell. -- **Imports:** Import the visibility-branch modules directly. -- **Paths:** Anchor every path to the workspace root. -- **Calibrated Input:** Resolve the calibrated measurement sets (or exit with guidance). -- **Field Split:** `casatasks.split` pass one — isolate the science field. -- **Channel Averaging Split:** `casatasks.split` pass two — average channels by `width`. -- **Smearing Limits:** Compute the bandwidth-smearing cost of the chosen width (Bridle & Schwab). -- **Extract:** `casatools.table` reads into the `MsColumns` contract. -- **Weights:** What WEIGHT means, why archival weights need scrutiny, the statwt story. -- **Weight Verification:** Check sigma = 1/sqrt(WEIGHT) against the visibility scatter. -- **UV Wavelengths:** Baselines in metres to (u, v) in wavelengths, per channel. -- **Stokes I:** The weighted polarization combine, and zero-weight rows dropped loudly. -- **Concatenate And Package:** Assemble the blocks and write the product triplet. -- **Continuum Only:** What this branch deliberately does not do yet. -- **Wrap Up:** Summary and good places to check out next. - -__Measurement Set Anatomy__ - -A measurement set (MS) is not a file — it is a directory of binary tables (the CASA table -system), and it is the container every ALMA delivery ultimately becomes. The main table -has one row per (baseline, integration timestamp), with the columns this pipeline reads: - -- `DATA` — the complex visibilities, shaped (n_polarizations, n_channels, n_rows). A - calibrated `.ms.split.cal` delivery carries the calibrated science data here. (A raw MS - being calibrated grows a `CORRECTED_DATA` column beside it, and modeling workflows add - `MODEL_DATA`; after the observatory's final split, `DATA` is the calibrated column.) -- `UVW` — the projected baseline vector in metres, (3, n_rows). -- `WEIGHT` — the noise bookkeeping, (n_polarizations, n_rows): nominally 1/sigma^2 per - complex visibility. Its sibling `SIGMA` describes the *raw* data's per-datum noise; for - calibrated data `WEIGHT` is the authoritative one (more below). -- `ANTENNA1`, `ANTENNA2`, `TIME`, `SCAN_NUMBER` — which baseline, when, in which scan. - -Around the main table sit subtables: `SPECTRAL_WINDOW` (channel frequencies and widths -per spw — a typical ALMA observation carries four spws across the band), `ANTENNA`, -`FIELD` (the science target *and* the calibrators share one MS until you split), and -more. The full data model is documented at https://casadocs.readthedocs.io/ — see the -Measurement Set pages and the Data Weights notebook referenced below. - -__Headless CASA__ - -Historically this extraction recipe ran inside the interactive `casa` shell, with `tb` -and `split` existing only as globals injected into that session — fine at the telescope, -hopeless for a scripted, testable pipeline. CASA 6 solved this by shipping **modular** -packages on PyPI: `casatools` (the `table` tool and friends) and `casatasks` (`split`, -`simobserve`, ...), plain-Python importable and proven headless -(https://casadocs.readthedocs.io/). - -**PyAutoReduce** builds only on the modular route, with one careful boundary: extraction -of an *already-calibrated* MS has no CASA-version coupling — any recent modular CASA -works. The notorious version-matching constraint binds only the `scriptForPI.py` -calibration restore (see `start_here.py`, "The Archive Reality"), which is exactly the -step the pipeline leaves manual rather than pretending to automate. - -__Imports__ - -Because the visibility branch's stages are public modules, we import them directly: -`split` (the casatasks orchestration), `extract` (the casatools table reads), `assemble` -(pure numpy) and the interferometer packaging. casatools/casatasks themselves are -imported lazily inside those modules; we check for them up front to fail friendly. -""" -import os -import sys -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np - -from autoreduce.acquire import alma as alma_acquire -from autoreduce.package.interferometer import write_products -from autoreduce.visibilities.assemble import ( - assemble_ms_products, - concatenate, - stokes_i_combine, - uv_wavelengths_from_uvw, -) -from autoreduce.visibilities.extract import columns_from, getcol, num_channels_per_spw -from autoreduce.visibilities.split import resolve_width, split_field, split_spw - -try: - import casatools # noqa: F401 (the `table` tool behind the extract stage) - import casatasks # noqa: F401 (the `split` task behind the split stage) -except ImportError: - print( - "Modular CASA is not installed (pip install casatools casatasks). " - "Exiting cleanly." - ) - sys.exit(0) - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: CASA tasks write -scratch and logs relative to the working directory, so relative paths would break. - -This script works in its own output folder, but points its split work directory at the -one `start_here.py` uses — the splits are idempotent, so if you ran `start_here.py` first -the CASA passes below complete instantly by reusing its measurement sets. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -OUTPUT_ROOT = WORKSPACE / "output" - -NAME = "alma_g09v140" -PROJECT = "2016.1.00282.S" -FIELD = "G09v1.40" -UIDS = ("A002_Xb9b1b9_X3046", "A002_Xb99cbd_X2456") -SPWS = ("1", "2") -WIDTH = 240 - -WORK_DIR = OUTPUT_ROOT / NAME / "work" # shared with start_here.py (idempotent splits) -STEPS_OUT = OUTPUT_ROOT / f"{NAME}_steps" # this script's own packaged products -WORK_DIR.mkdir(parents=True, exist_ok=True) - -""" -__Calibrated Input__ - -As in `start_here.py`: the canonical input is a directory of calibrated measurement sets -(`uid___.ms.split.cal`), obtained via an ARC delivery or a local scriptForPI restore -at the matching CASA version. If it is absent we print the pipeline's guidance and exit -cleanly. To keep the walk-through readable we work on the *first* execution block only; -the full pipeline simply loops what you see here over every (uid, spw). -""" -ALMA_MS_DIR = Path( - os.environ.get( - "AUTOREDUCE_ALMA_MS_DIR", WORKSPACE / "dataset" / "alma" / "g09v140_calibrated" - ) -) - -if not ALMA_MS_DIR.is_dir(): - print( - f"No calibrated measurement-set directory at {ALMA_MS_DIR} " - f"(or set AUTOREDUCE_ALMA_MS_DIR).\n" - ) - print(alma_acquire.restore_guidance(PROJECT, WORKSPACE / "cache" / NAME / "tarballs")) - print("\nExiting cleanly — nothing was written.") - sys.exit(0) - -try: - ms_paths = alma_acquire.resolve_calibrated_ms(ALMA_MS_DIR, UIDS) -except FileNotFoundError as error: - print(f"{error}\n\nExiting cleanly.") - sys.exit(0) - -uid = UIDS[0] -parent_ms = ms_paths[0] -print(f"Working execution block: {parent_ms.name}") - -""" -Before splitting anything, interrogate the parent MS's spectral layout. `NUM_CHAN` per -spectral window is what the `width=0` continuum default reads to collapse each spw fully; -here we print it so the `width=240` choice below is transparent — spws 1 and 2 each carry -240 channels, so one width value collapses each to a single continuum channel. -""" -num_chan = num_channels_per_spw(parent_ms) -print(f"NUM_CHAN per spectral window: {num_chan.tolist()}") - -""" -__Field Split__ - -Pass one of `casatasks.split` isolates the science field. A delivered MS interleaves the -science target with its calibrators — bandpass (a bright quasar), the interleaved phase -calibrator, the flux standard — because that is how the observation was scheduled. The -calibration solved from those sources is already *applied* to the data (this is a -calibrated MS); their visibilities themselves are not lens-modeling data, so the first -split keeps only rows whose field is G09v1.40. - -Two dials on every split in this pipeline deserve their comment: - -- `datacolumn="data"` — calibrated `.ms.split.cal` deliveries carry the calibrated - visibilities in `DATA` (the observatory's own final split already moved them there). -- `keepflags=False` — rows flagged bad by the calibration pipeline are *dropped*, not - carried as zero-weight placeholders. Flagged placeholder rows are a classic source of - silent dataset bloat and downstream weight confusion; the pipeline refuses to carry - them. - -The split is idempotent: if the output MS already exists (e.g. from `start_here.py`), it -is reused. Interrupted runs are safe too — the split writes to a `.partial` directory -renamed into place only on success. -""" -print(f"\nSplitting field {FIELD!r} out of {parent_ms.name} (idempotent)...") -field_ms = split_field(parent_ms, uid, FIELD, WORK_DIR) -print(f" field MS: {field_ms.name}") - -""" -__Channel Averaging Split__ - -Pass two splits out each spectral window and averages its channels by `width`. This is -the step that makes uv-plane lens modeling computationally tractable: a raw dataset holds -~10^7-10^9 visibilities (rows x channels), and every likelihood evaluation must Fourier -transform the model to all of them. Published practice has always averaged first — -Hezaveh et al. 2016 (https://arxiv.org/abs/1601.01388, section 2) averaged the SDP.81 -long-baseline data in exactly this way before their subhalo search — although modern -NUFFT solvers can now handle the un-averaged problem when needed (Powell et al. 2021, -https://arxiv.org/abs/2005.03609). - -`resolve_width` turns the user dial into a concrete per-spw width: a positive value -passes through, `0` means "collapse the whole spw" by reading its channel count. -""" -spw_ms_by_spw = {} -for spw in SPWS: - width = resolve_width(WIDTH, spw, num_chan) - print(f"Splitting spw {spw} at width {width} (idempotent)...") - spw_ms_by_spw[spw] = split_spw(field_ms, uid, FIELD, spw, width, WORK_DIR) - print(f" spw MS: {spw_ms_by_spw[spw].name}") - -""" -__Smearing Limits__ - -Averaging is not free. Averaging channels assigns one mean frequency to visibilities that -were measured across a band of frequencies — but (u, v) scales with frequency, so the -average smears the response to emission *away from the phase centre* radially outward. -This is **bandwidth smearing**; its time-domain sibling (averaging integrations while the -Earth rotates the baselines) smears azimuthally. The standard reference is Bridle & -Schwab 1999 (ASP Conf. Ser. 180, 371): to first order the radial smearing extent for a -source at offset theta from the phase centre is - - delta_theta ~ (delta_nu / nu) * theta - -and it decorrelates (attenuates) long-baseline amplitudes when delta_theta becomes -comparable to the synthesized beam. Since lensed arcs sit ~1 arcsecond off-centre, this -hard-limits how far you may average on long-baseline data. - -Let's compute the worked exercise for *this* dataset and *this* width choice: read the -averaged channel width from the split MS, take the observing frequency and the longest -baseline, and compare the smearing extent at a 1" offset against the synthesized-beam -scale ~ 1/(max uv distance). -""" -OFFSET_ARCSEC = 1.0 # a typical arc offset from the phase centre -ARCSEC_PER_RAD = 180.0 / np.pi * 3600.0 - -for spw, spw_ms in spw_ms_by_spw.items(): - chan_width_hz = float( - np.mean(np.abs(np.atleast_1d(np.squeeze(getcol(spw_ms, "SPECTRAL_WINDOW", "CHAN_WIDTH"))))) - ) - chan_freq_hz = np.atleast_1d( - np.squeeze(getcol(spw_ms, "SPECTRAL_WINDOW", "CHAN_FREQ")) - ).astype(float) - nu0_hz = float(np.mean(chan_freq_hz)) - uvw = getcol(spw_ms, "", "UVW").astype(float) - max_uv_wavelengths = float(np.max(np.hypot(uvw[0], uvw[1])) * nu0_hz / 299792458.0) - - beam_arcsec = (1.0 / max_uv_wavelengths) * ARCSEC_PER_RAD - smear_arcsec = (chan_width_hz / nu0_hz) * OFFSET_ARCSEC - print( - f"\nspw {spw}: averaged channel width = {chan_width_hz / 1e6:.1f} MHz at " - f"{nu0_hz / 1e9:.1f} GHz (fractional bandwidth {chan_width_hz / nu0_hz:.2e})" - ) - print( - f" longest baseline = {max_uv_wavelengths / 1e3:.0f} klambda -> beam scale " - f"~ {beam_arcsec:.3f} arcsec" - ) - print( - f" radial smearing at {OFFSET_ARCSEC:.1f}\" offset ~ {smear_arcsec:.4f} arcsec " - f"= {smear_arcsec / beam_arcsec:.2f} of the beam scale" - ) - -""" -Read the last line for each spw: if the smearing extent is a small fraction of the beam -scale, the width choice is safe for emission at that offset; as it approaches unity you -are actively blurring your arcs and biasing the lens model. Re-run this cell with your -own `width` and offset before averaging your own data — this five-line estimate is the -difference between a deliberate averaging choice and a silent resolution loss. - -(Time averaging — `timebin` in CASA's split/mstransform — trades against the azimuthal -version of the same limit. **PyAutoReduce** deliberately exposes no time-averaging dial -yet; channel collapse alone tames the anchor dataset, and a time dial without its -smearing guard would invite exactly the silent loss described above.) - -__Extract__ - -With the per-spw measurement sets on disk, extraction is a set of `casatools.table` -column reads — no CASA tasks, no shell, just arrays. `columns_from` returns the frozen -`MsColumns` contract the assemble stage consumes, with shapes normalised so continuum -(one channel) and line (many channels) widths flow through one code path: - -- `data` complex (n_pol, n_chan, n_rows) -- `uvw` metres (3, n_rows) -- `weight` 1/sigma^2 (n_pol, n_rows) -- `chan_freq` Hz (n_chan,) -- `antenna1` / `antenna2` / `time` / `scan` (n_rows,) - -The antenna/time/scan columns are the diagnostic paper trail: the pipeline packages them -as per-block sidecars (`antennas_*.fits`, `scans_*.fits`, `times_*.fits`, -`frequencies_*.fits`) so a bad scan or antenna can be traced back after the fact. A -worked example of exactly this kind of extraction is the visread documentation -(https://mpol-dev.github.io/visread/), whose output arrays are precisely the contract -**PyAutoLens** consumes. -""" -columns_by_spw = {} -for spw, spw_ms in spw_ms_by_spw.items(): - columns = columns_from(spw_ms) - columns_by_spw[spw] = columns - n_pol, n_chan, n_rows = columns.data.shape - print( - f"spw {spw}: DATA ({n_pol} pol x {n_chan} chan x {n_rows} rows), " - f"{np.unique(np.stack((columns.antenna1, columns.antenna2))).size} antennas, " - f"{np.unique(columns.scan).size} scans" - ) - -""" -__Weights__ - -The WEIGHT column is where uv-plane lens modeling is won or lost, because it becomes the -noise map, and a mis-scaled noise map biases *every posterior width* your model reports. -The convention (casadocs, "Data Weights and Combination" notebook, -https://casadocs.readthedocs.io/) is: - -- `WEIGHT` = 1/sigma^2 per complex visibility, describing the calibrated data; -- `SIGMA` = the per-datum noise of the *raw* data (superseded once calibrated); -- weights are initialised proportional to (channel width x integration time) at import, - then rescaled by the system temperature and gain solutions when calibration is applied - with `calwt=True`; per-channel variants live in `WEIGHT_SPECTRUM`. -- channel averaging in `split` propagates the weights: averaging N channels sums their - weights, which is why the collapsed continuum visibilities above carry usefully large - weights. - -Why the scrutiny? The ALMA Knowledgebase article "How does CASA calculate the visibility -weights?" documents that the bookkeeping *changed across CASA versions* — only data -calibrated with CASA >= 4.2.2 initialises per-channel weights properly — so for archival -data the weights are guaranteed to be *relative* at best, not absolute 1/sigma^2. The -standard remedy is `statwt`, which recomputes WEIGHT/SIGMA empirically from the scatter -of the (line-free) visibilities themselves. Published uv-plane lens models treat this as -a first-class step: Hezaveh et al. 2016 (section 2) rescaled their weights from the -variance of difference visibilities, and Dye et al. 2018 -(https://arxiv.org/abs/1705.05413) recalibrated weights before modeling. - -**PyAutoReduce** extracts the weights as delivered and converts sigma = 1/sqrt(WEIGHT) — -it does not silently rescale data it cannot verify. The verification below is how you -check whether *your* MS needs `statwt` before you trust the noise map. - -__Weight Verification__ - -The cleanest empirical estimate of the true noise uses **difference visibilities** -(the Hezaveh et al. 2016 approach): subtract successive integrations on the *same -baseline* within the same scan. The sky (a constant continuum source) cancels in the -difference; what remains is pure noise with variance twice the per-visibility variance. -Comparing that scatter to the WEIGHT-predicted sigma gives a single ratio: - -- ratio ~ 1 : the weights are absolute; sigma = 1/sqrt(WEIGHT) is your noise map. -- ratio != 1 : the weights are only relative — run `statwt` (or rescale) before modeling. -""" -for spw, columns in columns_by_spw.items(): - a1, a2, t, scan = columns.antenna1, columns.antenna2, columns.time, columns.scan - order = np.lexsort((t, scan, a2, a1)) # sort by baseline, then scan, then time - same_pair = ( - (a1[order][1:] == a1[order][:-1]) - & (a2[order][1:] == a2[order][:-1]) - & (scan[order][1:] == scan[order][:-1]) - ) - data_sorted = columns.data[:, 0, :][:, order] # first channel, both polarizations - diff = data_sorted[:, 1:][:, same_pair] - data_sorted[:, :-1][:, same_pair] - scatter_sigma = float( - np.std(np.concatenate((diff.real.ravel(), diff.imag.ravel()))) / np.sqrt(2.0) - ) - w = columns.weight - predicted_sigma = float(np.mean(1.0 / np.sqrt(w[np.isfinite(w) & (w > 0.0)]))) - print( - f"spw {spw}: scatter-derived sigma = {scatter_sigma:.4f} Jy, " - f"WEIGHT-predicted sigma = {predicted_sigma:.4f} Jy, " - f"ratio = {scatter_sigma / predicted_sigma:.2f}" - ) - -""" -(Caveats worth knowing: the difference estimator assumes the source is constant between -successive integrations — true for continuum — and the comparison of means glosses over -weight variation across baselines; treat a ratio within a few tens of percent of unity as -healthy, and anything factors from unity as a statwt flag.) - -__UV Wavelengths__ - -The MS stores one metric baseline vector per row, but the Fourier plane is sampled in -*wavelengths*: the same physical baseline samples a different (u, v) at each channel -frequency. The conversion is simply - - u [wavelengths] = u [metres] * frequency / c - -and `uv_wavelengths_from_uvw` applies it per channel, returning (n_chan, n_rows, 2) — at -continuum width (one channel) that is one (u, v) per row, but the same function serves a -future line cube unchanged. Note only u and v are kept: the w term encodes non-coplanar -effects negligible over ALMA's arcsecond-scale fields of view. -""" -columns = columns_by_spw[SPWS[0]] -uv = uv_wavelengths_from_uvw(columns.uvw, columns.chan_freq) -uv_dist_klambda = np.hypot(uv[..., 0], uv[..., 1]).ravel() / 1e3 -print( - f"spw {SPWS[0]}: uv wavelengths {uv.shape}; baselines span " - f"{uv_dist_klambda.min():.0f}-{uv_dist_klambda.max():.0f} klambda" -) - -""" -__Stokes I__ - -The two parallel hands (XX, YY) both measure total intensity for an unpolarized source, -so the assemble stage forms the inverse-variance-weighted Stokes-I average per -visibility — the same estimator CASA's own Stokes conversion uses: - - I = (w_xx * XX + w_yy * YY) / (w_xx + w_yy) - sigma_I = 1 / sqrt(w_xx + w_yy) - -`stokes_i_combine` implements it with two loud edge-case rules: a hand contributes only -where its weight is positive *and* its datum finite (a non-finite visibility must not -leave its weight in the denominator, which would silently bias the average low), and -visibilities where *neither* hand contributes are flagged out via the returned `keep` -mask — dropped and counted, never zero-filled. There is no Casertano correlated-noise -factor here, and never will be: that correction exists for *resampled pixels* whose noise -drizzling correlates; visibilities are independent samples. -""" -stokes_i, sigma, keep = stokes_i_combine(columns.data, columns.weight) -print( - f"spw {SPWS[0]}: Stokes I {stokes_i.shape}; kept {int(np.count_nonzero(keep))} / " - f"{keep.size} visibilities ({int(keep.size - np.count_nonzero(keep))} dropped " - f"zero-weight/invalid); median sigma_I = {float(np.median(sigma[keep])):.4f} Jy" -) - -""" -__Concatenate And Package__ - -`assemble_ms_products` wraps the three operations you just saw (Stokes-I combine, uv -conversion, flatten) for one (uid, spw) block, and `concatenate` stacks the blocks into -the final dataset, keeping per-block provenance. `write_products` then writes the -`(Nvis, 2)` triplet — validating on the way out that every array is finite and the noise -map strictly positive (a zero or negative sigma is a corrupt product, and the packager -crashes loudly rather than shipping it) — plus whichever diagnostic sidecars we hand it. - -The result of this hand-built chain lands in its own folder; compare it against -`start_here.py`'s `output/alma_g09v140/` — for the shared execution block the numbers are -identical, because you just ran the same public functions the pipeline runs. -""" -sets, labels, sidecars = [], [], {} -for spw, cols in columns_by_spw.items(): - sets.append(assemble_ms_products(cols)) - labels.append(f"{uid}/spw{spw}") - tag = f"{uid}_spw_{spw}" - sidecars[f"antennas_{tag}"] = np.stack((cols.antenna1, cols.antenna2)) - sidecars[f"scans_{tag}"] = cols.scan - sidecars[f"times_{tag}"] = cols.time - sidecars[f"frequencies_{tag}"] = cols.chan_freq - -combined = concatenate(sets, labels) -print(f"\nConcatenated {len(sets)} blocks: {combined.provenance['n_visibilities']} visibilities") - -products = write_products( - STEPS_OUT, - combined.visibilities, - combined.uv_wavelengths, - combined.noise_map, - sidecars=sidecars, -) -print(f"Packaged products in {STEPS_OUT}: {products}") - -plot_dir = STEPS_OUT / "plots" -plot_dir.mkdir(exist_ok=True) -plt.figure(figsize=(6, 4)) -plt.hist(np.hypot(combined.uv_wavelengths[:, 0], combined.uv_wavelengths[:, 1]) / 1e3, bins=60) -plt.xlabel(r"uv distance [k$\lambda$]") -plt.ylabel("visibilities") -plt.title(f"{FIELD} ({uid}): baseline distribution") -hist_png = plot_dir / "uv_distance_hist.png" -plt.savefig(hist_png, dpi=150, bbox_inches="tight") -plt.close() -print(f"Saved baseline-distribution plot to {hist_png.resolve()}") - -""" -__Continuum Only__ - -Honesty section. This branch reduces **continuum** data: line-free spectral windows, -collapsed in frequency. What it deliberately does not do yet: - -- **Emission-line / cube extraction.** Lines (like G09v1.40's OH+ and CO(9-8), Butler et - al. 2021, https://arxiv.org/abs/2104.10077) need finer `width` values and *per-channel* - product sets — an `al.Interferometer` per channel. The `width` dial and the per-channel - `uv_wavelengths_from_uvw` machinery above already support it; the per-channel packaging - is deferred until a line-modeling dataset needs it. -- **Line-channel flagging inside an spw.** Choosing the line-free channels (inspect with - CASA's `listobs`/plotms before deciding your spws) remains your judgment call at spec - time — that is why `alma_spws` is a user dial and spws 0 and 3 of this project are not - in it. -- **statwt / self-calibration.** Weight recomputation and self-cal (see the CASA Guide - "First Look at Self Calibration" and Richards et al. 2022, ALMA Memo 620, - https://arxiv.org/abs/2207.05591 — bright lensed DSFGs are frequently self-calibrated) - belong to the calibration side of the fence, upstream of this pipeline's input - contract. The weight verification above tells you whether that upstream work is needed. - -__Wrap Up__ - -You ran the visibility branch by hand: two idempotent CASA splits (field isolation, then -channel averaging with a computed smearing budget), casatools extraction into plain -arrays, a weight audit against the visibility scatter, and the pure-numpy assembly — -uv conversion, weighted Stokes-I combine, loud dropping of dead rows — into the packaged -`(Nvis, 2)` triplet. - -Good places to checkout next: - -- `scripts/alma/simulator.py` — the same chain fed by CASA's simobserve instead of the - archive: simulate an ALMA observation of a source you control and validate flux - recovery. -- `scripts/alma/start_here.py` — the one-call version of everything above, plus the - **PyAutoLens** loading round-trip. -- `autolens_workspace/scripts/interferometer/` — uv-plane lens modeling on these - products. -""" - -""" -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/check_sizes.sh b/workspace/scripts/check_sizes.sh deleted file mode 100755 index 55a4e29..0000000 --- a/workspace/scripts/check_sizes.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Compare script sizes vs the workspace .script_sizes.json snapshot. -# Warns and exits non-zero when any script has shrunk >50% — a heuristic for -# accidental whole-file truncation by a bulk edit (see CLAUDE.md "Bulk-edit -# safety"). -# -# Usage: -# scripts/check_sizes.sh # check working tree against snapshot -# scripts/check_sizes.sh --update # rewrite snapshot from current scripts/ -# -# Override: -# ALLOW_SHRINK=1 scripts/check_sizes.sh # accept shrinkage, exit 0 - -set -e -WS_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -SNAPSHOT="$WS_ROOT/.script_sizes.json" - -if [ "${1:-}" = "--update" ]; then - cd "$WS_ROOT" - python3 - <<'PY' > "$SNAPSHOT" -import json, os -sizes = {} -for root, _, files in os.walk("scripts"): - for f in sorted(files): - if f.endswith(".py"): - p = os.path.join(root, f) - sizes[p] = os.path.getsize(p) -print(json.dumps(sizes, indent=2, sort_keys=True)) -PY - echo "Updated $SNAPSHOT" - exit 0 -fi - -if [ ! -f "$SNAPSHOT" ]; then - echo "No snapshot at $SNAPSHOT — generate one with: scripts/check_sizes.sh --update" >&2 - exit 1 -fi - -cd "$WS_ROOT" -ALLOW_SHRINK="${ALLOW_SHRINK:-}" python3 - <<'PY' -import json, os, sys -with open(".script_sizes.json") as f: - snapshot = json.load(f) -shrunk = [] -for path, prev in snapshot.items(): - if not os.path.isfile(path): - continue - cur = os.path.getsize(path) - if prev > 200 and cur < prev * 0.5: - pct = 100 * (prev - cur) // prev - shrunk.append((pct, path, prev, cur)) -shrunk.sort(reverse=True) -if shrunk: - print("WARNING: scripts shrunk by >50% since the snapshot:", file=sys.stderr) - for pct, p, prev, cur in shrunk: - print(f" {pct:3d}% {p} {prev}b -> {cur}b", file=sys.stderr) - print("", file=sys.stderr) - print("This may indicate accidental whole-file truncation (see CLAUDE.md", file=sys.stderr) - print("Bulk-edit safety). If the shrinkage is intentional, re-run with", file=sys.stderr) - print("ALLOW_SHRINK=1 and refresh via: scripts/check_sizes.sh --update", file=sys.stderr) - sys.exit(0 if os.environ.get("ALLOW_SHRINK") else 1) -print("OK: all scripts within size tolerance.") -PY diff --git a/workspace/scripts/guides/README.md b/workspace/scripts/guides/README.md deleted file mode 100644 index cd7c391..0000000 --- a/workspace/scripts/guides/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# guides/ - -Cross-instrument guides: the concepts every reduction in this workspace shares, independent of -which telescope the data came from. Read them after `scripts/start_here.py`, in this order: - -- `output_contract.py` — the five products every reduction emits (`data.fits`, `noise_map.fits`, - `psf.fits`, `psf_full.fits`, `reduction.json`) in depth: headers, units, the masked-by-noise - convention, frame products, the ALMA visibility triplet, and loading into **PyAutoLens**. Runs - offline on the output of `scripts/start_here.py` (clean-exits with a message if that hasn't - run yet). -- `noise_maps.py` — the noise story: the per-domain recipes, the Casertano correlated-noise - factor across pixfrac values, blank-sky closure diagnostics, and why chi-squared needs honest - uncorrelated noise. Mostly runs offline; the real-data closure section is guarded. -- `target_spec.py` — every `TargetSpec` dial annotated, spec-YAML round-trips, - `dataclasses.replace` variants, and the validation guard rails. Runs fully offline. diff --git a/workspace/scripts/guides/__init__.py b/workspace/scripts/guides/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/workspace/scripts/guides/noise_maps.py b/workspace/scripts/guides/noise_maps.py deleted file mode 100644 index 1650d0c..0000000 --- a/workspace/scripts/guides/noise_maps.py +++ /dev/null @@ -1,253 +0,0 @@ -""" -Guides: Noise Maps -================== - -The noise map is the least glamorous product of a reduction and the one your science leans on -hardest: every chi-squared and every Bayesian evidence **PyAutoLens** computes trusts it -completely. This guide develops the noise story across the workspace — the per-domain recipes, -the Casertano correlated-noise factor and where it comes from, the closure diagnostics that -validate a finished noise map, and the honest reason **PyAutoReduce** noise maps run ~30% higher -than the legacy SLACS ones. - -Almost everything here runs offline: the Casertano factor and the construction recipe are pure -functions we demonstrate on synthetic arrays. Only the final closure section reads real data (the -output of `scripts/start_here.py`), and it is guarded — the script runs to completion without it. - -__Contents__ - -- **Why Chi-Squared Cares:** Independent Gaussian noise is the likelihood's founding assumption — and resampling breaks it. -- **Imports:** The public noise helpers and the instrument adapters. -- **The Recipes Per Domain:** Constructed (HST/Keck), read (JWST), weighted (ALMA) — one recipe per data domain. -- **The Casertano Factor:** `casertano_r` demonstrated across pixfrac values, including the fine-grid branch. -- **A Synthetic Construction:** `noise_map_from` on a toy mosaic — see the recipe do its work. -- **The Masked-By-Noise Convention:** Bad pixels carry noise 1e8 so the likelihood ignores them. -- **Blank-Sky Closure:** The diagnostic that validates a noise map against the sky it ships with — synthetic first, then the real SLACS reduction (guarded). -- **Thirty Percent Above Legacy, By Design:** Why honest noise maps are bigger than the ones you may be used to. -- **Wrap Up:** Summary and good places to checkout next. - -__Why Chi-Squared Cares__ - -A pixel-level likelihood is built on one assumption: that the noise map gives the RMS of -*independent* Gaussian noise in each pixel. Under that assumption chi-squared is calibrated, -posterior widths mean what they claim, and Bayesian evidence comparisons between lens models are -fair. Pixelized source reconstructions — the workhorse of substructure detection — lean on it -hardest of all. - -Image resampling breaks the assumption. Drizzle (Fruchter & Hook 2002, PASP 114, 144, -https://ui.adsabs.harvard.edu/abs/2002PASP..114..144F) shares each input pixel among several -output pixels, so neighbouring output pixels carry common noise: the map of per-pixel RMS values -is still correct pixel-by-pixel, but it *understates* the uncertainty of any structure larger -than a pixel, and a likelihood that ignores the covariance is over-confident. The canonical -treatment is Casertano et al. (2000, AJ 120, 2747), whose appendix derives a scalar ratio R -between the true noise of large-scale structure and the naive per-pixel RMS as a function of the -drizzle geometry; the DrizzlePac Handbook (https://hst-docs.stsci.edu/drizzpac, section 3.4) -carries the same analysis. Inflating the per-pixel map by R is the standard scalar compromise — -exact for large scales, conservative in between. - -The lensing literature takes this seriously. The SL2S survey drizzled preserving native -orientation and pixel scale *explicitly* to avoid correlated noise (Gavazzi et al. 2012, -https://arxiv.org/abs/1202.3852); Bayer et al. treat drizzle-correlated noise explicitly in their -lens-modeling error budgets (https://arxiv.org/abs/1803.05952, -https://arxiv.org/abs/2302.00480). **PyAutoReduce** applies R to every resampled noise map and -records it in the provenance — and offers two escape hatches for science that wants no -correlation at all: per-exposure frame products (nothing resampled, no R — see the -`individual.py` scripts) and the ALMA visibility branch (fit in the uv-plane, where the noise is -independent by construction). - -__Imports__ - -The noise helpers are public **PyAutoReduce** API: `casertano_r` and `noise_map_from` are the -construction recipe, `empirical_background_rms` the closure diagnostic, `MASKED_NOISE_VALUE` the -masking sentinel. The instrument adapters supply the scale ratios the factor depends on. -""" - -import json -from pathlib import Path - -import numpy as np - -from autoreduce import instruments -from autoreduce.noise.rms import ( - MASKED_NOISE_VALUE, - casertano_r, - empirical_background_rms, - noise_map_from, -) - -""" -__The Recipes Per Domain__ - -One recipe per data domain, chosen by what the upstream pipeline reliably provides: - -- **HST and Keck — construct.** The drizzle/native combine emits a science mosaic (e-/s) and an - inverse-variance weight map, so the pipeline constructs the RMS itself: - - sigma_i = R * sqrt( max(sci_i, 0) / t_exp + 1 / W_i ) - - The first term is the source's Poisson noise (floored at zero so blank sky does not go - imaginary), the second the background variance encoded in the IVM weight — the recipe of Bayer - et al. (https://arxiv.org/abs/1803.05952, section 3.1), scaled by the Casertano R. - -- **JWST — read, don't construct.** The calwebb_image3 resample step already propagates a full - ERR array (Poisson + read noise + flat) through the drizzle, so the pipeline *reads* it and - applies the same R — reconstructing what the observatory pipeline already did well would only - add ways to be wrong. A consistency block (below) guards against trusting a broken ERR. - -- **ALMA — weights.** Each visibility carries a weight nominally equal to 1/sigma^2, so the - per-visibility noise is sigma = 1/sqrt(weight) — and **no** Casertano factor, because nothing - is resampled in the uv-plane. (Whether archival weights are correctly scaled is its own story - — the uv-plane literature routinely recalibrates them against the visibility scatter, e.g. - Hezaveh et al. 2016, https://arxiv.org/abs/1601.01388, section 2 — told in - `scripts/alma/step_by_step.py`.) - -__The Casertano Factor__ - -`casertano_r(pixfrac, scale_ratio)` returns R for a drizzle with drop size `pixfrac` (p) onto an -output grid `scale_ratio` (s) times the native pixel. Two limits anchor the intuition: p = 1 at -s = 1 is plain shift-and-add, the most correlated case (R = 1.5); shrinking the drops toward -interlacing (p -> 0) removes the sharing and R -> 1. - -The table below is the trade every HST reduction navigates (at native output scale, s = 1) — the -same numbers the `scripts/hst_acs/dials.py` trade study explores with real data: -""" -print("Casertano R at s = 1.0 (output grid = native pixels):") -for pixfrac in (1.0, 0.8, 0.6, 0.4): - print(f" pixfrac {pixfrac:.1f} -> R = {casertano_r(pixfrac, 1.0):.3f}") - -""" -Reading it: the SLACS convention (pixfrac 0.8) accepts a 36% noise inflation; pushing to -pixfrac 0.6 buys R = 1.25 but demands a dither pattern rich enough to fill the coverage (the -weight-uniformity diagnostic in every provenance record polices exactly this — RMS/median of the -weight map above ~0.2 means the drops got too small for the dithers). - -There is a second branch worth respecting: when the output grid is *finer* than the drop -(s < p), correlation grows quickly — this is the fine-grid regime WFC3/IR reductions live in, -because the undersampled 0.128" native pixels are typically drizzled to ~0.065". The adapters -carry the native scales, so we can compute the real case: -""" -ir = instruments.get("wfc3_ir") -s_ir = ir.scale_ratio(ir.recommended_final_scale) # s = 0.065 / 0.128 -print( - f"WFC3/IR at its recommended {ir.recommended_final_scale}\"/pix " - f"(native {ir.native_scale}\"/pix, s = {s_ir:.3f}):" -) -for pixfrac in (1.0, 0.8): - print(f" pixfrac {pixfrac:.1f} -> R = {casertano_r(pixfrac, s_ir):.3f}") - -""" -R materially above the s = 1 numbers is the price of recovering resolution from undersampled -pixels — worth paying, but only with the noise map telling the truth about it. (The -`scripts/hst_wfc3_ir/` examples add the companion rule: few-dither IR data at sub-native scales -needs pixfrac 1.0, or the coverage develops zero-weight speckles and packaging refuses to ship.) - -__A Synthetic Construction__ - -The recipe is a pure function, so we can watch it work on a toy mosaic: flat sky with a bright -source, a uniform weight map, 1000 seconds of exposure, drizzled at the SLACS dials. -""" -rng = np.random.default_rng(1) - -shape = (61, 61) -sigma_sky = 0.005 # e-/s background RMS -exptime = 1000.0 # seconds -wht = np.full(shape, 1.0 / sigma_sky**2) # IVM weight: inverse background variance -sci = rng.normal(0.0, sigma_sky, shape) # blank sky ... -sci[28:33, 28:33] += 5.0 # ... plus a bright source (e-/s) - -r_factor = casertano_r(pixfrac=0.8, scale_ratio=1.0) - -noise = noise_map_from(sci, wht, exptime=exptime, correlated_noise_factor=r_factor) - -print(f"Blank-sky corner noise: {noise[:10, :10].mean():.4f} e-/s " - f"(R * sigma_sky = {r_factor * sigma_sky:.4f})") -print(f"Source-peak noise: {noise[30, 30]:.4f} e-/s (Poisson term now dominates)") - -""" -Blank sky lands at R * sigma_sky (a hair above it, because positive sky fluctuations contribute -a whisper of Poisson term — the floor at zero only protects the negative ones); on the source -the Poisson term sqrt(sci/exptime) takes over. Note also what the function refuses to do: zero or negative weights propagate as NaN, and the -packaging stage fails loudly if any land inside the cutout — a hole in the coverage is a problem -to fix, never a pixel to patch silently. - -__The Masked-By-Noise Convention__ - -The one sanctioned exception to that loudness: *isolated* dead or fully-rejected pixels (routine -in deep resampled stacks) are shipped with their noise set to `MASKED_NOISE_VALUE` and the data -zeroed, so any chi-squared ignores them without a separate mask file. The policy is strict — -scattered singletons only, bounded fraction, never near the target — and every masked pixel is -counted in the provenance (`scripts/guides/output_contract.py` inspects the block). -""" -print(f"MASKED_NOISE_VALUE = {MASKED_NOISE_VALUE:.0e} (effectively infinite noise)") - -""" -__Blank-Sky Closure__ - -How do you *validate* a noise map? Close the loop against the data it ships with: the noise -floor the map claims for blank sky must match the sky RMS the mosaic actually shows. The -empirical side is a sigma-clipped RMS of the mosaic (`empirical_background_rms`); the claimed -side is the low percentile of the noise map divided by R. On the synthetic mosaic the closure is -exact by construction: -""" -sky_measured = empirical_background_rms(sci) -floor_claimed = float(np.nanpercentile(noise, 5)) / r_factor - -print(f"Empirical sky RMS: {sky_measured:.4f} e-/s") -print(f"Noise-map floor (pre-R): {floor_claimed:.4f} e-/s") -print(f"Closure ratio (want ~1): {floor_claimed / sky_measured:.3f}") - -""" -On real data the same two numbers are computed for you: every HST/Keck provenance records -`noise.empirical_background_rms`, and the JWST read-don't-construct path records an equivalent -`sky_over_err_floor` consistency ratio (large disagreement there means the upstream ERR model -and the sky disagree — to be investigated, never absorbed). If you have run -`scripts/start_here.py`, we can close the loop on the real SLACS reduction: -""" -WORKSPACE = Path(__file__).resolve().parents[2] -out_dir = WORKSPACE / "output" / "slacs0008-0004" - -if (out_dir / "reduction.json").exists(): - from astropy.io import fits - - record = json.loads((out_dir / "reduction.json").read_text()) - real_noise = fits.getdata(out_dir / "noise_map.fits").astype(float) - physical = real_noise[real_noise < MASKED_NOISE_VALUE] - - r_real = record["noise"]["correlated_noise_factor"] - sky_real = record["noise"]["empirical_background_rms"] - floor_real = float(np.nanpercentile(physical, 5)) / r_real - - print(f"SLACS J0008-0004: R = {r_real:.3f}, empirical sky RMS = {sky_real:.3e} e-/s") - print(f" Noise-map floor / (R * sky RMS) closure: {floor_real / sky_real:.3f} (want ~1)") -else: - print( - "Real-data closure skipped: no output at output/slacs0008-0004 — run " - "scripts/start_here.py (network required) to produce it. Everything above ran offline." - ) - -""" -__Thirty Percent Above Legacy, By Design__ - -If you compare a **PyAutoReduce** HST noise map against a legacy lens-modeling dataset of the -same target, expect the new one to run ~30% higher (the SLACS validation measured a registered -noise ratio of ~1.31). This is not a bug to normalise away: the legacy SLACS noise maps do not -carry the Casertano correction, and at the SLACS drizzle dials R is ~1.36. The historical maps -understate the uncertainty of extended structure; the new ones state it. Posterior widths from -models fitted to the new datasets will be honestly wider — that is the point. - -__Wrap Up__ - -One recipe per domain, one scalar R where resampling correlates the noise, loud failure where -the coverage breaks, a strict masking convention for the survivable cases, and a closure -diagnostic that ties the shipped map back to the shipped sky. When a noise map obeys all of -that, the chi-squared downstream can be trusted — which is the only reason any of this exists. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/dials.py`: the R-vs-pixfrac trade on real data, with the weight-uniformity limit. -- `scripts/hst_acs/individual.py`: frame products — the uncorrelated-noise escape hatch. -- `scripts/alma/step_by_step.py`: visibility weights, and why they get recalibrated before modeling. -- `scripts/guides/output_contract.py`: where the noise map sits in the full product contract. - -This guide needs no network: the demonstrations are synthetic, and the single real-data section -is skipped cleanly when `scripts/start_here.py` has not been run. -""" diff --git a/workspace/scripts/guides/output_contract.py b/workspace/scripts/guides/output_contract.py deleted file mode 100644 index 3c35a5b..0000000 --- a/workspace/scripts/guides/output_contract.py +++ /dev/null @@ -1,304 +0,0 @@ -""" -Guides: The Output Contract -=========================== - -Every **PyAutoReduce** reduction, on every instrument, honours one output contract: four FITS -products plus a provenance record, landing in `output//`. This guide walks that contract -in depth — what each file contains, what its header promises, the conventions (masked-by-noise, -the drizzled-PSF invariant) that make the products modeling-ready with no further preparation, -and how `reduction.json` documents everything that happened. - -This is a runnable script that operates on the output of `scripts/start_here.py` — the SLACS -J0008-0004 HST/ACS reduction. It needs no network itself: if you have not run `start_here.py` -yet, it prints a pointer and exits cleanly, and the prose still reads as documentation. - -__Contents__ - -- **Imports:** Import the standard libraries and the **PyAutoReduce** constants we inspect. -- **Paths:** Anchor to the workspace root and guard on the `start_here.py` output existing. -- **The Five Products:** The contract at a glance. -- **data.fits:** The science cutout — WCS, units (electrons/second vs MJy/sr), header keywords, and the strict-coverage cutout rule. -- **noise_map.fits:** The matching RMS map and the masked-by-noise convention (bad pixels carry noise 1e8, data zeroed). -- **psf.fits vs psf_full.fits:** Compact kernel for convolution vs extended wings, and the drizzled-PSF invariant. -- **reduction.json:** The provenance record, block by block. -- **Frame Products:** The opt-in per-exposure tree — `frames/manifest.json` and per-frame product sets. -- **The Visibility Triplet:** What the contract becomes for ALMA — three `(N_vis, 2)` arrays. -- **Loading With PyAutoLens:** `al.Imaging.from_fits` / `al.Interferometer.from_fits`, guarded. -- **Wrap Up:** Summary and good places to checkout next. - -__Imports__ - -Alongside the standard FITS/plotting libraries we import `MASKED_NOISE_VALUE` from -**PyAutoReduce**'s public noise module — the sentinel the masked-by-noise convention is built on. -""" - -import json -import sys -from pathlib import Path - -import numpy as np -from astropy.io import fits -from astropy.wcs import WCS - -from autoreduce.noise.rms import MASKED_NOISE_VALUE - -""" -__Paths__ - -We anchor to the workspace root and point at the dataset `scripts/start_here.py` produces. The -guard below keeps this guide honest: it never fabricates products, so without the `start_here.py` -output on disk there is nothing to inspect and we exit cleanly. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -OUTPUT_ROOT = WORKSPACE / "output" - -TARGET = "slacs0008-0004" -out_dir = OUTPUT_ROOT / TARGET - -if not (out_dir / "reduction.json").exists(): - print( - f"No reduction found at {out_dir}.\n" - "Run `python scripts/start_here.py` first (network + autoreduce[hst] required) — " - "it produces the SLACS J0008-0004 dataset this guide inspects.\n" - "Exiting cleanly; the prose in this file still reads as documentation." - ) - sys.exit(0) - -record = json.loads((out_dir / "reduction.json").read_text()) - -""" -__The Five Products__ - -The contract, identical for every imaging instrument in the workspace: - -- `data.fits` — float32 science cutout at the modeling pixel scale, WCS intact. -- `noise_map.fits` — the matching per-pixel RMS, correlated-noise corrected, masked pixels at 1e8. -- `psf.fits` — compact (default 21x21) unit-normalised PSF for fit convolution. -- `psf_full.fits` — extended (default 61x61) PSF carrying the wings. -- `reduction.json` — the provenance record (whose content `reduce_target` also returns). - -The `package` block of the provenance lists exactly what shipped — on a Keck tier-A reduction, -for example, you would also see `psf_candidate_.fits` epoch candidates here, and with -`frame_products=True` a `frames/manifest.json` entry. -""" -print(f"Products shipped for {TARGET}: {record['package']['products']}") - -""" -__data.fits__ - -The science cutout. Three promises the header makes, and why each matters for modeling: - -**Units.** `BUNIT` records the pixel units. For HST and Keck reductions this is electrons/second -(the drizzle combines in `cps` units), which is what **PyAutoLens** assumes when computing -physical quantities like magnitudes. JWST is the deliberate exception: NIRCam mosaics stay in -their native surface-brightness units of MJy/sr — flux work then goes through the pixel area, -and the `jwst_nircam` scripts spell out the conversion. Never assume; read `BUNIT` (or -`package.data_units` in the provenance). - -**WCS.** The cutout keeps its world coordinate system, so the products remain on-sky addressable -— you can overlay catalogues, cross-match with survey cutouts, or check the target's position. -This is a deliberate deviation from legacy lens-modeling cutouts, whose stripped headers made -that impossible. - -**Exposure metadata.** `EXPTIME`/`TEXPTIME`, `FILTER`, `INSTRUME` and `TELESCOP` ride along, so a -dataset found on disk two years from now identifies itself. - -One packaging rule worth knowing: the cutout is taken in strict-coverage mode — if the requested -`cutout_shape` extends off the drizzled mosaic, packaging *raises* rather than padding with -garbage. Size the cutout to the coverage, not the other way round. -""" -data_hdu = fits.open(out_dir / "data.fits")[0] -data = data_hdu.data.astype(float) -header = data_hdu.header - -print(f"data.fits shape: {data.shape}, dtype float32 on disk") -for key in ("BUNIT", "EXPTIME", "FILTER", "INSTRUME", "TELESCOP"): - if key in header: - print(f" {key:8s} = {header[key]}") - -wcs = WCS(header) -centre_world = wcs.pixel_to_world(data.shape[1] // 2, data.shape[0] // 2) -print(f" Cutout centre on sky: {centre_world.to_string('decimal')} (deg)") - -""" -__noise_map.fits__ - -The per-pixel RMS matching `data.fits` — the same shape, the same units, and already carrying the -Casertano correlated-noise correction (the full derivation and closure checks live in -`scripts/guides/noise_maps.py`). - -The convention that saves you a mask file downstream is **masked-by-noise**: any pixel the -pipeline rejected (an isolated dead or fully-rejected pixel in the resampled stack) is shipped -with its noise set to `MASKED_NOISE_VALUE` = 1e8 and its data zeroed. A chi-squared then ignores -those pixels automatically — no separate mask FITS, no mask surgery in **PyAutoLens**. - -The policy behind it is deliberately strict, and recorded in the `bad_pixel_policy` block of the -provenance: only *isolated* bad pixels may be masked, never more than 0.5% of the cutout, and -never within 1.5" of the target — a structured defect, a bad fraction, or a bad pixel on the lens -itself fails the reduction loudly instead. Masking is a convenience for scattered singletons, not -a licence to paper over a broken reduction. -""" -noise_map = fits.getdata(out_dir / "noise_map.fits").astype(float) - -masked = noise_map >= MASKED_NOISE_VALUE -print(f"noise_map.fits shape: {noise_map.shape}") -print(f" Masked-by-noise pixels (noise = {MASKED_NOISE_VALUE:.0e}): {int(masked.sum())}") -print(f" Data zeroed at every masked pixel: {bool(np.all(data[masked] == 0.0))}") -print(f" Physical noise range: {noise_map[~masked].min():.3e} - {noise_map[~masked].max():.3e}") -print(f" bad_pixel_policy block: {json.dumps(record['bad_pixel_policy'], indent=2)}") - -""" -__psf.fits vs psf_full.fits__ - -Two PSFs ship with every imaging reduction, and the split is about convolution cost versus wing -science: - -- `psf.fits` (default 21x21) is the compact kernel for fit convolution. Convolution cost scales - with kernel area, and 21x21 captures the great majority of the blurring for these instruments - — the size **PyAutoLens**'s data standards recommend. -- `psf_full.fits` (default 61x61) carries the extended wings, for science where scattered light - at larger radii matters (bright-deflector contamination, photometry checks). Fit with the - compact one; consult the full one. - -Both are odd-shaped (so the PSF has a centre pixel — an even kernel shifts every model image by -half a pixel) and unit-normalised (so convolution conserves flux). - -The invariant behind them is the one to remember: **the delivered PSF is the drizzled PSF**. The -star images it was built from went through the same kernel, pixfrac, scale and rotation as the -science mosaic, so the kernel describes the blurring actually present in `data.fits`. A PSF -processed differently from its data — however carefully made — describes a different image. (The -precedent runs deep: SLACS processed its Tiny Tim PSFs through the identical rectification as its -science frames; see Bolton et al. 2008, https://arxiv.org/abs/0805.1931.) -""" -psf = fits.getdata(out_dir / "psf.fits").astype(float) -psf_full = fits.getdata(out_dir / "psf_full.fits").astype(float) - -for name, kernel in (("psf.fits", psf), ("psf_full.fits", psf_full)): - ny, nx = kernel.shape - peak = np.unravel_index(np.argmax(kernel), kernel.shape) - print( - f"{name}: shape {kernel.shape} (odd: {ny % 2 == 1 and nx % 2 == 1}), " - f"sum = {kernel.sum():.6f}, peak at {peak} (centre: {(ny // 2, nx // 2)})" - ) - -""" -__reduction.json__ - -The provenance record — the reduction's lab notebook, and the same dictionary `reduce_target` -returned. It answers, block by block, every "what exactly produced this dataset?" question: - -- **Envelope** — `written_at` and the software versions (autoreduce and the instrument stack) - that produced the dataset. Reproducibility starts here: the pipeline is a pure function of - spec + archive *given* these versions. -- **`target`** — the full `TargetSpec` as declared, every dial included. The spec is the - reduction; this block is the reduction's definition. -- **`instrument`** — the adapter key (here `acs_wfc`). -- **`acquire`** — which exposures, from where: proposal/exposure identities and `n_exposures`. - An audit of the stack starts by checking nothing unexpected joined it. -- **`align`** — the astrometric story. The HST pipeline trusts the archive's Gaia-tied a-priori - WCS and *records* the cross-correlation evidence for that trust rather than silently - re-registering. -- **`drizzle`** — the combine: the resolved dial set, the weight-uniformity diagnostic (with its - 0.2 rule-of-thumb limit and verdict), the `correlated_noise_factor` R, and the CR method. -- **`noise`** — the recipe string (construction for HST/Keck, propagated-ERR for JWST), R again, - the exposure time, and the empirical blank-sky RMS closure number. -- **`psf`** — method, stars used, and which drizzle pass fed star finding (`star_source_pass`). -- **`bad_pixel_policy`** — how many pixels the masked-by-noise convention touched, and where. -- **`package`** — the shipped products, `cutout_shape`, `pixel_scale` and `data_units`: the - three numbers a modeling script needs, read from provenance rather than remembered. - -(On other paths, extra blocks appear in the same spirit: `calibrate`/`sky` for Keck's ground -stages, `inject` when synthetic-source injection ran, `frames` for frame products.) -""" -print(f"reduction.json blocks: {sorted(record.keys())}") -print(json.dumps(record["drizzle"], indent=2, default=str)[:1200]) - -""" -__Frame Products__ - -Setting `frame_products=True` on a `TargetSpec` (HST, JWST and Keck) adds a second, parallel -output tree: every calibrated exposure chip packaged as its own modeling-ready dataset at -*native* pixel scale, under `output//frames/`: - -- `frames/manifest.json` — the index (schema version 2): the frame cutout shape, native scale, - data units, CR method, DQ semantics, and — per frame — the target's pixel position with the - measured registration residuals and a reliability flag. -- `frames/_chip/` — per frame: `data.fits`, `noise_map.fits`, `dq.fits`, - `cr_mask.fits`, and (with per-frame PSFs) `psf.fits`/`psf_full.fits`. - -Why bother? Nothing in a native frame was resampled — so its noise is *uncorrelated* and needs no -Casertano factor, at the price of modeling several frames jointly instead of one mosaic. The -`individual.py` scripts (`scripts/hst_acs/individual.py`, `scripts/jwst_nircam/individual.py`) -walk the trade in full; the Keck variant differs honestly (outlier masks instead of DQ/CR files, -offset-based rather than WCS registration). - -The default reduction above did not request frames, so we just report their absence. -""" -frames_manifest = out_dir / "frames" / "manifest.json" -if frames_manifest.exists(): - manifest = json.loads(frames_manifest.read_text()) - print(f"Frame products present: manifest version {manifest.get('manifest_version')}") -else: - print("No frame products (frame_products=False for this reduction) — see individual.py scripts.") - -""" -__The Visibility Triplet__ - -For ALMA the contract changes shape, because the modeling-ready dataset is not an image at all: -fitting interferometer data in the uv-plane keeps the noise independent and the likelihood -well-defined, so the products are the calibrated visibilities themselves — - -- `data.fits` — the complex visibilities as an `(N_vis, 2)` array of (real, imaginary). -- `uv_wavelengths.fits` — the `(N_vis, 2)` baseline coordinates (u, v) in wavelengths. -- `noise_map.fits` — the `(N_vis, 2)` per-visibility sigma, derived as 1/sqrt(weight) — and with - **no** Casertano factor, because nothing was resampled. - -Diagnostic sidecars (antennas, scans, times, frequencies per spectral window) ride along, and the -triplet loads with `al.Interferometer.from_fits`. The `scripts/alma/` folder owns this branch. - -__Loading With PyAutoLens__ - -The contract's whole purpose: the products load directly, with the pixel scale read from the -provenance. The import is guarded — **PyAutoReduce** never depends on the modeling stack. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "PyAutoLens is not installed (pip install autolens) — skipping the loading demo. " - "The products above are complete and ready for any machine that has it." - ) - -if al is not None: - dataset = al.Imaging.from_fits( - data_path=out_dir / "data.fits", - noise_map_path=out_dir / "noise_map.fits", - psf_path=out_dir / "psf.fits", - pixel_scales=record["package"]["pixel_scale"], - ) - print( - f"Loaded {TARGET} into PyAutoLens: shape {dataset.data.shape_native} at " - f"{dataset.pixel_scales} arcsec/pix — no further data preparation needed." - ) - -""" -__Wrap Up__ - -The contract in one breath: four FITS files that load straight into **PyAutoLens**, conventions -(masked-by-noise, odd unit-normalised drizzle-consistent PSFs, strict-coverage cutouts) that -remove every downstream preparation step, and a provenance record that makes the whole reduction -auditable. Every instrument folder in this workspace ships this same contract; only the physics -that fills it changes. - -The following locations of the workspace are good places to checkout next: - -- `scripts/guides/noise_maps.py`: how `noise_map.fits` is constructed, and the closure checks that validate it. -- `scripts/guides/target_spec.py`: the declaration that produced everything this guide inspected. -- `scripts/hst_acs/individual.py`: the frame-products tree in practice. -- `scripts/alma/start_here.py`: the visibility triplet in practice. -- `autolens_workspace/scripts/imaging/data_preparation/start_here.py`: the modeling-side statement of these same standards. - -This guide itself needs no network — it reads whatever `scripts/start_here.py` already produced, -and exits cleanly when that output is absent. -""" diff --git a/workspace/scripts/guides/target_spec.py b/workspace/scripts/guides/target_spec.py deleted file mode 100644 index c41c182..0000000 --- a/workspace/scripts/guides/target_spec.py +++ /dev/null @@ -1,271 +0,0 @@ -""" -Guides: The Target Spec -======================= - -A **PyAutoReduce** reduction is declared, not scripted: everything the pipeline will do to your -target is stated up front on one frozen `TargetSpec`, and `reduce_target` is a pure function of -that spec plus the archive. This guide is the reference for the declaration itself — every dial -annotated, the YAML round-trip that makes specs committable, the `dataclasses.replace` idiom for -variants, and the validation guard rails that reject a bad declaration before a single byte is -downloaded. - -Constructing specs touches no network and no heavy dependency, so this entire guide runs offline -in seconds — it is the one workspace script you can always run. - -__Contents__ - -- **The Declaration Philosophy:** Why the spec is the reduction, and what that buys you. -- **Imports:** `TargetSpec`, the instrument registry, and the standard library. -- **The Full Dial Set:** One construction naming every dial, with what each one does. -- **Instrument Adapters:** The registry behind the `instrument` dial — native scales and recommendations. -- **YAML Round Trip:** Write a spec to YAML, load it back, prove they are equal. -- **Variants With replace:** The frozen-dataclass idiom for dial studies. -- **Validation Guard Rails:** The `ValueError`s a bad declaration earns, demonstrated. -- **Pre-Flight Guards:** The cross-dial combinations `reduce_target` rejects before any download. -- **Wrap Up:** Summary and good places to checkout next. - -__The Declaration Philosophy__ - -Reduction scripts rot: a chain of tool calls with tweaked parameters, run once, half-remembered. -A declaration does not. The `TargetSpec` is a frozen dataclass carrying the target's identity, -the instrument, and every literature-contested dial — pixel scale, pixfrac, kernel, cosmic-ray -method, PSF options — so that the spec *is* the reduction: same spec plus same archive equals -same dataset, modulo upstream reference-file updates, which the provenance records. - -This buys three things. Reproducibility: commit one small YAML per target (see -`dataset/README.md`) and anyone regenerates your sample. Auditability: the full spec is embedded -in every `reduction.json`, so a dataset always carries its own definition. And honesty: dials the -literature disagrees on (published HST lens reductions span no-drizzle to pixfrac 0.6 to 1.0) -are surfaced as user-facing configuration, never buried defaults. - -__Imports__ - -Everything here is public API and light: the spec class, the instrument registry, and the -standard library for the YAML round-trip and variants. -""" - -import dataclasses -import tempfile -from pathlib import Path - -from autoreduce import TargetSpec, instruments -from autoreduce.instruments import nircam_adapter_for_filter - -""" -__The Full Dial Set__ - -One construction, every dial, each with its job. The values below are the defaults (so this spec -is equivalent to `TargetSpec(name=..., ra=..., dec=...)`) except where a trailing comment says -otherwise — the point is the annotated tour, not the target. -""" -spec = TargetSpec( - # Identity: - name="slacs0008-0004", # Names the output folder: output//. - ra=2.012333, # Target right ascension, degrees (J2000). - dec=-0.068944, # Target declination, degrees (J2000). - instrument="acs_wfc", # Adapter key — see the registry section below for all 11. - filter_name="F814W", # The band to reduce. - proposal_ids=("10886",), # Pin acquisition to these programmes; None = everything at the coords. - # Acquisition: - sync_references=True, # Re-sync CRDS reference files each HST run; False = offline opt-out - # for a previously-warmed cache (raises without one). - # Packaging: - cutout_shape=(281, 281), # Science/noise cutout, output pixels (~14" at 0.05"/pix). Strict - # coverage: a cutout off the mosaic raises — size to the data. - # Drizzle (imaging combine): - final_scale=0.05, # Output pixel scale, arcsec/pix (the SLACS convention for ACS). - final_pixfrac=0.8, # Drizzle drop size, (0, 1]; smaller = less correlated noise but - # needs richer dithers (see guides/noise_maps.py). - final_kernel="square", # Drizzle kernel; published practice varies, so it is a dial. - cr_method="driz_cr", # Cosmic rays: "driz_cr" (STScI stack rejection, default) or - # "deepcr" (per-frame CNN masks; HST ACS/UVIS only, opt-in). - # PSF: - psf_shape=(21, 21), # Compact convolution kernel; must be odd. - psf_full_shape=(61, 61), # Extended wings product; must be odd. - psf_backend="epsf", # Mosaic-path back-end: "epsf" (photutils) or "starred" - # (super-sampled ePSF; the [starred] extra). - psf_star_pass="auto", # Which drizzle pass feeds star finding: "auto" (no extra cost), - # "science" (pin to shipped mosaic), "no_cr" (dedicated - # CR-flag-ignoring second drizzle; HST opt-in). - psf_from_frames=False, # Build the mosaic PSF by combining per-frame ePSFs through the - # drizzle geometry instead of from mosaic stars (HST/JWST). - # Frame products: - frame_products=False, # Also package every calibrated exposure as a native-scale - # per-frame dataset under output//frames/ (HST/JWST/Keck). - # Alignment: - alignment_tolerance_pix=0.1, # Recorded intent for TweakReg refinement; currently unread — - # the pipeline trusts the archive's Gaia-tied WCS and records - # the cross-correlation evidence instead (honest caveat). - # Synthetic-source injection (see the simulator.py scripts): - inject_image=None, # Path to a plain FITS image to inject into the real frames - # (e-/s per pixel for HST/Keck, Jy per pixel for JWST). - inject_pixel_scale=None, # arcsec/pix of that image; REQUIRED with inject_image. - inject_position=None, # (ra, dec) degrees to centre it on; None = the target. - inject_psf=None, # PSF FITS to convolve with; None = each frame's own ePSF. - inject_seed=0, # Seed for the injected source's Poisson realisations. - # ALMA / visibility branch (ignored by imaging instruments): - alma_uids=None, # Execution-block uids pinning the measurement sets. - alma_field=None, # Science field name inside the MS, e.g. "G09v1.40". - alma_spws=None, # Spectral windows to extract; leave line-bearing spws out. - alma_width=0, # Channel-averaging width; 0 = collapse each spw (continuum). - alma_ms_dir=None, # Local calibrated-MS directory; None = archive acquisition. - alma_project_code=None, # ALMA project code for archive acquisition. - alma_sim_antennalist="alma.cycle8.3.cfg", # simobserve dials (active only when injecting - alma_sim_totaltime_s=1800.0, # on a visibility-domain instrument): array config, - alma_sim_integration_s=10.0, # total on-source time, integration time, - alma_sim_freq_ghz=230.0, # observing frequency, - alma_sim_pwv_mm=0.5, # and precipitable water vapour (0 = noiseless). - # Keck / ground-based branch (ignored by space-based instruments): - koa_science_ids=None, # Explicit KOA frame ids pinning the science set (the raw - # archive has no association tables). - koa_psf_star_ids=None, # PSF-star frames reduced pipeline-identically (tier A). - sky_window=9, # Running-sky window: temporally adjacent frames medianed - # per frame (K' sky varies on minutes timescales). -) - -print(f"Declared: {spec.name} ({spec.instrument}/{spec.filter_name}), " - f"{spec.final_scale}\"/pix, pixfrac {spec.final_pixfrac}") - -""" -__Instrument Adapters__ - -The `instrument` dial selects an adapter from a registry — everything instrument-specific -(native pixel scale, calibrated product type, combine backend, archive, recommended output -scale) lives behind it, which is why the rest of the spec is instrument-agnostic. -""" -print(f"Registered instruments: {sorted(instruments.registered_keys())}") - -for key in ("acs_wfc", "wfc3_uvis", "wfc3_ir", "nircam_sw", "nircam_lw", "nirc2_narrow"): - adapter = instruments.get(key) - print( - f" {key:13s} native {adapter.native_scale:.4f}\"/pix, " - f"recommended final_scale {adapter.recommended_final_scale}\"/pix" - ) - -""" -The adapter's `recommended_final_scale` documents sensible sampling for the detector — -`final_scale` remains your dial, but deviating from the recommendation is a choice the -instrument scripts discuss (WFC3/IR's 0.065" recommendation against its 0.128" native pixels is -the sharpest example). - -NIRCam has a routing helper you should always use rather than picking `nircam_sw`/`nircam_lw` by -hand — the filter name determines the channel: -""" -for filter_name in ("F115W", "F277W"): - adapter = nircam_adapter_for_filter(filter_name) - print(f" {filter_name} -> {adapter.key} ({adapter.native_scale:.3f}\"/pix native)") - -""" -__YAML Round Trip__ - -Specs are committable: `TargetSpec.from_yaml` loads a per-target YAML file (coercing lists to -the tuples the dataclass wants), which is how a reduced sample stays reproducible — one YAML per -target in `dataset/`, never the FITS. We write one to a temporary path and prove the round trip -is exact. -""" -yaml_text = """\ -name: slacs0008-0004 -ra: 2.012333 -dec: -0.068944 -proposal_ids: [10886] -cutout_shape: [281, 281] -final_pixfrac: 0.8 -""" - -with tempfile.TemporaryDirectory() as tmp: - yaml_path = Path(tmp) / "slacs0008-0004.yaml" - yaml_path.write_text(yaml_text) - spec_from_yaml = TargetSpec.from_yaml(yaml_path) - -print(f"Round trip exact: {spec_from_yaml == spec}") -print(f"as_dict() keys: {len(spec.as_dict())} dials serialised into every reduction.json") - -""" -__Variants With replace__ - -The spec is a frozen dataclass — you never mutate one, you derive variants with -`dataclasses.replace`. This is the idiom for dial studies (compare pixfracs, try the deepCR CR -route, request frame products) while everything unstated stays identical, and it is exactly how -`scripts/hst_acs/dials.py` builds its trade study. -""" -spec_pixfrac06 = dataclasses.replace(spec, name="slacs0008-0004_p06", final_pixfrac=0.6) -spec_deepcr = dataclasses.replace(spec, name="slacs0008-0004_deepcr", cr_method="deepcr") -spec_frames = dataclasses.replace(spec, name="slacs0008-0004_frames", frame_products=True) - -for variant in (spec_pixfrac06, spec_deepcr, spec_frames): - print(f" {variant.name}: pixfrac {variant.final_pixfrac}, cr {variant.cr_method}, " - f"frames {variant.frame_products}") - -""" -Give each variant its own `name` — the name is the output folder, and two specs sharing one -would overwrite each other's products. - -__Validation Guard Rails__ - -A declaration is only trustworthy if a bad one cannot exist, so `TargetSpec` validates at -construction and raises `ValueError` immediately — long before `reduce_target`, the network, or -an hour of drizzling could discover the problem for you. The guard rails, demonstrated: -""" -attempts = [ - ("RA outside +/-360 degrees", - dict(name="bad", ra=400.0, dec=0.0)), - ("pixfrac outside (0, 1]", - dict(name="bad", ra=0.0, dec=0.0, final_pixfrac=0.0)), - ("even PSF shape (no centre pixel)", - dict(name="bad", ra=0.0, dec=0.0, psf_shape=(20, 20))), - ("unknown cr_method", - dict(name="bad", ra=0.0, dec=0.0, cr_method="lacosmic")), - ("injection dials without inject_image", - dict(name="bad", ra=0.0, dec=0.0, inject_pixel_scale=0.05)), - ("inject_image without its pixel scale", - dict(name="bad", ra=0.0, dec=0.0, inject_image="arc.fits")), -] - -for label, kwargs in attempts: - try: - TargetSpec(**kwargs) - except ValueError as e: - print(f" [rejected] {label}: {e}") - else: - print(f" [UNEXPECTED] {label} was accepted — this should not happen") - -""" -The full rule set: RA within +/-360 and Dec within +/-90 degrees; `final_pixfrac` in (0, 1]; -`cr_method` one of "driz_cr"/"deepcr" and `psf_star_pass` one of "auto"/"science"/"no_cr"; the -cutout and both PSF shapes two positive integers with the PSF shapes odd; the `alma_sim_*` times -and frequency positive; and injection all-or-nothing — `inject_image` requires -`inject_pixel_scale`, and no injection dial may be set without `inject_image`. - -__Pre-Flight Guards__ - -A second layer of guards lives at the top of `reduce_target` and checks *combinations* of dials -against the chosen instrument — still before any download, so an unsupported combination costs -you an exception in milliseconds, not a broken dataset in an hour: - -| Declaration | Supported on | Why it is bounded | -|---|---|---| -| `frame_products=True` | HST, JWST, Keck | Per-frame packaging exists for these paths only. | -| `psf_from_frames=True` | HST, JWST | The Keck AO mosaic PSF is the tier-A epoch design instead. | -| `cr_method="deepcr"` | HST AstroDrizzle path, ACS/WFC + WFC3/UVIS | Needs a registered deepCR model; WFC3/IR cosmic rays are already ramp-flagged. | -| `psf_star_pass="no_cr"` | HST AstroDrizzle path, without `psf_from_frames` | It is a second full AstroDrizzle pass for the stars. | -| `inject_image=...` | HST, JWST and Keck imaging paths, or visibility simulation | Injection is built per combine backend (simobserve on the ALMA branch). | - -These raise as loud design boundaries rather than silently skipped options — the instrument -scripts point out which apply as they use each feature. - -__Wrap Up__ - -The `TargetSpec` is the whole interface: identity, instrument, and every contested dial in one -frozen, validated, committable declaration. Everything downstream — the provenance, the -reproducibility story, the dial studies — follows from that one design decision. - -The following locations of the workspace are good places to checkout next: - -- `dataset/README.md`: the one-YAML-per-target convention for committing your sample's specs. -- `scripts/start_here.py`: the spec above driving a real end-to-end reduction. -- `scripts/hst_acs/dials.py`: `dataclasses.replace` variants powering a real drizzle trade study. -- `scripts/guides/output_contract.py`: where the spec reappears verbatim inside `reduction.json`. - -This guide runs fully offline — spec construction touches neither the network nor the heavy -instrument stacks. -""" diff --git a/workspace/scripts/hst_acs/README.md b/workspace/scripts/hst_acs/README.md deleted file mode 100644 index 61f43a1..0000000 --- a/workspace/scripts/hst_acs/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# HST ACS/WFC - -The flagship folder of the workspace: ACS/WFC is **PyAutoReduce**'s reference instrument, and -its SLACS parity reductions set the quality bar every other pipeline is described against. The -other instrument folders teach their *deltas* relative to what is established here. - -Recommended reading order: - -- `start_here.py` — the default pipeline end to end on the SLACS lens SDSS J0008-0004: one - frozen `TargetSpec`, one `reduce_target` call, a modeling-ready dataset. -- `step_by_step.py` — the same reduction dissected stage by stage: MAST/CRDS acquisition, WCS, - drizzle, noise construction, PSF and packaging, each grounded in the STScI handbooks. -- `dials.py` — the trade study for the four user-facing dials (`final_scale`, `final_pixfrac`, - `final_kernel`, `cr_method`), with the Casertano R factor computed live. -- `psf.py` — the PSF story: star selection, the ePSF tiers, the star-pass dial, STARRED, and - the diagnostics shipped in `reduction.json`. -- `individual.py` — per-exposure frame products (`frame_products=True`): native-frame cutouts - with uncorrelated noise, per-frame deepCR masks and per-frame PSFs. -- `simulator.py` — injection testing: place a synthetic lensed arc into the real exposures, run - the identical pipeline, and verify the flux comes back out. diff --git a/workspace/scripts/hst_acs/__init__.py b/workspace/scripts/hst_acs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/workspace/scripts/hst_acs/dials.py b/workspace/scripts/hst_acs/dials.py deleted file mode 100644 index 7466307..0000000 --- a/workspace/scripts/hst_acs/dials.py +++ /dev/null @@ -1,331 +0,0 @@ -""" -HST ACS: Dials -============== - -Most of a **PyAutoReduce** reduction is deliberately not up for debate: the calibrated `_flc` -exposures come straight from MAST, the noise recipe is fixed, the PSF is drizzled exactly like the -science mosaic. But four dials on the `TargetSpec` are exposed precisely because the strong-lensing -literature does *not* agree on them: the output pixel scale (`final_scale`), the drizzle drop size -(`final_pixfrac`), the drizzle kernel (`final_kernel`) and the cosmic-ray strategy (`cr_method`). - -This script is the trade study for those dials. It explains what each one does physically, what it -costs in correlated noise (with the Casertano factor computed live), what the SLACS, SL2S and -Bayer et al. reductions actually chose, and how to decide for your own target. Everything down to -the final section runs offline in seconds — the only network-touching part is an optional -re-reduction at a second pixfrac, which is switched off by default. - -If you have not yet run `start_here.py` in this folder, read it first: this script assumes you -know the shape of the default SLACS reduction it dissects. - -__Contents__ - -- **Imports:** Import **PyAutoReduce** and the supporting libraries. -- **The Four Dials:** Where they live on the `TargetSpec` and why only these four are dials. -- **Final Scale:** The output pixel size — resolution versus per-pixel depth and noise correlation. -- **Final Pixfrac:** The drizzle drop size — the PSF/noise-correlation trade at the heart of drizzling. -- **The Casertano Factor, Live:** Compute R across pixfrac and scale with `casertano_r`. -- **Weight Uniformity:** The diagnostic that tells you when pixfrac is too small for your dithers. -- **Final Kernel:** The drop shape — square default, and what the alternatives buy you. -- **What The Literature Chose:** SLACS V, SLACS IX, SL2S and Bayer et al. disagree — a field guide. -- **Cosmic-Ray Strategy:** `driz_cr` median flagging, its failure mode on lens cores, and the deepCR opt-in. -- **Decision Guidance:** A short flowchart for choosing dials on a new target. -- **A Second Reduction (Optional):** Re-reduce SLACS J0008-0004 at a different pixfrac and compare. -- **Wrap Up:** Where to go next. - -__The Four Dials__ - -Every dial that the literature is agreed on is hard-wired: the mosaic is north-up -(`final_rot=0`), inverse-variance weighted (`final_wht_type='IVM'`), in electrons per second -(`final_units='cps'`), sky-matched (`skymethod='globalmin+match'`). Those defaults are documented -deviations-or-adoptions of the AstroDrizzle defaults (DrizzlePac Handbook, -https://hst-docs.stsci.edu/drizzpac), justified once in the **PyAutoReduce** design docs and not -re-litigated per target. - -The four that remain are exposed on the `TargetSpec` because published lensing reductions genuinely -differ on them, so an informed user must be able to differ too: - -- `final_scale` (default 0.05"/pixel — the SLACS convention, equal to the ACS/WFC native scale). -- `final_pixfrac` (default 0.8). -- `final_kernel` (default "square"). -- `cr_method` (default "driz_cr"). - -Every choice is recorded in `reduction.json`, so two datasets reduced with different dials are -always distinguishable after the fact. -""" - -""" -__Imports__ - -The dial mathematics below needs only the `casertano_r` helper and numpy — no network, no -drizzlepac. The optional re-reduction at the end imports `reduce_target` like every other script. -""" - -from pathlib import Path -import os - -import numpy as np - -from autoreduce import TargetSpec -from autoreduce.noise.rms import casertano_r -from autoreduce.drizzle.diagnostics import WEIGHT_UNIFORMITY_LIMIT - -""" -__Final Scale__ - -`final_scale` sets the output mosaic's pixel size in arcseconds. For ACS/WFC the native detector -scale is 0.05"/pixel, and the default keeps it: SLACS drizzled to 0.05"/pixel, every legacy SLACS -modeling dataset is on that grid, and matching it means masks, light-profile sizes and pixel-scale -conventions carry over unchanged into **PyAutoLens**. - -Drizzling to a *finer* grid (0.03"/pixel is common in time-delay-lens work) partially recovers -resolution from well-dithered data, because the drizzle algorithm (Fruchter & Hook 2002, -PASP 114, 144, https://ui.adsabs.harvard.edu/abs/2002PASP..114..144F) interlaces the dithered -samples onto the sub-pixel grid. The costs are real, though: - -- Each output pixel receives fewer electrons, so the per-pixel signal-to-noise drops. -- Fine grids need *more, well-placed* dithers to fill every output pixel — with too few, the - weight map develops holes and the reduction fails its own uniformity check (see below). -- Noise correlation between neighbouring pixels grows (next section) unless pixfrac shrinks - with the scale — and shrinking pixfrac needs yet more dithers. - -For galaxy-scale lens modeling on typical 2-8 exposure ACS visits, the native 0.05"/pixel default -is the robust choice; reach for finer scales only with rich dither sets and a science case that -needs them. -""" - -""" -__Final Pixfrac__ - -`final_pixfrac` is the fraction by which each input pixel is shrunk before being "dripped" onto -the output grid — the central free parameter of the drizzle algorithm. At `pixfrac=1.0` drizzle -behaves like shift-and-add: every input pixel overlaps several output pixels, photometry is -maximally stable, but neighbouring output pixels share input electrons and their noise is -correlated. As pixfrac shrinks toward 0, drizzle approaches pure interlacing: sharper effective -PSF, less noise correlation — but each drop lands on fewer output pixels, so sparse or poorly -placed dithers leave under-covered pixels with wildly varying weights. - -There is no free lunch here, only a trade, which is exactly why **PyAutoReduce** exposes the dial -instead of hiding it. The default of 0.8 follows the SLACS IX-era convention for dithered ACS -data: most of the noise-decorrelation benefit of shrinking the drop, while staying robust on the -4-8 exposure visits typical of lens programs. -""" - -""" -__The Casertano Factor, Live__ - -Whatever pixfrac you choose, the drizzled mosaic's pixel-to-pixel RMS *understates* the noise a -lens-model chi-squared actually experiences, because drizzling shares each input electron between -neighbouring output pixels. The standard correction is the analytic factor R of Casertano et al. -2000 (AJ 120, 2747, appendix) — a function of only pixfrac p and the scale ratio s -(output scale / native scale). **PyAutoReduce** multiplies every mosaic noise-map by R and records -it in `reduction.json` as `drizzle.correlated_noise_factor`. - -`casertano_r` is a public helper, so we can map the whole trade space in a few lines. The s = 1 -column is the ACS default grid; the s = 0.6 column shows how much harsher the correction gets on -a 0.03"/pixel fine grid. -""" - -print("Casertano correlated-noise factor R:\n") -col_native = 'R (s=1.0, 0.05")' -col_fine = 'R (s=0.6, 0.03")' -print(f"{'pixfrac':>8} | {col_native:>17} | {col_fine:>17}") -print("-" * 50) -for pixfrac in [1.0, 0.9, 0.8, 0.7, 0.6, 0.5]: - r_native = casertano_r(pixfrac, 1.0) # output at the ACS native 0.05"/pixel - r_fine = casertano_r(pixfrac, 0.6) # output at 0.03"/pixel, s = 0.03/0.05 - print(f"{pixfrac:>8.1f} | {r_native:>17.3f} | {r_fine:>17.3f}") - -""" -The native-scale column reproduces the numbers quoted throughout this workspace: R = 1.500 at -pixfrac 1.0, 1.364 at the 0.8 default, 1.250 at 0.6. Two readings of this table matter: - -- Shrinking pixfrac from 1.0 to 0.6 buys you ~17% less noise inflation at native scale — real, - but not transformative. The fine-grid column is where R gets punishing, which is another reason - the default stays at the native scale. -- R is a *scalar approximation* to a correlation that is really a neighbourhood covariance. It - makes the noise-map faithful for a chi-squared that treats pixels as independent — the honest - correction, not a perfect one. `guides/noise_maps.py` develops this point in full. -""" - -""" -__Weight Uniformity__ - -The guard-rail on shrinking pixfrac is coverage: if the shrunken drops no longer tile the output -grid evenly, the drizzle weight map becomes ragged and per-pixel depths vary wildly. The -DrizzlePac Handbook's rule of thumb is that the RMS of the weight map should stay below ~20-30% -of its median; **PyAutoReduce** computes exactly this statistic on every run, stores it at -`reduction.json["drizzle"]["weight_uniformity"]`, and flags it against a limit of -""" - -print(f"\nWEIGHT_UNIFORMITY_LIMIT = {WEIGHT_UNIFORMITY_LIMIT} (RMS/median of the drizzle weight map)") - -""" -The default SLACS J0008-0004 reduction (7 exposures, pixfrac 0.8) measures ~0.066 — comfortably -uniform. If you shrink pixfrac or refine the scale and this diagnostic climbs toward the limit, -the dials have outrun your dither set: back off pixfrac toward 1.0, coarsen the scale, or accept -the reduction refusing to ship (the packaging stage crashes loudly on non-finite noise pixels -rather than silently interpolating holes — see `hst_wfc3_ir/start_here.py` for a real example of -that guard firing). -""" - -""" -__Final Kernel__ - -`final_kernel` sets the shape of the drop each input pixel is drizzled through. The default -"square" (a shrunken image of the pixel itself) is the DrizzlePac default and the workhorse of -published reductions. The alternatives ("gaussian", "point", "turbo", "lanczos3") trade edge -sharpness against noise properties and speed; SLACS IX used a Gaussian kernel with MultiDrizzle, -which is part of why the legacy SLACS mosaics cannot be reproduced bit-for-bit today. - -Unless you are chasing parity with a specific historical reduction, keep "square": it is the -best-characterised choice, and the Casertano R treatment above is derived for it. -""" - -""" -__What The Literature Chose__ - -The reason these dials exist: three exemplary lensing reductions of HST imaging made three -different sets of choices, each defensible. - -- **SLACS V (Bolton et al. 2008, https://arxiv.org/abs/0805.1931)** — for single-exposure - snapshot F814W imaging, did *not* drizzle at all ("not well suited to single-exposure Snapshot - data"): frames were rectified by bilinear interpolation (ACSPROC), cosmic rays masked with - L.A.Cosmic (van Dokkum 2001, PASP 113, 1420), and the TinyTim model PSF rectified through the - identical resampling — the precedent for **PyAutoReduce**'s drizzled-PSF invariant. -- **SLACS IX (Auger et al. 2009)** — drizzled the dithered subset to 0.05"/pixel with a Gaussian - kernel and an unstated pixfrac. The convention this workspace's defaults descend from, and a - cautionary tale about recording your dials (which `reduction.json` now does). -- **Bayer et al. (arXiv:1803.05952)** — the WFC3/UVIS F390W reduction of SDSS J0252+0039 used - the native scale with pixfrac 1.0, and handled noise correlation not with a scalar R but with - blank-sky *realizations* drizzled through the same footprint. Maximum robustness, at the price - of the strongest per-pair correlation (R = 1.5) — coherent because their analysis measured the - noise power spectrum directly. -- **SL2S (Gavazzi et al. 2012, arXiv:1202.3852)** — drizzled WFPC2 data preserving the native - CCD orientation and pixel scale *explicitly to avoid producing correlated noise*, and - CR-cleaned single ACS/WFC3 exposures with L.A.Cosmic. The cleanest published statement that - uncorrelated noise-maps are a lens-modeling requirement, not a nicety. - -None of these is wrong. The defaults in this workspace (0.05", pixfrac 0.8, square, driz_cr) are -the choice **PyAutoReduce** validated against SLACS parity; the dials let you reproduce any of -the others. -""" - -""" -__Cosmic-Ray Strategy__ - -`cr_method` selects how cosmic rays are removed at the combine stage, and it earns its place as a -dial because the default has a measured failure mode on exactly the object you care about. - -**"driz_cr" (default).** AstroDrizzle's standard approach: drizzle each exposure separately, -build a median image, blot it back to each frame's geometry, and flag pixels that deviate -sharply from the blotted median (DrizzlePac Handbook). It needs >= 2-3 overlapping exposures and -it works well on flat sky — but on the steep brightness gradient of a lens-galaxy core, -sub-pixel dither offsets make the blotted median a poor predictor of the pixel it is compared -against. On the SLACS acceptance targets this mis-flagging was measured to cost roughly a third -of the deflector's central flux before the comparison thresholds were tuned — the kind of bias -that propagates straight into a lens-light model. - -**"deepcr" (opt-in).** Per-frame cosmic-ray masks from the deepCR convolutional network -(Zhang & Bloom 2020, ApJ 889, 24, https://ui.adsabs.harvard.edu/abs/2020ApJ...889...24Z), -trained largely on ACS F814W data, written into each frame's DQ array; the combine then becomes a -plain weighted-mean drizzle with the median/blot/driz_cr machinery switched off. Because each -frame is cleaned independently, the lens core's gradient never enters a frame-to-frame -comparison. Requires the `[frames]` extra (deepCR pulls torch) and is available for `acs_wfc` and -`wfc3_uvis` (WFC3/IR needs no CR model — its up-the-ramp readout rejects cosmic rays at -calibration; see `hst_wfc3_ir/start_here.py`). - -One implementation detail is worth knowing because it is invisible when it works: AstroDrizzle's -`resetbits` default *clears* DQ bit 4096 — the very bit the per-frame masks are written into — so -the deepCR path must run with `resetbits=0`. **PyAutoReduce** pins this internally (and tests -it); if you ever hand-roll a drizzle over deepCR-flagged frames, carry the same setting or your -masks will be silently ignored. - -The library default remains "driz_cr" with tuned thresholds: flipping the default to deepCR is -deliberately gated on a human-reviewed SLACS validation, not on this workspace. Use -`cr_method="deepcr"` today when your visit has few exposures, sub-pixel dithers, or you see the -tell-tale flux depression at the lens centre in the difference between a driz_cr and a no-CR -star-pass mosaic (`psf.py` shows how to make one). -""" - -""" -__Decision Guidance__ - -A short field guide, dial by dial, for a new galaxy-scale lens target on ACS: - -- **final_scale** — keep 0.05" (native). Go finer only with >= 8 well-placed dithers *and* a - science case (e.g. point-image astrometry) that needs it; then expect to lower pixfrac and - re-check weight uniformity. -- **final_pixfrac** — keep 0.8. Raise to 1.0 if the weight-uniformity diagnostic climbs (few - dithers, IR-style speckle holes); lower toward 0.6 only with rich dither sets when you want the - last ~8% off the R factor. -- **final_kernel** — keep "square" unless chasing parity with a Gaussian-kernel legacy reduction. -- **cr_method** — keep "driz_cr" for well-dithered visits; switch to "deepcr" for few-exposure - visits or when the lens core shows CR-rejection bites. - -And in every case: read `reduction.json` afterwards. The dials you chose, the R they implied and -the uniformity they achieved are all recorded — a reduction whose diagnostics you have not read -is not finished. -""" - -""" -__A Second Reduction (Optional)__ - -Everything above ran offline. To *feel* the pixfrac trade on real data, re-reduce the -`start_here.py` target at pixfrac 1.0 and compare the two `reduction.json` records (and, if you -wish, the two noise maps — their ratio should track R(1.0)/R(0.8) = 1.100 in the blank-sky -limit). - -This re-runs the full pipeline (the exposure cache makes it a re-drizzle, not a re-download), so -it is gated behind an environment variable rather than running on import: - - AUTOREDUCE_DIALS_RERUN=1 python scripts/hst_acs/dials.py -""" - -if os.environ.get("AUTOREDUCE_DIALS_RERUN") == "1": - from autoreduce import reduce_target - - WORKSPACE = Path(__file__).resolve().parents[2] - CACHE_ROOT = WORKSPACE / "cache" # shared with start_here.py — the exposures are already here - OUTPUT_ROOT = WORKSPACE / "output" - - spec = TargetSpec( - name="slacs0008-0004_pixfrac10", # a distinct name so the default products are untouched - ra=2.012333, # SDSS J0008-0004 (SLACS) - dec=-0.068944, - proposal_ids=("10886",), # the SLACS ACS program - final_pixfrac=1.0, # the dial under study; all other dials stay at their defaults - ) - - print("\nRe-reducing SLACS J0008-0004 at pixfrac 1.0 (re-uses the cached exposures)...") - record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - - print("\npixfrac 1.0 diagnostics:") - print(f" correlated_noise_factor : {record['drizzle']['correlated_noise_factor']:.3f}") - print(f" weight_uniformity : {record['drizzle']['weight_uniformity']}") - print("\nCompare against output/slacs0008-0004/reduction.json from start_here.py.") -else: - print( - "\nOptional re-reduction skipped (set AUTOREDUCE_DIALS_RERUN=1 to re-reduce the" - " start_here.py target at pixfrac 1.0 and compare diagnostics)." - ) - -""" -__Wrap Up__ - -You have seen the four dials **PyAutoReduce** exposes on an ACS reduction, the correlated-noise -mathematics that couples the scale and pixfrac choices, the uniformity diagnostic that polices -them, and the published reductions whose disagreements motivated making them dials at all. - -Good places to checkout next: - -- `hst_acs/psf.py` — the star-pass dial (`psf_star_pass`) interacts with `cr_method`: CR flags - eat star cores, and the no-CR star pass exists precisely to win them back. -- `hst_acs/individual.py` — per-exposure frame products sidestep the drizzle dials entirely - (nothing is resampled, so R = 1 by construction). -- `guides/noise_maps.py` — the full noise story these dials feed into. -- `hst_wfc3_ir/start_here.py` — the pixfrac rule colliding with real few-dither IR data. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_acs/individual.py b/workspace/scripts/hst_acs/individual.py deleted file mode 100644 index c3d0297..0000000 --- a/workspace/scripts/hst_acs/individual.py +++ /dev/null @@ -1,403 +0,0 @@ -""" -HST ACS: Individual Frame Products -================================== - -Every script so far has produced one drizzled mosaic per target. Drizzling is a resampling, and -resampling has a price a lens modeler pays forever after: **correlated noise**. Each input pixel -is shared among neighbouring output pixels (Fruchter & Hook 2002, PASP 114, 144), so adjacent -mosaic pixels do not carry independent noise — while the chi^2 of every pixel-based lens -likelihood assumes exactly that independence. **PyAutoReduce** compensates with the scalar -Casertano correction R, but a scalar is an approximation to a full covariance, and precision -applications (pixelized source reconstructions, Bayesian evidence comparisons, substructure -searches) feel the difference. - -There is a clean way out: don't resample. Model the individual calibrated exposures -*simultaneously*, each on its own native pixel grid, with its own PSF and its own genuinely -uncorrelated noise-map. The SL2S survey took a version of this stance years ago — reducing -WFPC2 data while explicitly preserving native pixels to avoid correlated noise (Gavazzi et al. -2012, https://arxiv.org/abs/1202.3852) — and **PyAutoReduce**'s per-exposure frame products are -that idea as a first-class packaging mode: set `frame_products=True` and every calibrated `_flc` -chip that covers the target ships as its own modeling-ready `al.Imaging` dataset, alongside (not -instead of) the mosaic. - -This script runs that mode on the SLACS anchor, then walks everything it produces: the `frames/` -tree, the manifest, the per-frame noise (ERR-based, no R — the whole point), the deepCR cosmic -ray masks, the per-frame PSFs and their drop-convolution combination, and the registration -residuals that tell you how well the frames agree about where the sky is. - -**Dependency note:** per-frame cosmic-ray masking uses deepCR, an optional extra — install with -`pip install "autoreduce[frames]"` before running. The run itself reuses the exposure cache from -`start_here.py`. - -__Contents__ - -- **Imports:** Import **PyAutoReduce** and the other libraries we need. -- **Paths:** Anchor the cache and output locations to the workspace root. -- **Target Spec:** The same anchor with `frame_products=True` and `psf_from_frames=True`. -- **Run:** The reduction — mosaic products plus the frames/ tree. -- **The Frames Tree:** What lands on disk, per exposure and chip. -- **Manifest Walk:** The frames/manifest.json schema, entry by entry. -- **Per-Frame Noise — No R:** ERR-based noise-maps with genuinely uncorrelated pixels. -- **Sky Pedestal:** Why MDRIZSKY must be subtracted from frames but not the mosaic. -- **deepCR Masks:** Per-frame cosmic-ray masks — mask-only, never inpainted. -- **Per-Frame PSFs:** Native ePSFs per chip, and the honest star-poor outcome. -- **The Frame-Combined Mosaic PSF:** `psf_from_frames` and the drop-convolution combine. -- **Registration Residuals:** Measured frame-to-frame alignment, and its reliability flag. -- **Load A Frame In PyAutoLens:** Each frame pair loads at native scale. -- **Wrap Up:** Where to go next in the workspace. - -__Imports__ -""" -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. - -The output goes to its own subfolder so the `start_here.py` products stay untouched; the -exposure cache is shared, so no re-download happens if you ran that script first. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # Shared with every other script in this folder. -OUTPUT_ROOT = WORKSPACE / "output" / "individual" # Own output root: mosaic + frames/ land here. - -""" -__Target Spec__ - -Two dials change from the default reference reduction: - -- `frame_products=True` — package every calibrated chip covering the target as a - native-pixel dataset under `frames/`, in addition to the mosaic. (HST, JWST and Keck only; - the pipeline fails fast on any other instrument.) - -- `psf_from_frames=True` — build the *mosaic's* PSF by combining the per-frame ePSFs through - the drizzle geometry, instead of measuring stars on the resampled mosaic. More on this below. - -The mosaic path itself is untouched by `frame_products` — the pipeline's validation -byte-compares the mosaic products between a flag-off and flag-on run. -""" -SPEC = TargetSpec( - name="slacs0008-0004", # Same target and cache entry as start_here.py. - ra=2.012333, # Right ascension in degrees. - dec=-0.068944, # Declination in degrees. - proposal_ids=("10886",), # The SLACS ACS program. - frame_products=True, # Package per-exposure native-pixel datasets under frames/. - psf_from_frames=True, # Mosaic PSF from combined per-frame ePSFs (drop-convolution route). -) - -""" -__Run__ - -The frame packaging runs after the ordinary pipeline (so the driz_cr DQ flags exist in the -exposures) and before any cache eviction could delete the frames it reads. -""" -print( - "\n" - "Running the reduction with frame products enabled.\n" - "Warm cache (after start_here.py): a few minutes, plus deepCR inference per chip.\n" - "Cold cache: add the ~0.5 GB MAST download (~10-30 minutes).\n" -) - -record = reduce_target(SPEC, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / SPEC.name -frames_dir = out_dir / "frames" - -print("Reduction complete.") -print("\n--- frames fragment (reduction.json) ---") -print(json.dumps(record["frames"], indent=2)) - -""" -__The Frames Tree__ - -On disk, next to the usual mosaic products: - - output/individual/slacs0008-0004/ - data.fits, noise_map.fits, psf.fits, psf_full.fits, reduction.json <- the mosaic, as ever - frames/ - manifest.json - _chip1/ {data.fits, noise_map.fits, dq.fits, cr_mask.fits [, psf.fits, psf_full.fits]} - _chip2/ {...} - -One directory per (exposure, SCI chip) that covers the target. ACS/WFC has two CCD chips per -exposure; chips whose footprint misses the target are skipped and recorded — so do not expect -2 x n_exposures directories. - -Each chip directory is a self-contained dataset: `data.fits` + `noise_map.fits` load directly as -an imaging dataset, `dq.fits` (int32) keeps the full calibration DQ bit information, and -`cr_mask.fits` (uint8) the cosmic-ray mask, for any consumer wanting a different masking policy -than the shipped one. -""" -chip_dirs = sorted(d for d in frames_dir.iterdir() if d.is_dir()) -print("\n--- frames tree ---") -for d in chip_dirs: - contents = sorted(p.name for p in d.iterdir()) - print(f" {d.name}: {contents}") - -""" -__Manifest Walk__ - -`frames/manifest.json` (schema version 2) is the frame products' contract. Top level: - -- `frame_cutout_shape` — the native-pixel stamp shape, derived from the mosaic dials - (`cutout_shape * final_scale / native_scale`, odd-forced): the same sky footprint as the - mosaic cutout, no new user dial. -- `native_scale` — 0.05 arcsec/pix for ACS/WFC; the pixel scale every frame loads at. -- `data_units` — e-/s: each SCI/ERR chip is converted from electrons so frames and mosaic share - the cps flux scale. -- `cr_method` — the per-frame CR machinery (deepCR on ACS; see below). -- `dq_semantics` — the masking policy the shipped noise-maps encode. -- `max_registration_residual_px` — the at-a-glance registration verdict (final section). -- `frames` — one entry per chip directory; `skipped_chips` the off-target ones. -""" -manifest = json.loads((frames_dir / "manifest.json").read_text()) - -print("\n--- manifest (top level) ---") -print(f"version : {manifest['version']}") -print(f"frame_cutout_shape : {manifest['frame_cutout_shape']}") -print(f"native_scale : {manifest['native_scale']}") -print(f"data_units : {manifest['data_units']}") -print(f"cr_method : {manifest['cr_method']}") -print(f"driz_cr_run : {manifest['driz_cr_run']}") -print(f"max_registration_residual_px: {manifest['max_registration_residual_px']}") -print(f"frames : {len(manifest['frames'])}") -print(f"skipped_chips : {len(manifest['skipped_chips'])}") - -entry = manifest["frames"][0] -print("\n--- first frame entry ---") -print(json.dumps(entry, indent=2)) - -""" -__Per-Frame Noise — No R__ - -Here is the selling point, in the numbers. The per-frame noise-map is the `calacs`-propagated -ERR extension — native-pixel Poisson + read noise + dark, computed by the calibration pipeline -itself (ACS Data Handbook, https://hst-docs.stsci.edu/acsdhb) — unit-converted alongside the -science array. **No Casertano factor is applied, because nothing has been resampled.** Every -pixel's noise is genuinely independent, so a chi^2 over these frames is exactly what its -statistics claim, with no scalar-correction approximation anywhere. - -Compare the provenance: the mosaic's noise block records `correlated_noise_factor` = R > 1; the -frame products record none, structurally. The masking policy is "masked-by-noise": any nonzero -DQ bit, deepCR cosmic-ray pixel, off-chip pixel or non-finite ERR pixel gets noise = 1e8 with the -data zeroed, so masked pixels drop out of any likelihood without a separate mask file. -""" -frame_data = fits.getdata(frames_dir / entry["dir"] / "data.fits").astype(float) -frame_noise = fits.getdata(frames_dir / entry["dir"] / "noise_map.fits").astype(float) - -good = frame_noise < 1.0e7 -print("\n--- per-frame noise (first frame) ---") -print(f"mosaic R (for contrast) : {record['drizzle']['correlated_noise_factor']:.3f}") -print("frame R : none — nothing resampled") -print(f"median frame noise (e-/s): {np.median(frame_noise[good]):.4f}") -print(f"masked pixels : {entry['n_masked_pixels']} " - f"({entry['n_masked_pixels'] / frame_noise.size:.2%} of the stamp)") - -""" -__Sky Pedestal__ - -A subtlety the manifest records per frame: AstroDrizzle's sky subtraction is *virtual* — the -`globalmin+match` sky is stored in each chip's `MDRIZSKY` header keyword and subtracted only -during the drizzle, leaving the `_flc` files untouched. A naively-cut frame would therefore carry -a sky pedestal the mosaic lacks, and joint frame+mosaic modeling would disagree about the -background by exactly that amount. - -The frame packaging subtracts each chip's `MDRIZSKY` explicitly, and records the value and -keyword (`sky_subtracted`, `sky_keyword`) in the manifest entry — auditable, like everything -else. -""" -print("\n--- sky pedestal (first frame) ---") -print(f"sky_subtracted : {entry['sky_subtracted']} (keyword {entry['sky_keyword']})") - -""" -__deepCR Masks__ - -Frames need a cosmic-ray answer the mosaic path cannot give them. `driz_cr` flags CRs against a -median *stack*, so its DQ flags exist only where several exposures overlap — and per-frame -modeling needs a mask for every frame *on its own*, single-exposure visits included. - -The per-frame machinery is deepCR (Zhang & Bloom 2020, ApJ 889, 24, -https://ui.adsabs.harvard.edu/abs/2020ApJ...889...24Z): a CNN that flags cosmic-ray pixels in -individual exposures, trained largely on exactly this ACS/F814W regime, with higher completeness -than the classical Laplacian-edge method (LACosmic, van Dokkum 2001, PASP 113, 1420) at fixed -false-positive rate. On ACS/WFC the published `ACS-WFC` model is used, and the manifest records -the exact model and threshold so datasets remain re-maskable later. - -**Mask-only by contract**: deepCR can also *inpaint* the flagged pixels, and **PyAutoReduce** -never uses that — bad pixels are masked (noise = 1e8), never fabricated. `cr_mask.fits` keeps -the raw mask. -""" -total_cr = sum(e["n_cr_pixels"] for e in manifest["frames"]) -print("\n--- deepCR ---") -print(f"CR pixels flagged across frames: {total_cr}") -for e in manifest["frames"]: - print(f" {e['dir']}: {e['n_cr_pixels']} CR pixels") - -""" -Let's look at one frame: data, noise-map and CR mask side by side. Cosmic rays are obvious in -the native frame — sharp tracks that the mosaic (where the stack rejection removed them) never -shows you. -""" -cr_mask = fits.getdata(frames_dir / entry["dir"] / "cr_mask.fits") - -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -sky_sigma = np.median(frame_noise[good]) - -fig, axes = plt.subplots(1, 3, figsize=(15, 5)) -axes[0].imshow(np.arcsinh(frame_data / (3.0 * sky_sigma)), origin="lower", cmap="magma") -axes[0].set_title(f"{entry['dir']} data (native pixels)") -axes[1].imshow(np.clip(frame_noise, 0.0, 5.0 * sky_sigma), origin="lower", cmap="viridis") -axes[1].set_title("noise_map (ERR-based, no R)") -axes[2].imshow(cr_mask, origin="lower", cmap="gray") -axes[2].set_title("cr_mask (deepCR)") -for ax in axes: - ax.set_xticks([]) - ax.set_yticks([]) -fig.tight_layout() -plot_path = plot_dir / "individual_frame.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"\nPlot saved to: {plot_path}") - -""" -__Per-Frame PSFs__ - -Each chip directory that can support it also ships `psf.fits` / `psf_full.fits`: an effective PSF -built from the frame's *own* full chip, on native (undrizzled, distorted) pixels — the correct -PSF for modeling that frame. DQ-flagged pixels are screened out of the star finding, the target -itself is excluded, and the saturation cap is formed in the frame's native units. - -The honest outcome is recorded, not hidden: **insufficient stars is a recorded result, not a -hard stop**. A single ~500 s frame legitimately may lack the minimum usable stars; its data -products remain useful, the manifest `psf` block says `method: "none"` with the reason, and the -reduction prints a loud notice that such a frame is not modelable until a model-PSF tier exists -(it currently does not — see `psf.py` for that honesty story on the mosaic side too). -""" -print("\n--- per-frame PSFs ---") -for e in manifest["frames"]: - psf_block = e["psf"] - method = psf_block["method"] - extra = "" if method == "none" else f", n_stars={psf_block.get('n_stars_used')}" - print(f" {e['dir']}: method={method}{extra}") - -""" -__The Frame-Combined Mosaic PSF__ - -`psf_from_frames=True` changed how the *mosaic's* PSF was built, too. Instead of measuring stars -on the resampled mosaic, each frame's native ePSF is: - -1. convolved with the drizzle drop (a `final_pixfrac`-wide box, applied as an exact - fractional-width Fourier convolution), -2. resampled onto the mosaic grid through the local frame-to-mosaic WCS Jacobian at the target - position, and -3. exposure-time-weighted averaged. - -This honours the drizzled-PSF invariant *by construction* — it applies the drizzle geometry to -the PSF itself — while sidestepping mosaic resampling artifacts and star scarcity (every frame's -full star field contributes, rather than only the stars that survive on one resampled image). -The recorded approximation: local-affine geometry plus drop convolution; sub-pixel -output-sampling phases are not modelled. It is loud when no frame yields an ePSF — it never -silently falls back to mosaic stars. -""" -print("\n--- mosaic PSF (frame-combined) ---") -print(json.dumps(record["psf"], indent=2)) - -""" -__Registration Residuals__ - -Joint multi-frame modeling needs to know how well the frames agree about where the sky is. Each -manifest entry carries a `registration` block with two very different kinds of number — the -manifest's own `registration_note` spells out the distinction, because the header keywords invite -misreading: - -- `wcsname` / `wcstype` / `rms_ra_mas` / `rms_dec_mas` / `nmatches` — the astrometric solution - behind the frame's WCS, stating the group's **absolute** alignment to an external catalog - (for this target, ~44 mas to GSC 2.4.2). Absolute accuracy is *not* what frame-joint modeling - consumes. - -- `residual_dy_px` / `residual_dx_px` — the **measured relative** registration error against the - reference frame: resample through both shipped WCS, phase-correlate, read the shift. This is - the number that matters, and on this target it sits at or below ~0.1 native pixels. - -- `residual_reliable` — the measurement's own honesty flag: where CR-masked pixels bite the - source, the estimator degrades to ~0.1-0.3 px, and heavily-masked pairs are flagged - unreliable rather than reported as precise. - -The default modeling stance: treat the shipped WCS as known (the residuals sit below the scales -standard modeling constrains); for precision work, free per-frame (dy, dx) nuisance parameters -with Gaussian priors of the recorded residual width. -""" -print("\n--- registration residuals ---") -for e in manifest["frames"]: - reg = e["registration"] - print( - f" {e['dir']}: dy={reg['residual_dy_px']:+.3f} px, dx={reg['residual_dx_px']:+.3f} px, " - f"reliable={reg['residual_reliable']}, reference={reg['reference']}, " - f"absolute={reg['wcsname']}" - ) -print(f"headline: max residual {manifest['max_registration_residual_px']} px") - -""" -__Load A Frame In PyAutoLens__ - -Each frame pair loads as an imaging dataset at the native pixel scale from the manifest — and, -because of the masked-by-noise convention, needs no mask file for its DQ/CR/off-chip pixels. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "\nPyAutoLens is not installed, so the loading demonstration is skipped.\n" - "Install it with `pip install autolens`. The frame products themselves are complete." - ) - -if al is not None: - frame_dir = frames_dir / entry["dir"] - - dataset = al.Imaging.from_fits( - data_path=frame_dir / "data.fits", - noise_map_path=frame_dir / "noise_map.fits", - psf_path=frame_dir / "psf.fits" if (frame_dir / "psf.fits").exists() else None, - pixel_scales=manifest["native_scale"], # Native ACS pixels — 0.05"/pix, from the manifest. - ) - print(f"\nLoaded frame {entry['dir']} into PyAutoLens: shape {dataset.shape_native} " - f"at {dataset.pixel_scales} arcsec/pix.") - print( - "Fit all frames simultaneously by creating one analysis per frame and summing the\n" - "likelihoods — see the autolens_workspace multi-dataset examples for the pattern." - ) - -""" -__Wrap Up__ - -You now have both representations of the same photons: one drizzled mosaic (convenient, one -PSF, one dataset — but correlated noise, scalar-corrected) and N native frames (uncorrelated -noise, exact per-frame PSFs, registration shipped as information — but N datasets to model -jointly). Which to use is a modeling decision; the reduction ships both honestly. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/psf.py`: the PSF story, including the per-frame tiers seen here. -- `scripts/hst_acs/dials.py`: the correlated-noise numbers (R vs pixfrac) that motivate frame - products in the first place. -- `scripts/hst_acs/simulator.py`: injection testing — the frames are where injection happens. -- `scripts/guides/noise_maps.py`: noise recipes and why chi^2 cares about correlation. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_acs/psf.py b/workspace/scripts/hst_acs/psf.py deleted file mode 100644 index 67e926b..0000000 --- a/workspace/scripts/hst_acs/psf.py +++ /dev/null @@ -1,402 +0,0 @@ -""" -HST ACS: PSF -============ - -No product **PyAutoReduce** ships is more consequential — or easier to get subtly wrong — than -the PSF. A lens model convolves every trial image with it before comparing to the data, so a PSF -error does not average away like noise: it imprints *structured* residuals exactly where the -science lives, at the lensed arcs and the deflector core. Those residuals bias mass-model -parameters, and in substructure work they are actively dangerous — a PSF mismatch can mimic the -perturbation of a dark subhalo, or mask a real one. Empirical PSF craft is why the weak-lensing -community abandoned pure optical models for HST years ago, and lens modeling inherits that -lesson wholesale. - -This script is the full ACS PSF story: where the effective-PSF idea comes from, how stars are -selected, which drizzle pass they are measured on (and the dial that controls it), the STARRED -alternative backend and the regimes where it wins and loses, what happens on star-poor fields -(an honest hard stop, currently), the model-PSF literature **PyAutoReduce** does *not* yet wrap, -and the diagnostics that let you judge every shipped kernel. - -It runs the SLACS anchor up to three times (default pass, `no_cr` star pass, and — if the -optional extra is installed — the STARRED backend), sharing one exposure cache throughout. Warm -cache: a few minutes per run, plus one extra AstroDrizzle pass for the `no_cr` leg. - -__Contents__ - -- **Imports:** Import **PyAutoReduce** and the other libraries we need. -- **Paths:** Anchor the cache and output locations to the workspace root. -- **Why PSF Errors Bias Lens Models:** Structured residuals at arcs; fake or masked substructure. -- **The ePSF Lineage:** Anderson & King 2000 to `photutils.EPSFBuilder`. -- **Star Selection:** The cuts, their defaults, and why they are not TargetSpec dials. -- **The Default Run:** Tier-1 ePSF on the SLACS anchor. -- **The Star Pass Dial:** `psf_star_pass` — "auto", "science", "no_cr", and the 344-to-599 story. -- **STARRED (Tier 1b):** The super-sampled backend, its regimes, and its validation numbers. -- **The Star-Poor Hard Stop:** Tier 2 is a stub — honesty about what cannot ship yet. -- **Model PSFs In The Literature:** TinyTim and focus-diverse ePSF libraries, as context. -- **Quality Diagnostics:** fwhm_pix, n_stars_used, moment FWHM — and the FWHM-definition wart. -- **The Drizzled-PSF Invariant:** Why the delivered PSF matches the mosaic geometry, always. -- **psf.fits vs psf_full.fits:** The compact kernel and the wings. -- **Plots:** The PSF at log stretch, and its radial profile. -- **Wrap Up:** Where to go next in the workspace. - -__Imports__ -""" -import dataclasses -import importlib.util -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target -from autoreduce.psf.moments import moment_fwhm -from autoreduce.psf.stars import StarSelection - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. - -Each PSF experiment gets its own output root; the exposure cache is shared, so the archive is -hit at most once. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # Shared with every other script in this folder. -OUTPUT_ROOT = WORKSPACE / "output" / "psf" # One subfolder per PSF experiment below. - -SPEC = TargetSpec( - name="slacs0008-0004", # The SLACS parity anchor; same cache entry as start_here.py. - ra=2.012333, # Right ascension in degrees. - dec=-0.068944, # Declination in degrees. - proposal_ids=("10886",), # The SLACS ACS program. -) - -""" -__Why PSF Errors Bias Lens Models__ - -A pixel-based lens likelihood compares `PSF (x) model_image` against the data. If the kernel is -too narrow, every fit leaves a positive-ringed residual at the deflector core and sharpened arc -edges; too broad, the reverse. Unlike noise these residuals are deterministic and spatially -coherent — they sit at the arcs, which is precisely where the mass model gets its constraining -power. Consequences, in increasing severity: - -- biased deflector light subtraction (the core is pure PSF at small radii); -- biased mass-model parameters (the arc surface-brightness gradients drive the fit); -- spurious or hidden substructure: a percent-level PSF error produces exactly the kind of - localized arc residual a dark subhalo would. - -This is why pure optical-model PSFs (TinyTim, below) fell out of favour at lensing fidelity — -they struggle with HST's time-variable focus ("breathing"), source-SED effects, CTE trailing and -charge diffusion, and their mismatch shows up as structured arc residuals that mimic or mask -substructure. The modern default, here and across HST practice, is *empirical*: measure the PSF -from stars in your own data. - -__The ePSF Lineage__ - -The construction **PyAutoReduce** uses is the *effective PSF* (ePSF) of Anderson & King 2000 -(PASP 112, 1360, https://ui.adsabs.harvard.edu/abs/2000PASP..112.1360A): the instrumental PSF -convolved with the pixel response, derived empirically by iterating between star-centroid -estimates and an oversampled PSF model built from many dithered star images. It is the -foundation of essentially all modern HST PSF work. - -The implementation is `photutils.EPSFBuilder` -(https://photutils.readthedocs.io/en/stable/user_guide/epsf.html) — the open-source Anderson & -King iteration — run on star cutouts from the drizzled mosaic, at 2x oversampling, then -evaluated back onto the mosaic pixel grid and unit-normalised. At least 8 usable stars are -required; fewer is a loud `InsufficientStarsError`, not a degraded kernel. - -__Star Selection__ - -Which stars feed the builder is governed by fixed selection cuts — deliberately *not* -`TargetSpec` dials. PSF fidelity should not vary target-to-target with user whim; the cuts are -tuned once for ACS-like mosaics and recorded here so you know exactly what they are: -""" -selection = StarSelection() - -print("--- StarSelection defaults (not TargetSpec-tunable) ---") -print(f"detection_sigma : {selection.detection_sigma} (DAOStarFinder threshold, x sky RMS)") -print(f"fwhm_pix : {selection.fwhm_pix} (detection kernel FWHM)") -print(f"sharp_range : {selection.sharp_range} (rejects CRs/extended sources)") -print(f"round_limit : {selection.round_limit} (rejects elongated detections)") -print(f"saturation_fraction : {selection.saturation_fraction} (of full well — ACS ~80 ke-)") -print(f"min_separation_pix : {selection.min_separation_pix} (uncrowded: no neighbour within)") -print(f"edge_margin_pix : {selection.edge_margin_pix} (full extraction window on-image)") -print(f"exclusion_radius_pix: {selection.exclusion_radius_pix} (around the lens itself)") - -""" -The physics behind the cuts: saturated stars have flat-topped, bleeding profiles (rejected by -the peak cap at 70% of the ~80,000 e- ACS full well); crowded stars carry neighbour flux into -the model; edge stars have truncated extraction windows; and the target itself — the lens galaxy -— must never contaminate its own PSF. - -__The Default Run__ - -Run the reference reduction and read its PSF provenance. -""" -print( - "\n" - "Run 1/2: default PSF configuration.\n" - "Warm cache: a few minutes. Cold cache: adds the ~0.5 GB MAST download.\n" -) - -record_auto = reduce_target(SPEC, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT / "auto") - -print("\n--- psf block, default run ---") -print(json.dumps(record_auto["psf"], indent=2)) - -""" -__The Star Pass Dial__ - -Note `star_source_pass` in that block. It answers a question you might not have thought to ask: -*which image were the stars measured on?* — and it exists because the obvious answer is wrong. - -The science mosaic has been through `driz_cr` cosmic-ray rejection, and driz_cr's blotted-median -reference reads systematically low on steep gradients — which includes *star cores*. Sub-pixel -dither shifts smear the median's peak, the core pixels deviate, and genuine stellar flux gets -flagged as cosmic rays. Star finding on the science mosaic then sees cored, hole-punched stars: -fewer survive the sharpness cuts, and the survivors are damaged. Measured across the pipeline's -validation fields, rebuilding star selection from a CR-flag-ignoring pass raised the usable star -count from 344 to 599 (+74%) and rescued four lens/filter pairs that would otherwise have had no -viable ePSF at all. - -`TargetSpec.psf_star_pass` controls this, with the star pass decoupled from the shipped science -mosaic: - -- `"auto"` (default) — never adds a drizzle. The science mosaic is used, and the *reason* is - recorded: on the single-exposure branch and the `cr_method="deepcr"` route it is genuinely the - least-CR-rejected pass available, and otherwise it is the only pass available without doubling - combine time. -- `"no_cr"` — the explicit opt-in: drizzle a second, CR-flag-ignoring star pass (`final_bits` - gains the CR bit; `resetbits=0` so the science DQ flags survive for frame products) onto the - same grid — same kernel, pixfrac, scale and rotation, so the drizzled-PSF invariant holds. - Costs one extra AstroDrizzle run. -- `"science"` — pin star finding to the shipped mosaic, explicitly. - -Whatever happens, `reduction.json` records `star_source_pass` (and the reason, when no second -pass ran) — the coupling can never silently regress. Let's buy the second pass and compare: -""" -print( - "\n" - "Run 2/2: psf_star_pass='no_cr' — adds a second, CR-flag-ignoring AstroDrizzle pass\n" - "for star finding. Warm cache: a few extra minutes.\n" -) - -spec_no_cr = dataclasses.replace(SPEC, psf_star_pass="no_cr") # Frozen spec -> variants via replace(). - -record_no_cr = reduce_target( - spec_no_cr, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT / "no_cr" -) - -print("\n--- star pass comparison ---") -print(f"auto : pass={record_auto['psf']['star_source_pass']}, " - f"n_stars={record_auto['psf']['n_stars_used']}") -print(f" reason: {record_auto['psf'].get('star_source_reason')}") -print(f"no_cr: pass={record_no_cr['psf']['star_source_pass']}, " - f"n_stars={record_no_cr['psf']['n_stars_used']}") - -""" -On sparse fields like this one the star-count gain is modest; on richer fields it is the -difference between a viable ePSF and none. If your reduction reports few stars and a -`star_source_pass` of "science", the `no_cr` pass is the first thing to try. - -__STARRED (Tier 1b)__ - -`TargetSpec.psf_backend="starred"` swaps the photutils builder for STARRED (Michalewicz et al. -2023, JOSS 8(85), 5340; Millon et al. 2024, https://arxiv.org/abs/2402.08725) — the -COSMOGRAIL/lensing community's JAX-based PSF reconstruction: an analytic Moffat core plus a -starlet-regularized, super-sampled residual grid, fit jointly to the same field stars Tier 1 -selects, weighted by the per-pixel noise-map. - -It is a *conditional* upgrade, not a universal one. The pipeline's validation mapped the regimes -on real data: - -- **Wins — well-sampled and crowded/few-star fields.** On an Omega Cen WFC3/UVIS F606W field, - STARRED's ePSF concentration (0.54) matched the empirical star-stack (0.58) far better than - the photutils Tier-1 build (0.39 — under-concentrated and neighbour-contaminated in the - crowd). On JWST NIRCam it matched the empirical PSF on the well-sampled long-wavelength - channel (F277W) where the photutils build collapsed on few blended stars. - -- **Loses — undersampled data.** On undersampled NIRCam short-wavelength imaging (F150W at - 0.03"/pix, PSF ~1.7 px) STARRED *broadens* — excess starlet-channel wings — and photutils - wins. The backend flags this regime itself: kernels with FWHM below 1.6 px carry an - `undersampled` warning in the diagnostics. ACS/WFC at 0.05"/pix sits near this boundary, so - check the flag. - -Two constraints keep STARRED optional: it is GPL-licensed (the PyAuto* stack is permissive) and -it depends on JAX — so it ships only as the extra `pip install "autoreduce[starred]"`, imported -lazily. If requested and unavailable the pipeline raises loudly (`StarredUnavailableError`); it -**never** silently falls back to Tier 1. We run it only if importable: -""" -if importlib.util.find_spec("starred") is not None: - print("\nOptional run: psf_backend='starred' (STARRED tier 1b).\n") - spec_starred = dataclasses.replace(SPEC, psf_backend="starred") - record_starred = reduce_target( - spec_starred, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT / "starred" - ) - print("\n--- psf block, STARRED run ---") - print(json.dumps(record_starred["psf"], indent=2)) -else: - print( - "\n[starred] the STARRED extra is not installed — skipping the tier-1b run.\n" - '[starred] install it with: pip install "autoreduce[starred]" (GPL + JAX).' - ) - -""" -__The Star-Poor Hard Stop__ - -Honesty section. SLACS-like fields — an isolated elliptical in a snapshot pointing — are often -*star-poor*: fewer than the 8 usable stars Tier 1 needs. The designed answer is a Tier-2 *model* -PSF (a focus-diverse ePSF grid or TinyTim raytrace, drizzled through the same footprint as the -science frames). **That tier is currently a stub**: requesting it raises -`ModelPSFUnavailableError` unconditionally. A star-poor ACS field therefore cannot ship a PSF -today — the reduction stops loudly rather than delivering a kernel it cannot stand behind, and -the frame products (`individual.py`) record `method: "none"` per star-poor frame for the same -reason. - -If you hit this: try `psf_star_pass="no_cr"` first (it exists exactly to rescue marginal -fields), then STARRED (robust at low star counts on well-sampled data). If both fail, the field -genuinely lacks the stars, and you should treat any externally-sourced PSF with the suspicion -this script's opening section motivates. - -__Model PSFs In The Literature__ - -Context for what the Tier-2 stub will eventually wrap — literature, not implemented capability: - -- **TinyTim** (Krist 1995; Krist, Hook & Stoehr 2011, Proc. SPIE 8127): the classical HST - optical raytrace model. SLACS itself used it — Bolton et al. 2008 (SLACS V, - https://arxiv.org/abs/0805.1931) rectified a TinyTim PSF with *identical sampling* to their - rectified snapshot images, a clean precedent for the drizzled-PSF invariant below. Its known - weaknesses at lensing fidelity: time-variable focus/breathing, SED dependence, CTE trailing, - charge diffusion. -- **Focus-diverse ePSF libraries**: STScI's modern empirical alternative — ePSF grids indexed by - focus, with the focus estimated from a handful of stars in the science frame itself (the - COSMOS weak-lensing method recovered focus to <1 micron rms from few stars; Rhodes et al. - 2007, https://arxiv.org/abs/astro-ph/0701480). For ACS/WFC: Bellini et al. 2018 (ACS ISR - 2018-08) built the library, and ACS ISR 2023-06 documents the retrieval tooling - (`acstools.focus_diverse_psfs`); STScI's maintained notebooks demonstrate it - (https://spacetelescope.github.io/hst_notebooks/). - -- **Not any tier — target-based PSF reconstruction.** Reconstructing the PSF from the science - point sources themselves (lensed quasar images) is deliberately out of *reduction* scope: it - entangles the PSF with the lensed arc and belongs to the modeling stage. **PyAutoReduce** - ships the inputs and stops. - -__Quality Diagnostics__ - -Every shipped PSF carries diagnostics in `reduction.json`; judge the kernel before you model -with it: - -- `n_stars_used` — more stars, better-constrained ePSF; near the minimum of 8, be cautious. -- `fwhm_pix` — a *crude* radial half-max estimate, recorded as a build diagnostic. -- `star_source_pass` — which drizzle pass the stars came from (above). - -One wart to know about, stated plainly: **different PSF tiers use different FWHM definitions.** -The tier-1 `fwhm_pix` is a radial half-max-extent estimate; the moment-based estimator below is -a Gaussian-sigma proxy (2.3548 x second moment); and the Keck tier-A vetting statistic is an -equivalent-area measure. They agree only for a Gaussian kernel — never cross-compare FWHM -numbers between tiers or instruments. Within one tier, they track relative quality fine. - -The moment estimator is public, so we can compute it ourselves and see the wart directly: -""" -out_auto = OUTPUT_ROOT / "auto" / SPEC.name -psf = fits.getdata(out_auto / "psf.fits").astype(float) -psf_full = fits.getdata(out_auto / "psf_full.fits").astype(float) - -print("\n--- FWHM, two definitions, same kernel ---") -print(f"recorded fwhm_pix (radial half-max): {record_auto['psf']['fwhm_pix']:.2f}") -print(f"moment_fwhm (2.3548 x sigma proxy) : {moment_fwhm(psf):.2f}") -print("These differ by construction — compare within one definition only.") - -""" -__The Drizzled-PSF Invariant__ - -The non-negotiable rule, whatever tier produced the kernel: **the delivered PSF is the drizzled -PSF** — measured (or evaluated) at the same kernel, pixfrac, scale and orientation as the -science mosaic. The mosaic's blur is the *drizzled* blur; a native-frame PSF paired with a -drizzled image is simply the wrong kernel, and every path in **PyAutoReduce** honours this by -construction: mosaic stars are measured on a same-geometry drizzle pass, the `no_cr` star pass -reuses the exact science grid, and the frame-combined route (`individual.py`) applies the -drizzle drop and WCS geometry to the per-frame ePSFs explicitly. - -__psf.fits vs psf_full.fits__ - -Two kernels ship per dataset, both odd-shaped, centred and unit-normalised: - -- `psf.fits` (21x21, ~1" across) — the fit-convolution kernel. Big enough to capture the core - and the first Airy structure — the great majority of the blurring — while keeping model-image - convolution fast. -- `psf_full.fits` (61x61, ~3") — the extended kernel with the wings and diffraction-spike - structure, for work where far-flung PSF flux matters (bright point sources, lensed quasars). -""" -print("\n--- kernel properties ---") -print(f"psf.fits : {psf.shape}, sum={psf.sum():.6f}") -print(f"psf_full.fits : {psf_full.shape}, sum={psf_full.sum():.6f}") - -""" -__Plots__ - -A PSF should always be inspected at log stretch — the core saturates any linear display, and the -structure you need to check (asymmetry, neighbour contamination, wings) lives orders of -magnitude down. We also plot the radial profile of both kernels. -""" -plot_dir = out_auto / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(1, 3, figsize=(15, 5)) - -floor = psf_full[psf_full > 0].min() -axes[0].imshow(np.log10(np.clip(psf, floor, None)), origin="lower", cmap="magma") -axes[0].set_title("psf.fits (log10)") -axes[1].imshow(np.log10(np.clip(psf_full, floor, None)), origin="lower", cmap="magma") -axes[1].set_title("psf_full.fits (log10)") - -def radial_profile(kernel): - ny, nx = kernel.shape - y, x = np.mgrid[0:ny, 0:nx] - r = np.hypot(y - ny // 2, x - nx // 2).astype(int) - profile = np.bincount(r.ravel(), weights=kernel.ravel()) / np.bincount(r.ravel()) - return profile - -axes[2].semilogy(radial_profile(psf), label="psf (21x21)") -axes[2].semilogy(radial_profile(psf_full), label="psf_full (61x61)", linestyle="--") -axes[2].set_xlabel("radius (pix, 0.05\"/pix)") -axes[2].set_ylabel("mean pixel value") -axes[2].set_title("radial profiles") -axes[2].legend() - -fig.tight_layout() -plot_path = plot_dir / "psf_diagnostics.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"\nPlot saved to: {plot_path}") - -""" -The two profiles should lie on top of each other over the inner ~10 pixels — same PSF, different -window — with `psf_full` continuing smoothly into the wings. - -__Wrap Up__ - -The PSF story in one paragraph: empirical ePSFs from carefully-selected field stars, measured on -the least-CR-damaged drizzle pass available (a recorded, dialable choice), on the exact geometry -of the science mosaic; a super-sampled STARRED alternative for well-sampled or crowded fields; a -loud hard stop — not a degraded kernel — where stars are insufficient; and diagnostics with -every dataset so none of this is taken on faith. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/individual.py`: per-frame native ePSFs and the frame-combined mosaic PSF. -- `scripts/hst_acs/simulator.py`: injection testing — the end-to-end check that the shipped PSF - and noise-map describe the data. -- `scripts/hst_acs/dials.py`: how pixfrac and kernel choices propagate into the PSF. -- `scripts/guides/output_contract.py`: the kernel standards (odd, centred, normalised) and how - **PyAutoLens** consumes them. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_acs/simulator.py b/workspace/scripts/hst_acs/simulator.py deleted file mode 100644 index dd87d65..0000000 --- a/workspace/scripts/hst_acs/simulator.py +++ /dev/null @@ -1,360 +0,0 @@ -""" -HST ACS: Simulator (Injection) -============================== - -How do you know a reduction pipeline is *right*? Parity against a legacy dataset (the SLACS -comparison behind this folder) checks consistency with history — but history's own reductions -carry their own choices. The sharper test is a controlled experiment: put a source of **known -brightness** into the data and check that the pipeline hands it back. If a synthetic lensed arc -of exactly 30 e-/s goes into the raw exposures, and 30 e-/s (within the noise-map's prediction!) -comes out of the packaged cutout, then the flux calibration, the drizzle, the PSF treatment and -the noise model have all been validated *together*, end to end. - -**PyAutoReduce** builds this in as the inject stage: give the `TargetSpec` a plain FITS image -and the pipeline deposits it into the real calibrated exposures — through each frame's own -distortion, blurred by each frame's own PSF, with its own Poisson noise — before the unmodified -pipeline runs. This script does exactly that on the SLACS anchor: it builds a synthetic -lensed-arc image in pure numpy (the math is shown — no lensing library needed), injects it, -reduces the field twice (clean and injected) off one shared cache, and measures the recovery. - -Warm cache: two pipeline runs, a few minutes each. Cold cache: add the ~0.5 GB MAST download. - -__Contents__ - -- **Imports:** Import **PyAutoReduce** and the other libraries we need. -- **Paths:** Anchor the cache and output locations to the workspace root. -- **Why Inject Into Real Frames:** What real data gives you that simulation from scratch cannot. -- **The Input Image Contract:** Units, sampling, non-negativity — and no PSF convolution. -- **Building A Synthetic Arc:** An analytic ring/arc profile, formula by formula. -- **The Clean Reduction:** The baseline run. -- **The Injected Reduction:** `dataclasses.replace` with the `inject_*` dials. -- **What The Inject Stage Did:** Per-frame deposit, PSF, Poisson draw, and the untouched cache. -- **The Difference Image:** Injected minus clean — two identically-processed datasets. -- **Flux Recovery:** Aperture photometry of the difference vs the injected truth. -- **The Injected Provenance Block:** Semi-synthetic data must say so, permanently. -- **Wrap Up:** Where to go next in the workspace. - -__Imports__ -""" -import dataclasses -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits -from astropy.wcs import WCS - -from autoreduce import TargetSpec, reduce_target - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. - -The clean and injected runs write to separate output roots but share one exposure cache — the -injection never touches the cached files (more on that below), so a single download serves both. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # Shared; never mutated by injection. -OUTPUT_ROOT = WORKSPACE / "output" / "simulator" # clean/ and injected/ subfolders below. - -SPEC = TargetSpec( - name="slacs0008-0004", # The SLACS parity anchor; same cache entry as start_here.py. - ra=2.012333, # Right ascension in degrees. - dec=-0.068944, # Declination in degrees. - proposal_ids=("10886",), # The SLACS ACS program. -) - -""" -__Why Inject Into Real Frames__ - -You could instead simulate an observation from scratch: render a lensed source, convolve with a -PSF model, add Gaussian-plus-Poisson noise on a clean grid. Every lens-modeling workspace does -this for method development, and it is the right tool there. But as a *pipeline validation* — or -as training data for machine learning that must survive contact with real images — synthetic- -from-scratch is missing everything that makes real data hard: - -- cosmic rays, with realistic morphology, at the real rate; -- the real sky pedestal and its frame-to-frame variation; -- the real correlated-noise structure the drizzle geometry induces; -- real bad pixels, saturation bleeds, and DQ structure; -- the real dither geometry and its coverage non-uniformity; -- the real PSF, wings and all. - -Injecting synthetic sources into *real* survey images is the established answer in wide-field -astronomy — Balrog (Suchyta et al. 2016, MNRAS 457, 786) is the archetype, built on the GalSim -rendering machinery (Rowe et al. 2015, A&C 10, 121) — and lens-finding has adopted it for -exactly the realism argument: HOLISMOKES XV (Cañameras et al., https://arxiv.org/abs/2411.18694) -paints lensed features onto real cutouts because injection preserves the observational -complexity synthetic-only training data lacks. **PyAutoReduce**'s inject stage is the same idea -one level deeper: into the calibrated *exposures*, before the pipeline, so every downstream -stage is exercised for free. - -__The Input Image Contract__ - -The inject stage takes a plain 2-D FITS image — deliberately free of any lensing-library -dependency — with a strict contract: - -- **Units: e-/s per pixel** (the ACS adapter's `inject_units`) — surface brightness on your - input grid, converted per frame to electrons using each exposure's own time. -- **Finite and non-negative** everywhere. -- **North-up**, centred at `inject_position` (default: the target itself), sampled at - `inject_pixel_scale` — which may be *finer* than the detector; the deposit is - flux-conserving. -- **NOT PSF-convolved.** This is the one people trip over: the pipeline convolves your image - with each frame's own tier-1 ePSF (or your `inject_psf`) as it deposits. Hand it the *true* - sky-plane surface brightness; if you pre-blur it, it gets blurred twice. - -__Building A Synthetic Arc__ - -We build a lensed-arc-like ring in pure numpy. No ray tracing — just an analytic profile that -*looks* like the thin, curved arcs strong lenses produce, which is all a flux-recovery test -needs. On a polar grid (r, phi) centred on the lens: - - radial : I_r(r) = exp( -1.678 * |r - r_E| / w ) # Sersic n=1 cross-section, - # peaking on the ring r = r_E - azimuthal: I_phi(phi) = exp( -(phi - phi_0)^2 / (2 sigma_phi^2) ) # a bright arc segment - + f_c * exp( -(phi - phi_0 - pi)^2 / (2 sigma_c^2) ) # + faint counter-arc - - I(r, phi) = I_r(r) * I_phi(phi), then normalised so sum(I) * 1 = F_total [e-/s] - -with r_E = 1.2" (an Einstein-radius-like ring size), w = 0.15" (arc thickness), a ~100-degree -bright arc and a fainter counter-image on the opposite side — the classic morphology of a -galaxy-scale lens. The 1.678 factor makes w the radial half-light scale of the n=1 profile. -""" -INJECT_PIXEL_SCALE = 0.025 # arcsec/pix — finer than the 0.05" detector; deposit is flux-conserving. -INJECT_FLUX_CPS = 30.0 # Total injected flux, e-/s — bright enough for a clean recovery test. - -shape = (241, 241) # 241 x 0.025" = ~6" across; comfortably contains the r_E = 1.2" ring. -cy, cx = shape[0] // 2, shape[1] // 2 - -yy, xx = np.mgrid[0 : shape[0], 0 : shape[1]] -r = np.hypot(yy - cy, xx - cx) * INJECT_PIXEL_SCALE # radius from the lens centre, arcsec -phi = np.arctan2(yy - cy, xx - cx) # position angle, radians - -r_einstein = 1.2 # ring radius, arcsec -width = 0.15 # radial arc thickness (n=1 scale), arcsec -phi_0 = np.pi / 3.0 # bright arc centre angle -sigma_phi = 0.9 # bright arc angular extent (~100 deg FWHM-ish), radians -counter_frac = 0.25 # counter-image brightness relative to the main arc -sigma_counter = 0.35 # counter-image angular extent, radians - -radial = np.exp(-1.678 * np.abs(r - r_einstein) / width) - - -def wrapped(delta): - """Angular difference wrapped to [-pi, pi], so arcs don't tear at the branch cut.""" - return np.angle(np.exp(1j * delta)) - - -azimuthal = np.exp(-wrapped(phi - phi_0) ** 2 / (2.0 * sigma_phi**2)) -azimuthal += counter_frac * np.exp( - -wrapped(phi - phi_0 - np.pi) ** 2 / (2.0 * sigma_counter**2) -) - -arc = radial * azimuthal -arc = INJECT_FLUX_CPS * arc / arc.sum() # Normalise: total flux = INJECT_FLUX_CPS e-/s exactly. - -assert np.isfinite(arc).all() and (arc >= 0.0).all() # The input contract, checked. - -input_dir = OUTPUT_ROOT -input_dir.mkdir(parents=True, exist_ok=True) -input_path = input_dir / "input_arc.fits" -fits.PrimaryHDU(arc.astype(np.float32)).writeto(input_path, overwrite=True) - -print(f"Synthetic arc written to: {input_path}") -print(f" total flux : {arc.sum():.3f} e-/s") -print(f" peak : {arc.max():.4f} e-/s per {INJECT_PIXEL_SCALE}\" pixel") - -""" -__The Clean Reduction__ - -First the baseline: the unmodified field, reduced exactly as in `start_here.py`. The recovery -measurement below is a *difference* of two identically-processed datasets, so the real lens -galaxy, the real arc, the sky and every static artifact subtract out — only the injected flux -(and noise) remains. -""" -print( - "\n" - "Run 1/2: clean reduction (the baseline).\n" - "Warm cache: a few minutes. Cold cache: adds the ~0.5 GB MAST download.\n" -) - -record_clean = reduce_target( - SPEC, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT / "clean" -) - -""" -__The Injected Reduction__ - -`TargetSpec` is frozen, so the injected variant is a `dataclasses.replace` — the four `inject_*` -dials on top of the identical spec. Everything else about the pipeline is unchanged: same -exposures, same drizzle, same noise recipe, same PSF machinery. -""" -spec_injected = dataclasses.replace( - SPEC, - inject_image=str(input_path), # The plain-FITS arc built above (absolute path, like all paths). - inject_pixel_scale=INJECT_PIXEL_SCALE, # Required with inject_image: the input's arcsec/pix. - inject_seed=1, # Seeds the Poisson draws; same seed = bit-identical injection, re-run to re-run. -) -# inject_position is omitted -> defaults to the target coordinates: the ring lands on the lens. - -print( - "\n" - "Run 2/2: injected reduction — same spec + inject_* dials, same shared cache.\n" -) - -record_injected = reduce_target( - spec_injected, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT / "injected" -) - -print("Both reductions complete.") - -""" -__What The Inject Stage Did__ - -Between acquisition and combination, for every cached exposure and every CCD chip: - -1. **Render**: the input image is deposited onto the chip's native pixels through the frame's - *full* distortion WCS — a flux-conserving drizzle-style deposit, so your 0.025" input grid - lands correctly on the 0.05" distorted detector pixels. -2. **Convolve**: the deposited image is blurred with the frame's own tier-1 ePSF (built from - that frame's stars), or `inject_psf` if you supplied one. -3. **Poisson**: expected electrons = cps x exposure time; an actual Poisson realisation is - drawn, seeded deterministically per file from `(inject_seed, crc32(filename))` — so the run - is exactly reproducible, but no two frames share a realisation. -4. **Update**: SCI gains the counts, ERR is updated in quadrature (the injected source carries - its own shot noise), and the headers gain INJECTED/INJIMG/INJSEED keywords. - -Critically, **the cache is never mutated**: injection operates on copies under the run's -`work/injected/` directory. The clean run and every future run see pristine exposures — which -is what made sharing one cache across both runs safe. - -__The Difference Image__ - -Load both cutouts and difference them. What remains should be *only* the injected arc, blurred -by the real PSF and carrying the real noise — riding on nothing, because everything real -subtracted out. -""" -clean_dir = OUTPUT_ROOT / "clean" / SPEC.name -inj_dir = OUTPUT_ROOT / "injected" / SPEC.name - -data_clean = fits.getdata(clean_dir / "data.fits").astype(float) -data_inj = fits.getdata(inj_dir / "data.fits").astype(float) -noise_inj = fits.getdata(inj_dir / "noise_map.fits").astype(float) -header_inj = fits.getheader(inj_dir / "data.fits") - -diff = data_inj - data_clean - -sky_rms = record_injected["noise"]["empirical_background_rms"] - -plot_dir = OUTPUT_ROOT / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(1, 3, figsize=(15, 5)) -axes[0].imshow(np.arcsinh(data_clean / (3.0 * sky_rms)), origin="lower", cmap="magma") -axes[0].set_title("clean data") -axes[1].imshow(np.arcsinh(data_inj / (3.0 * sky_rms)), origin="lower", cmap="magma") -axes[1].set_title("injected data") -axes[2].imshow(np.arcsinh(diff / (3.0 * sky_rms)), origin="lower", cmap="magma") -axes[2].set_title("difference (injected - clean)") -for ax in axes: - ax.set_xticks([]) - ax.set_yticks([]) -fig.tight_layout() -plot_path = plot_dir / "injection_difference.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"\nPlot saved to: {plot_path}") - -""" -The difference panel should show the ring-plus-counter-arc morphology you built above — now -PSF-blurred and drizzled — centred on the (subtracted-away) lens. - -__Flux Recovery__ - -The quantitative test: sum the difference image inside a 3" aperture at the injection position -and compare against the injected truth. The noise-map supplies the uncertainty prediction — the -quadrature sum of the per-pixel RMS over the aperture — so this simultaneously tests the flux -calibration *and* the noise model's absolute scale. -""" -xy = WCS(header_inj).world_to_pixel_values(SPEC.ra, SPEC.dec) -pixel_scale = record_injected["package"]["pixel_scale"] - -yy_c, xx_c = np.mgrid[0 : diff.shape[0], 0 : diff.shape[1]] -aperture = np.hypot(yy_c - xy[1], xx_c - xy[0]) * pixel_scale <= 3.0 - -# Exclude masked-by-noise pixels (1e8) from the aperture prediction. -usable = aperture & (noise_inj < 1.0e7) - -recovered = float(diff[usable].sum()) -noise_pred = float(np.sqrt((noise_inj[usable] ** 2).sum())) - -report = { - "injected_flux_cps": INJECT_FLUX_CPS, - "recovered_flux_cps_3arcsec": recovered, - "recovery_ratio": recovered / INJECT_FLUX_CPS, - "aperture_noise_cps": noise_pred, - "total_injected_e": record_injected["inject"]["total_injected_e"], - "n_frames_injected": len(record_injected["inject"]["frames"]), -} -(OUTPUT_ROOT / "recovery_report.json").write_text(json.dumps(report, indent=2)) - -print("\n--- flux recovery ---") -print(json.dumps(report, indent=2)) - -ok = abs(report["recovery_ratio"] - 1.0) < max(0.05, 3.0 * noise_pred / INJECT_FLUX_CPS) -print(f"\nRECOVERY {'OK' if ok else 'DISCREPANT'} " - f"(criterion: within 5% or 3x the aperture noise prediction)") - -""" -A recovery ratio of ~1 says the whole chain conserves flux: the per-frame deposit, the PSF -convolution (the 3" aperture is wide enough to recapture the PSF-scattered flux), the drizzle, -the cps unit handling and the cutout. A ratio consistently off by more than the noise allows -would localise a real bug — which is exactly how this test is used in the pipeline's own -validation. - -__The Injected Provenance Block__ - -The governing principle: **semi-synthetic data must never masquerade as real.** The injected -run's `reduction.json` carries an `inject` block recording the input image, its units and pixel -scale, the total input flux, the position, the PSF source used for the convolution, the seed, -the total electrons deposited, and a per-frame record. Any consumer of this dataset — including -you, in two years — can see at a glance that it is an injection experiment, and reproduce it -bit-for-bit. -""" -print("\n--- inject block (per-frame records truncated) ---") -inject_block = dict(record_injected["inject"]) -inject_block["frames"] = f"[{len(inject_block['frames'])} per-frame records]" -print(json.dumps(inject_block, indent=2)) - -print("\nThe clean run's record has no inject block:", - "inject" in record_clean, "(False = clean, as it should be)") - -""" -__Wrap Up__ - -You built a synthetic lensed arc from two exponentials and a normalisation, pushed it through -the real SLACS exposures — real cosmic rays, real sky, real PSF, real drizzle — and measured it -back to within the noise. That closed loop is the strongest single statement this workspace can -make about the reduction being right; it is also the template for building injection-based -training sets with fully-realistic systematics. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/individual.py`: the per-frame products — injection happens at frame level, - and the frames are where you can see it before the drizzle. -- `scripts/hst_acs/dials.py`: rerun this experiment at different pixfrac values to see the - noise-correlation story in the recovery uncertainties. -- `scripts/hst_acs/psf.py`: the frame ePSFs that blurred your injected arc. -- `scripts/guides/target_spec.py`: all the inject_* dials, including inject_position and - inject_psf, in one place. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_acs/start_here.py b/workspace/scripts/hst_acs/start_here.py deleted file mode 100644 index 3faefaf..0000000 --- a/workspace/scripts/hst_acs/start_here.py +++ /dev/null @@ -1,398 +0,0 @@ -""" -HST ACS: Start Here -=================== - -The Hubble Space Telescope's Advanced Camera for Surveys (ACS/WFC) is the reference instrument of -**PyAutoReduce**: the instrument every other pipeline in this workspace is described relative to, -and the one whose products set the quality bar. That bar is the SLACS survey — the ACS/WFC F814W -reductions of Bolton et al. 2008 (SLACS V, https://arxiv.org/abs/0805.1931), which sit behind the -largest and most-modeled sample of galaxy-scale strong lenses in existence. If a reduction -pipeline can reproduce a SLACS dataset from the raw archive, it can be trusted with your lens. - -This script runs that reference reduction end to end: it points **PyAutoReduce** at the SLACS lens -SDSS J0008-0004, downloads the calibrated exposures from the MAST archive, drizzles them to the -SLACS convention, constructs the RMS noise-map and empirical PSF, and packages a modeling-ready -dataset that loads directly into **PyAutoLens**. Everything runs on defaults — one frozen spec, -one function call. The first run downloads ~0.5 GB of exposures and reference files and takes -roughly 10-30 minutes depending on your connection; re-runs reuse the cache and take a few -minutes. - -If you want to see what each stage does internally, `step_by_step.py` in this folder walks the -same reduction stage by stage; `psf.py`, `individual.py`, `simulator.py` and `dials.py` go deeper -on the PSF, per-exposure products, injection testing and the drizzle dials respectively. - -__Contents__ - -- **Why SLACS Is The Quality Bar:** The reference reduction this pipeline validates against. -- **Imports:** Import **PyAutoReduce** and the other libraries we need. -- **Paths:** Anchor the cache and output locations to the workspace root. -- **Target Spec:** Declare the reduction as a frozen `TargetSpec`. -- **What MAST Serves:** The `_flc` product, CTE correction and CRDS reference files. -- **The Reduction:** One call runs acquire, align, drizzle, noise, PSF and package. -- **The Drizzle Convention:** Why the mosaic is 0.05"/pixel, north-up, in electrons per second. -- **Provenance Walk:** Reading the diagnostics every reduction reports. -- **The Noise Recipe:** The RMS map and the Casertano correlated-noise factor R. -- **The PSF:** The empirical ePSF shipped with the dataset (full story in `psf.py`). -- **Output Products:** The four FITS files plus `reduction.json` on disk. -- **Plots:** Inspect the data, noise-map and PSF with matplotlib. -- **Load In PyAutoLens:** The dataset loads directly via `al.Imaging.from_fits`. -- **Wrap Up:** Where to go next in the workspace. - -__Why SLACS Is The Quality Bar__ - -The Sloan Lens ACS Survey (SLACS; Bolton et al. 2006, SLACS I; Bolton et al. 2008, SLACS V, -https://arxiv.org/abs/0805.1931) imaged ~100 galaxy-scale strong lenses with ACS/WFC, mostly in -the F814W filter. Its reductions — dithered exposures drizzled to a 0.05"/pixel mosaic (SLACS IX, -Auger et al. 2009) — became the de-facto standard dataset for strong lens modeling: the mass -models, source reconstructions and substructure analyses of the following fifteen years were -calibrated on them. - -**PyAutoReduce** therefore validates its ACS pipeline against SLACS directly: the target reduced -below, SDSS J0008-0004, is one of the pipeline's two acceptance targets, and its products have -been compared pixel-by-pixel against the legacy SLACS modeling dataset (a data-ratio parity of -~0.96, with the residual ~4-6% global flux offset documented and accepted — the legacy -reduction's exact provenance is unrecoverable, and lens-model inferences are invariant under a -global flux scale). The craft reference for ACS reduction more broadly is the COSMOS pipeline -(Koekemoer et al. 2007, https://arxiv.org/abs/astro-ph/0703095). -""" - -""" -__Imports__ - -**PyAutoReduce** exposes exactly two names: `TargetSpec` (the frozen declaration of what to -reduce) and `reduce_target` (the function that reduces it). Everything else in this script is -standard scientific Python. -""" -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. - -- `cache/` holds downloaded exposures and CRDS reference files, reused across runs and scripts. -- `output/` receives the reduced datasets, one folder per target. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # Downloaded exposures + CRDS references (re-used across runs). -OUTPUT_ROOT = WORKSPACE / "output" # Reduced datasets, one folder per target. - -""" -__Target Spec__ - -A **PyAutoReduce** reduction is *declared*, not scripted: you build a frozen `TargetSpec` and the -pipeline is a pure function of that spec plus the archive — re-running it reproduces the dataset -deterministically (modulo upstream calibration reference-file updates, which the provenance -record tracks). - -For this reference reduction every dial stays at its default, so the spec needs only four -entries: the target's name, its coordinates, and the HST proposal that observed it. -""" -SPEC = TargetSpec( - name="slacs0008-0004", # Names the output folder and the cache entry for this target. - ra=2.012333, # Target right ascension in degrees (SDSS J0008-0004). - dec=-0.068944, # Target declination in degrees. - proposal_ids=("10886",), # The SLACS ACS program; filters out exposures from unrelated visits at the same coordinates. -) - -""" -The defaults this spec inherits are the SLACS convention, and every one of them is a dial you can -change (see `dials.py` for the trade study): - -- `instrument="acs_wfc"`, `filter_name="F814W"` — the SLACS instrument/filter combination. -- `final_scale=0.05`, `final_pixfrac=0.8`, `final_kernel="square"` — the drizzle geometry. -- `cutout_shape=(281, 281)` — a 281x281 pixel cutout, ~14" across at 0.05"/pixel. -- `psf_shape=(21, 21)`, `psf_full_shape=(61, 61)` — the compact and extended PSF kernels. -- `cr_method="driz_cr"`, `psf_star_pass="auto"`, `psf_backend="epsf"` — cosmic-ray and PSF policy. - -Because `TargetSpec` is a frozen dataclass, variants are made with `dataclasses.replace(SPEC, -...)` — you will see that idiom throughout this folder. - -__What MAST Serves__ - -The acquire stage queries the Mikulski Archive for Space Telescopes (MAST) via `astroquery.mast` -(https://astroquery.readthedocs.io/en/latest/mast/mast.html) and downloads the `_flc` exposures: -fully calibrated individual exposures — bias, dark, flat and photometric calibration applied by -the `calacs` pipeline — **with the pixel-based CTE correction applied**. - -CTE (charge transfer efficiency) degradation matters enormously for lensing: radiation damage to -the CCD traps charge during readout, systematically trailing and dimming faint sources far from -the readout amplifiers — exactly the regime of faint lensed arcs on low sky backgrounds. The -correction is the empirical charge-trap model of Anderson & Bedin 2010 (PASP 122, 1035, -https://ui.adsabs.harvard.edu/abs/2010PASP..122.1035A), which forward-models the readout and -iteratively inverts it. The `_flc` product has this applied; the older `_flt` product does not. -The ACS Data Handbook (https://hst-docs.stsci.edu/acsdhb) documents the full product zoo — -`step_by_step.py` walks it. - -MAST's on-the-fly reprocessing keeps `_flc` files current with the best calibration reference -files, so **PyAutoReduce** never re-runs `calacs` itself. It does, however, sync the CRDS -reference files the drizzle stage needs (https://hst-crds.stsci.edu) into the cache on every run -— reference files are revised independently of the exposures, so even a cached re-run re-checks -(cheap when nothing is stale). `sync_references=False` is the explicit offline opt-out for -re-runs over a warm cache. - -__The Reduction__ - -One function call runs the whole pipeline: acquire -> align -> drizzle -> noise -> PSF -> -package. The return value is the provenance record — the same dictionary written to -`reduction.json` next to the products. -""" -print( - "\n" - "Starting the reduction of slacs0008-0004.\n" - "\n" - "On a cold cache this downloads the ACS exposures (~0.5 GB) and CRDS reference files from\n" - "MAST, then runs AstroDrizzle — expect ~10-30 minutes on the first run. Re-runs reuse the\n" - "cache under cache/ and finish in a few minutes.\n" -) - -record = reduce_target( - SPEC, - cache_root=CACHE_ROOT, # Exposures + CRDS references live here; warm cache = offline-capable re-runs. - output_root=OUTPUT_ROOT, # Products land in output/slacs0008-0004/. -) - -print("Reduction complete.") - -""" -__The Drizzle Convention__ - -The exposures were combined with AstroDrizzle (DrizzlePac Handbook, -https://hst-docs.stsci.edu/drizzpac; drizzle algorithm: Fruchter & Hook 2002, PASP 114, 144, -https://ui.adsabs.harvard.edu/abs/2002PASP..114..144F) onto a single mosaic with three -conventions chosen for lens modeling: - -- **0.05"/pixel** (`final_scale=0.05`) — the SLACS convention, matching the ACS/WFC native pixel - and every existing SLACS modeling dataset. - -- **North-up** (`final_rot=0`) — a uniform orientation across samples simplifies masks, position - angles and cross-dataset comparison. - -- **Electrons per second** (`final_units='cps'`) — **PyAutoLens** assumes data in e-/s, and the - exposure time recorded in the provenance keeps the Poisson noise term computable. - -The weighting is inverse-variance (`final_wht_type='IVM'`), because the noise-map construction -below needs exactly that weight map. The two genuinely contested dials — `final_pixfrac` (default -0.8, the SLACS value) and `final_kernel` (default `square`) — are first-class configuration, not -buried defaults; `dials.py` is the full trade study. - -__Provenance Walk__ - -Every reduction reports diagnostics so the dial choices are auditable per dataset. They live in -the returned record (== `reduction.json`). Four are worth checking on every reduction: -""" -wht_diag = record["drizzle"]["weight_uniformity"] - -print("\n--- provenance walk ---") -print(f"exposures combined : {record['drizzle']['n_exposures']}") -print(f"weight uniformity : {wht_diag['wht_rms_over_median']:.3f} " - f"(limit {wht_diag['limit']}, acceptable={wht_diag['acceptable']})") -print(f"correlated noise factor R: {record['drizzle']['correlated_noise_factor']:.3f}") -print(f"PSF stars used : {record['psf']['n_stars_used']}") -print(f"PSF star source pass : {record['psf']['star_source_pass']}") - -""" -Reading these: - -- **`weight_uniformity`** — the STScI rule-of-thumb statistic RMS/median of the drizzle weight - map. Values above ~0.2 mean the pixfrac is too small for the dither pattern (coverage - speckle/holes); the pipeline's validation run measured 0.066 at pixfrac 0.8, comfortably - uniform. - -- **`correlated_noise_factor`** — the Casertano et al. 2000 (AJ 120, 2747) scalar R applied to - the noise-map (next section). At the SLACS geometry (pixfrac 0.8, output scale = native scale) - R = 1.364. - -- **`n_stars_used` / `star_source_pass`** — how many field stars built the PSF, and which drizzle - pass they were measured on. `psf.py` explains why that second entry exists. - -__The Noise Recipe__ - -Drizzle does not emit an RMS map, but a lens-model likelihood is only as good as its noise-map — -chi^2 assumes it. **PyAutoReduce** constructs it per the strong-lensing literature (Bayer et al., -https://arxiv.org/abs/1803.05952, section 3.1, derived on exactly this kind of SLACS-style ACS -data): - - sigma_i = R * sqrt( max(N_i, 0) / t_exp + 1 / W_i ) - -- The **background term** `1/W_i` comes from the IVM weight map — read noise, dark current and - sky in one term, per STScI weight-map semantics (DrizzlePac Handbook section 3.4). -- The **Poisson term** uses the source counts/s and the total exposure time, floored at zero so - negative sky fluctuations never produce NaNs. -- **R** is the Casertano et al. 2000 correlated-noise correction: drizzling shares each input - pixel among neighbouring output pixels, so a naive per-pixel RMS underestimates the noise a - chi^2 over the mosaic actually sees. R (1.364 here) inflates the map to compensate, and is - recorded in the provenance. `dials.py` shows how R varies with pixfrac. - -The pipeline fails loudly — it refuses to package — if the noise-map contains NaN or zero-weight -pixels inside the cutout, rather than silently patching them. -""" -print("\nnoise recipe :", record["noise"]["recipe"]) -print("empirical sky RMS (e-/s):", f"{record['noise']['empirical_background_rms']:.5f}") - -""" -__The PSF__ - -The dataset ships with an empirical PSF built from field stars in the mosaic — an "effective PSF" -(ePSF) in the lineage of Anderson & King 2000 (PASP 112, 1360, -https://ui.adsabs.harvard.edu/abs/2000PASP..112.1360A), constructed with `photutils`'s -`EPSFBuilder` (https://photutils.readthedocs.io/en/stable/user_guide/epsf.html). - -The one invariant, whatever construction method is used: **the delivered PSF is the drizzled -PSF** — measured at the same kernel, pixfrac, scale and orientation as the science mosaic, so it -describes the blurring actually present in `data.fits`. Never pair a native-frame PSF with a -drizzled image. - -PSF accuracy matters more in lens modeling than in most photometric applications: PSF mismatch -produces structured residuals at the lensed arcs that bias mass models and can mimic (or mask) -dark-matter substructure. The full PSF story — star selection, the `psf_star_pass` dial, the -STARRED backend, quality diagnostics and honest limitations — lives in the module docstring and -sections of `psf.py` in this folder. Read it before trusting any PSF at high precision. -""" -print("\nPSF method :", record["psf"]["method"]) -print("PSF FWHM (pix):", f"{record['psf']['fwhm_pix']:.2f}") - -""" -__Output Products__ - -The products on disk are the exact input set **PyAutoLens** loads: - -- `data.fits` — the 281x281 drizzled science cutout (~14" at 0.05"/pixel), WCS and units intact - in the header (a deliberate deviation from the legacy SLACS cutouts, whose headers were - stripped). -- `noise_map.fits` — the matching per-pixel RMS map, R applied. -- `psf.fits` — the 21x21 compact PSF kernel for fit convolution. -- `psf_full.fits` — the 61x61 extended PSF capturing the wings (for point-source work). -- `reduction.json` — the complete provenance record you walked above, plus a software-version - envelope. - -All kernels are odd-shaped, centred and unit-normalised — the **PyAutoLens** input standards. -""" -out_dir = OUTPUT_ROOT / SPEC.name - -print("\n--- products ---") -for product in record["package"]["products"]: - print(f" {out_dir / product}") -print(f" {out_dir / 'reduction.json'}") -print("pixel scale :", record["package"]["pixel_scale"]) -print("data units :", record["package"]["data_units"]) - -""" -__Plots__ - -Let's look at what we made. The data is shown with an arcsinh stretch (linear in the noise, -logarithmic on the galaxy), the standard way to see faint lensed arcs next to a bright deflector. -""" -data = fits.getdata(out_dir / "data.fits").astype(float) -noise_map = fits.getdata(out_dir / "noise_map.fits").astype(float) -psf = fits.getdata(out_dir / "psf.fits").astype(float) - -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -sky_rms = record["noise"]["empirical_background_rms"] - -fig, axes = plt.subplots(1, 3, figsize=(15, 5)) - -axes[0].imshow(np.arcsinh(data / (3.0 * sky_rms)), origin="lower", cmap="magma") -axes[0].set_title("data.fits (arcsinh stretch)") - -# The noise-map is displayed capped just above the sky level: the Poisson term brightens it on -# the galaxy, and any masked pixels (noise = 1e8) would otherwise swamp the colour scale. -axes[1].imshow( - np.clip(noise_map, 0.0, 5.0 * sky_rms), origin="lower", cmap="viridis" -) -axes[1].set_title("noise_map.fits (capped)") - -axes[2].imshow(np.arcsinh(psf / psf.max() * 100.0), origin="lower", cmap="magma") -axes[2].set_title("psf.fits (arcsinh stretch)") - -for ax in axes: - ax.set_xticks([]) - ax.set_yticks([]) - -fig.tight_layout() -plot_path = plot_dir / "start_here_products.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"\nPlot saved to: {plot_path}") - -""" -You should see the elliptical deflector galaxy at the centre of the data panel with the faint -lensed arc around it, a noise-map that brightens over the galaxy (the Poisson term at work), and -a sharp, centred PSF. - -__Load In PyAutoLens__ - -The whole point of the output contract: these files load directly into **PyAutoLens** with no -further processing — no unit conversion, no header surgery, no mask file. Bad pixels are already -handled by the masked-by-noise convention (noise = 1e8 where a pixel must not contribute), so any -chi^2 ignores them automatically. - -The import is guarded — **PyAutoReduce** deliberately never depends on **PyAutoLens**, so this -workspace works even without it installed. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "\nPyAutoLens is not installed, so the final loading demonstration is skipped.\n" - "Install it with `pip install autolens` to model this dataset.\n" - "The reduction itself is complete — the products above are on disk." - ) - -if al is not None: - reduction = json.loads((out_dir / "reduction.json").read_text()) - - dataset = al.Imaging.from_fits( - data_path=out_dir / "data.fits", - noise_map_path=out_dir / "noise_map.fits", - psf_path=out_dir / "psf.fits", - pixel_scales=reduction["package"]["pixel_scale"], # 0.05 — read from provenance, never hardcoded. - ) - - print("\nLoaded into PyAutoLens:") - print(f" shape : {dataset.shape_native}") - print(f" pixel scale : {dataset.pixel_scales}") - print( - "\nYou are ready to model this lens — see autolens_workspace/scripts/imaging/start_here.py." - ) - -""" -__Wrap Up__ - -You have reduced a SLACS lens from the MAST archive to a modeling-ready dataset with a single -declared spec: CTE-corrected exposures, the SLACS drizzle convention (0.05"/pixel, north-up, -e-/s), a chi^2-faithful noise-map with the correlated-noise correction applied, an empirical -drizzled PSF, and a provenance record that makes every choice auditable. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/step_by_step.py`: the same reduction, one stage at a time, with the archive - anatomy, CRDS, drizzle parameters and noise construction taught in full. -- `scripts/hst_acs/psf.py`: the complete PSF story — selection cuts, star passes, STARRED, - diagnostics and honest limitations. -- `scripts/hst_acs/dials.py`: the pixfrac / kernel / CR-method trade study, with the literature - disagreements laid out. -- `scripts/hst_acs/individual.py`: per-exposure frame products — model the undrizzled frames and - sidestep correlated noise entirely. -- `scripts/hst_acs/simulator.py`: inject a synthetic lensed arc into the real exposures and - validate the pipeline end to end. -- `scripts/guides/output_contract.py`: the output contract in detail, and how the products map - onto **PyAutoLens** standards. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_acs/step_by_step.py b/workspace/scripts/hst_acs/step_by_step.py deleted file mode 100644 index fa0aae3..0000000 --- a/workspace/scripts/hst_acs/step_by_step.py +++ /dev/null @@ -1,546 +0,0 @@ -""" -HST ACS: Step By Step -===================== - -`start_here.py` reduced the SLACS lens SDSS J0008-0004 with one call. This script re-runs the -same reduction and takes it apart stage by stage: what the archive actually serves, how the -calibration reference files arrive, how the exposure cache makes re-runs cheap and offline, -what AstroDrizzle is told and why, where the noise-map numbers come from, and what evidence -each stage leaves behind in `reduction.json` and the `work/` directory. - -**PyAutoReduce**'s pipeline stages are internal by design — you declare a `TargetSpec` and call -`reduce_target`, and the stage functions are not public API. So this script teaches each stage in -three honest ways: (a) explaining what happens, with links to the STScI documentation and the -papers behind each choice; (b) reading the evidence out of the provenance record and work -directory after the run; and (c) demonstrating the public helper functions (`query_exposures`, -`casertano_r`, `noise_map_from`, `weight_uniformity`, `registered_ratios`, ...) standalone on the -run's own products, so you can see the machinery with your own hands. - -Expect the same runtime as `start_here.py` on a cold cache (~10-30 minutes); if you ran that -script first, the cache is warm and this one takes a few minutes. - -__Contents__ - -- **Imports:** Import **PyAutoReduce**, its public helpers and the other libraries we need. -- **Paths:** Anchor the cache and output locations to the workspace root. -- **The Instrument Adapter:** Where ACS/WFC's identity lives — scales, products, defaults. -- **MAST Query Anatomy:** The `astroquery.mast` idiom, the product zoo, and HAP exclusion. -- **CRDS Reference Files:** Best-reference syncing and the offline dial. -- **The Exposure Cache:** Transient full-frame storage, the manifest, and eviction. -- **Run The Pipeline:** The reduction itself, so the later sections have evidence to read. -- **Footprint And Quality Filtering:** Which exposures were kept, skipped and rejected. -- **WCS And Alignment:** The a-priori Gaia-tied WCS, and the honest TweakReg story. -- **The Drizzle Stage:** Every AstroDrizzle keyword the pipeline set, one by one. -- **Sky Subtraction:** What `skymethod='globalmin+match'` does and why it suits lens fields. -- **Cosmic-Ray Flagging:** The median-combine flow, and the flux-loss caveat. -- **Noise Construction By Hand:** `casertano_r` + `noise_map_from` standalone on the mosaic. -- **The PSF Stage:** A summary, deferring to `psf.py` for the full story. -- **Packaging And The Strict Cutout:** Cutout2D strict mode, bad-pixel policy, uniformity. -- **Parity:** `registered_ratios` against a reference reduction, when one exists. -- **Wrap Up:** Where to go next in the workspace. - -__Imports__ - -Alongside the two core names, we import the public helpers this script demonstrates standalone: -the instrument adapter registry, the MAST query helper, the exposure cache, the noise -constructors and the drizzle diagnostic. -""" -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target, instruments -from autoreduce.acquire.cache import ExposureCache -from autoreduce.acquire.mast import query_exposures -from autoreduce.drizzle.diagnostics import weight_uniformity, WEIGHT_UNIFORMITY_LIMIT -from autoreduce.noise.rms import casertano_r, noise_map_from, empirical_background_rms - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # Downloaded exposures + CRDS references (re-used across runs). -OUTPUT_ROOT = WORKSPACE / "output" # Reduced datasets, one folder per target. - -SPEC = TargetSpec( - name="slacs0008-0004", # The SLACS parity anchor, same as start_here.py — the cache is shared. - ra=2.012333, # Right ascension in degrees. - dec=-0.068944, # Declination in degrees. - proposal_ids=("10886",), # The SLACS ACS program (Bolton et al. 2008). -) - -""" -__The Instrument Adapter__ - -Everything instrument-specific in **PyAutoReduce** lives in one place: the instrument adapter. -The pipeline stages are generic; the adapter tells them what ACS/WFC is — its native pixel scale, -which calibrated product to download, which CRDS environment variable the drizzle needs, and the -default AstroDrizzle keywords. - -The registry is public, so we can inspect the ACS/WFC adapter directly: -""" -adapter = instruments.get("acs_wfc") - -print("--- the acs_wfc adapter ---") -print(f"MAST instrument name : {adapter.mast_instrument_name}") -print(f"native pixel scale : {adapter.native_scale} arcsec/pix") -print(f"calibrated product : _{adapter.calibrated_suffix.lower()}.fits") -print(f"saturation level : {adapter.saturation_dn:.0f} e-") -print(f"default drizzle kwargs: {adapter.default_drizzle_kwargs}") -print(f"all registered instruments: {instruments.registered_keys()}") - -""" -Note `calibrated_suffix="FLC"` — the adapter, not the user, decides which archive product a -reduction is built from. For ACS that is the CTE-corrected exposure, for reasons the next section -unpacks. - -__MAST Query Anatomy__ - -The acquire stage's query is the standard `astroquery.mast` idiom -(https://astroquery.readthedocs.io/en/latest/mast/mast.html): `Observations.query_criteria` by -coordinates, observation collection, instrument and filter, then `get_product_list` + -`filter_products` + `download_products` for the calibrated exposures. **PyAutoReduce** wraps this -in the public helper `query_exposures`, which adds two pieces of query hygiene you would -otherwise learn the hard way. - -**The product zoo.** For any HST pointing, MAST serves a family of products per exposure -(ACS Data Handbook, https://hst-docs.stsci.edu/acsdhb): - -- `_raw` — the untouched detector readout. -- `_flt` — calibrated by `calacs` (bias, dark, flat, DQ arrays, ERR arrays) but **without** the - pixel-based CTE correction. -- `_flc` — the same, **with** the CTE correction of Anderson & Bedin 2010 (PASP 122, 1035, - https://ui.adsabs.harvard.edu/abs/2010PASP..122.1035A) applied. This is what the ACS adapter - downloads: CTE trailing systematically dims faint arcs on low backgrounds, exactly what a lens - reduction cannot afford. (One caveat the handbook is honest about: the pixel-based correction - amplifies read noise in low-S/N pixels.) -- `_drz` / `_drc` — MAST's own drizzled combinations. **PyAutoReduce** never uses these: the - drizzle geometry (scale, pixfrac, orientation) is precisely what a lensing reduction must - control, so combination happens locally from the `_flc` frames. - -**HAP exclusion.** A plain coordinate query also matches Hubble Advanced Products -(https://outerspace.stsci.edu/spaces/HAdP/pages/54558799/Improvements+in+HST+Astrometry) — -skycell mosaics and visit-level associations whose obs-ids start with `hst_`. Their member lists -re-reference the same exposures many times over (for this very target, 31 matched products dedupe -to 7 actual files), and the visit-level `hst_*_flc.fits` files are renamed *copies* of the member -exposures MAST already serves directly. Ingesting both drizzles every exposure twice — doubled -IVM weights then suppress the computed noise by sqrt(2), a real bug the pipeline's validation -caught. `query_exposures` therefore keeps only direct calibration-level observations and filters -the product lists the same way. - -The query below is the exact one the pipeline runs internally (network required): -""" -observations = query_exposures( - ra=SPEC.ra, - dec=SPEC.dec, - adapter=adapter, # Supplies obs_collection + instrument_name for the MAST query. - filter_name=SPEC.filter_name, # F814W — the SLACS filter. - radius="0.5 arcmin", # The default search radius around the target. - proposal_ids=SPEC.proposal_ids, # Keeps the query to the SLACS program only. -) - -print("\n--- MAST query (direct observations only, HAP excluded) ---") -for row in observations: - print(f" obs_id={row['obs_id']} proposal={row['proposal_id']}") - -""" -__CRDS Reference Files__ - -Reference files are acquisition too. AstroDrizzle's IVM weighting resolves ACS calibration files -(darks, flats, bad-pixel tables) through the `jref$` environment prefix, served by the -Calibration Reference Data System (CRDS, https://hst-crds.stsci.edu). The acquire stage runs the -CRDS best-references sync (`crds.bestrefs --sync-references=1 --update-bestrefs`) for the -downloaded exposures and exports the environment the drizzle needs. - -Two behaviours worth knowing: - -- **The sync runs on every reduction, even over a warm exposure cache** — CRDS revises reference - files independently of the exposures, so a cached re-run re-checks. When nothing is stale this - is cheap. - -- **`sync_references=False` is the explicit offline opt-out**: valid only over a - previously-synced cache (it raises a `RuntimeError` on a cold one, rather than producing a - drizzle with missing reference files). Use it for repeated offline experimentation once one - networked run has warmed the cache: - - offline_spec = dataclasses.replace(SPEC, sync_references=False) - -Reference files are shared across targets and are the one cache component never evicted. - -__The Exposure Cache__ - -Full-frame exposures are transient by design: download per target, reduce, package, evict. -SLACS-like targets are single pointings (~0.5 GB); a whole-sample run streams one target at a -time so peak disk usage never grows with sample size. The machinery is the public -`ExposureCache`: - -- `cache_root//` holds the exposures; `cache_root/crds/` the shared references. -- `cache_root/cache_manifest.json` records what was downloaded from where, so eviction never - costs reproducibility — a re-run re-fetches deterministically. -- `reduce_target(..., size_cap_bytes=...)` enforces a size cap by evicting oldest completed - targets; `evict_when_done=True` drops a target's exposures as soon as its products are written. -- A warm manifest is **the** offline mechanism: `exposures_for()` short-circuits MAST entirely - when the files are already on disk. - -__Run The Pipeline__ - -Now run the reduction, so the remaining sections have real evidence to read. Everything below -this call is *reading* — the run itself is identical to `start_here.py`. -""" -print( - "\n" - "Running the full reduction now (acquire -> align -> drizzle -> noise -> psf -> package).\n" - "Cold cache: ~10-30 minutes (downloads ~0.5 GB). Warm cache: a few minutes.\n" -) - -record = reduce_target(SPEC, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / SPEC.name -work_dir = out_dir / "work" - -print("Reduction complete.") - -cache = ExposureCache(root=CACHE_ROOT) -manifest = cache.read_manifest() -entry = manifest["targets"][SPEC.name] -print("\n--- cache manifest entry ---") -print(f"files downloaded : {len(entry['files'])}") -print(f"source : {entry['source']}") -print(f"downloaded at : {entry['downloaded_at']}") -print(f"evicted : {entry['evicted']}") - -""" -__Footprint And Quality Filtering__ - -Between download and drizzle, two screens run — and both leave evidence in the `acquire` block: - -- **Usability screen.** MAST serves failed exposures (EXPFLAG "EXCESSIVE DOWNTIME", zero - exposure time) alongside the good ones. They carry no science content and are dropped before - combination — this target's archive set includes exactly such a failed exposure, which an - early version of the pipeline combined before the screen existed. - -- **Detector-footprint filter.** Only exposures whose detector footprint actually covers the - target (with a margin of the cutout half-extent plus 15") enter combination. Survey visits can - span many pointings that never touch the target; combining them wastes memory and time. -""" -acq = record["acquire"] - -print("\n--- acquire block ---") -print(f"exposures kept : {acq['n_exposures']}") -print(f"skipped off-target : {acq['n_skipped_off_target']}") -print(f"rejected as unusable : {acq['n_skipped_unusable']} {acq['unusable_exposures']}") -print(f"references synced : {acq['references_synced']}") -print(f"exposure files : {acq['exposures']}") - -""" -__WCS And Alignment__ - -Frame-to-frame registration is a prerequisite for both cosmic-ray rejection and PSF fidelity — -misaligned frames produce elongated stars and CR-flagged galaxy cores. The DrizzlePac Handbook's -alignment chapter (https://hst-docs.stsci.edu/drizzpac) covers the classical fix, TweakReg, which -catalog-matches sources across frames and updates each WCS to few-milliarcsecond registration. - -**PyAutoReduce**'s default is to *not* run TweakReg — and to be honest about what is and is not -wired: - -- **Default: trust the a-priori WCS.** MAST now attaches Gaia-tied WCS solutions to HST - exposures (the HAP astrometry programme — roughly 70% of ACS/WFC3 frames align to Gaia at - ~10 mas). Within a single visit, relative alignment is normally already adequate for - drizzling. The `align` block records which WCS solution each exposure carries, so you can see - what you trusted. - -- **The design intent** is TweakReg as a *deviation trigger*: run it only when a - cross-correlation diagnostic shows residual misregistration above ~0.1 pixel. **In the current - release that trigger is not wired** — the pipeline always records `tweakreg_run: False`, and - the `alignment_tolerance_pix` spec field is read by nothing. If your field shows elongated - stars in the mosaic, that is the symptom to look for; the per-frame registration residuals in - `individual.py` are the quantitative check. -""" -print("\n--- align block ---") -print(f"tweakreg_run : {record['align']['tweakreg_run']}") -for exposure, wcsname in record["align"]["wcs_solutions"].items(): - print(f" {exposure}: {wcsname}") - -""" -__The Drizzle Stage__ - -The combine stage hands AstroDrizzle -(https://drizzlepac.readthedocs.io; algorithm: Fruchter & Hook 2002, PASP 114, 144, -https://ui.adsabs.harvard.edu/abs/2002PASP..114..144F) a fully explicit keyword set, recorded -verbatim in the provenance. Let's walk the keywords that matter, in the order they act -(DrizzlePac Handbook, https://hst-docs.stsci.edu/drizzpac): - -- `skymethod='globalmin+match'` — sky subtraction before combination (next section). -- `median=True`, `blot=True`, `driz_cr=True` — the cosmic-ray flagging flow (section after). -- `final_scale=0.05` — output pixel scale in arcsec. The SLACS convention; equal to the ACS/WFC - native scale, so the scale ratio s = 1. -- `final_pixfrac=0.8` — each input pixel is shrunk to 80% of its size before being "dripped" - onto the output grid. The drizzle resolution/noise trade-off dial; `dials.py` is the full - study. -- `final_kernel='square'` — the drop shape. STScI's default; SLACS IX used `gaussian`, one of - the literature disagreements `dials.py` covers. -- `final_wht_type='IVM'` — inverse-variance weight maps, which the noise stage requires. -- `final_units='cps'` — electrons per second, the **PyAutoLens** flux convention. -- `final_rot=0.0` — north-up output. -- `preserve=False, build=False, clean=True` — housekeeping: no intermediate-file archaeology, - separate `_sci`/`_wht` outputs, scratch files removed. - -One operational quirk explains a workspace rule: drizzlepac lowercases output filenames -internally, so the combine runs inside a scratch directory (`work/`) — which is why **every path -handed to PyAutoReduce must be absolute**. -""" -print("\n--- drizzle kwargs (verbatim from provenance) ---") -for key, value in record["drizzle"]["drizzle_kwargs"].items(): - print(f" {key} = {value}") - -""" -__Sky Subtraction__ - -`skymethod='globalmin+match'` computes one global sky minimum across the exposure set and then -*matches* frame-to-frame offsets, rather than measuring the sky independently per frame -(`localmin`). This matters for lens fields: AstroDrizzle's per-frame sky estimate can be biased -by a large galaxy filling a small field — for a deflector at the field centre, an independent -per-frame sky would subtract galaxy light as "sky". The matched method estimates one consistent -pedestal and preserves relative photometry across the set. The DrizzlePac Handbook documents the -four `skymethod` options. - -The subtraction is *virtual*: AstroDrizzle records each frame's sky in the `MDRIZSKY` header -keyword and subtracts during the drizzle, leaving the `_flc` files untouched. `individual.py` -shows where that pedestal must be handled explicitly (per-frame products). - -__Cosmic-Ray Flagging__ - -With no shutter closed between reads, every HST exposure is peppered with cosmic rays. The -default flow is AstroDrizzle's median-combine flagging (DrizzlePac Handbook): drizzle each -exposure separately onto the common grid, median-combine those single drizzles into a clean -reference, "blot" the median back to each frame's geometry, and flag pixels that deviate beyond -`driz_cr_snr` — writing DQ bit 4096. It needs >= 2 overlapping exposures; on a single exposure -the pipeline automatically switches to its single-exposure branch (no median, no flagging — -recorded as `single_exposure_branch` in the provenance). - -The honest caveat: on steep gradients (deflector cores, PSF star cores) the blotted median reads -systematically low when sub-pixel dither shifts smear the peak, so `driz_cr` can flag genuine -core flux as cosmic rays — measured at ~37% deflector-core flux loss on SLACS-like data at -pipeline thresholds. The `cr_method="deepcr"` dial (Zhang & Bloom 2020, ApJ 889, 24, -https://ui.adsabs.harvard.edu/abs/2020ApJ...889...24Z — a CNN trained largely on exactly this -ACS/F814W regime) is the per-frame alternative; `dials.py` tells the full story, including why -the default has not (yet) been flipped. -""" -print("\n--- drizzle CR evidence ---") -print(f"cr_method : {record['drizzle']['cr_method']}") -print(f"single_exposure_branch : {record['drizzle']['single_exposure_branch']}") - -""" -__Noise Construction By Hand__ - -The noise stage is the most lensing-specific step, and its two ingredients — the science mosaic -and the IVM weight map — persist in `work/` after the run. So rather than just describing it, we -rebuild the noise-map ourselves with the same public functions the pipeline uses, and check we -get the shipped product. - -The recipe (Bayer et al., https://arxiv.org/abs/1803.05952, section 3.1): - - sigma_i = R * sqrt( max(N_i, 0) / t_exp + 1 / W_i ) - -- `1/W_i`: the IVM weight map is an inverse *background* variance per STScI semantics - (DrizzlePac Handbook section 3.4) — read noise, dark and sky in one term. -- `max(N_i, 0)/t_exp`: the source Poisson term from the cps image and total exposure time, - floored at zero. -- `R`: the Casertano et al. 2000 (AJ 120, 2747) correlated-noise factor. Drizzling shares each - input pixel among neighbouring output pixels, so adjacent output pixels carry correlated - noise; the per-pixel RMS then underestimates the noise a chi^2 (which assumes independent - pixels) actually experiences. R is the scalar correction, a function of pixfrac p and scale - ratio s only. -""" -mosaic_stem = f"{SPEC.name}_{SPEC.filter_name}".lower() -sci_path = sorted(work_dir.glob(f"{mosaic_stem}*_sci.fits"))[0] -wht_path = sorted(work_dir.glob(f"{mosaic_stem}*_wht.fits"))[0] - -sci = fits.getdata(sci_path).astype(float) -wht = fits.getdata(wht_path).astype(float) -header = fits.getheader(sci_path) -exptime = float(header.get("EXPTIME", header.get("TEXPTIME"))) - -R = casertano_r( - pixfrac=SPEC.final_pixfrac, # 0.8 — the drizzle drop size. - scale_ratio=adapter.scale_ratio(SPEC.final_scale), # 1.0 — output scale / native scale. -) - -noise_by_hand = noise_map_from( - sci=sci, # The cps science mosaic. - wht=wht, # The IVM weight map. - exptime=exptime, # Total exposure time, for the Poisson term. - correlated_noise_factor=R, # The Casertano correction. -) - -print("\n--- noise construction by hand ---") -print(f"mosaic : {sci_path.name}") -print(f"total exposure : {exptime:.0f} s") -print(f"R (p={SPEC.final_pixfrac}, s=1) : {R:.3f}") -print(f"pipeline recorded R : {record['drizzle']['correlated_noise_factor']:.3f}") - -""" -To compare against the shipped `noise_map.fits` we cut the same 281x281 stamp out of our -hand-built full-mosaic map — using the same strict `Cutout2D` the packaging stage uses (more on -strict mode below). -""" -from astropy.nddata import Cutout2D -from astropy.wcs import WCS - -cutout = Cutout2D( - data=noise_by_hand, - position=WCS(header).world_to_pixel_values(SPEC.ra, SPEC.dec), - size=SPEC.cutout_shape, - mode="strict", # Raises if the stamp leaves the mosaic — never pads silently. -) - -shipped_noise = fits.getdata(out_dir / "noise_map.fits").astype(float) - -# Compare away from masked-by-noise pixels (1e8) and any isolated bad pixels our raw map -# carries as NaN before the packaging policy handles them. -comparable = (shipped_noise < 1.0e7) & np.isfinite(cutout.data) -ratio = np.median(cutout.data[comparable] / shipped_noise[comparable]) -print(f"hand-built / shipped noise (median): {ratio:.4f}") - -sky_rms = empirical_background_rms(sci) -print(f"empirical blank-sky RMS of the mosaic : {sky_rms:.5f} e-/s") -print(f"pipeline recorded : {record['noise']['empirical_background_rms']:.5f} e-/s") - -""" -The ratio should be 1.0000 to numerical precision — the shipped map *is* this construction. The -empirical blank-sky RMS is the validation closure the pipeline records with every run: the -sigma-clipped RMS of the mosaic background, which the noise-map's background term must track. - -__The PSF Stage__ - -After the noise, the pipeline builds the PSF: field stars are selected on the mosaic -(unsaturated, uncrowded, away from edges and the target itself), and an effective PSF is built -from them following Anderson & King 2000 (PASP 112, 1360) via `photutils.EPSFBuilder`. The -delivered kernel honours the drizzled-PSF invariant — it is measured on the same -kernel/pixfrac/scale/orientation as the science mosaic. -""" -print("\n--- psf block (summary) ---") -print(f"method : {record['psf']['method']}") -print(f"n_stars_used : {record['psf']['n_stars_used']}") -print(f"fwhm_pix : {record['psf']['fwhm_pix']:.2f}") -print(f"star_source_pass : {record['psf']['star_source_pass']}") - -""" -The PSF deserves — and has — its own script: `psf.py` in this folder covers the selection cuts, -the `psf_star_pass` dial (what `star_source_pass` records and why), the STARRED backend, the -tier-2 honesty story and the quality diagnostics. We defer to it here. - -__Packaging And The Strict Cutout__ - -Packaging cuts the 281x281 stamp with astropy's `Cutout2D` in `mode="strict"` — if the requested -stamp is not fully inside the mosaic, the reduction *raises* rather than shipping a padded or -truncated dataset. Size the cutout to the coverage, never the other way round. - -Two more guards run at packaging time, both leaving provenance: - -- **The bad-pixel policy**: isolated non-finite noise pixels (fully-rejected or dead pixels — - routine in deep resampled stacks) are masked-by-noise (noise = 1e8, data zeroed) and counted. - The failure stays loud where it matters: a *structured* defect (contiguous bad pixels), more - than 0.5% of the cutout bad, or any bad pixel within 1.5" of the target centre is an error, - not a masking opportunity — the lens region must reduce cleanly. - -- **Weight uniformity over the cutout**: the same RMS/median diagnostic, re-measured on the - shipped stamp specifically. -""" -print("\n--- package block ---") -print(f"products : {record['package']['products']}") -print(f"cutout shape : {record['package']['cutout_shape']}") -print(f"pixel scale : {record['package']['pixel_scale']}") -print(f"data units : {record['package']['data_units']}") - -print("\n--- bad pixel policy ---") -print(json.dumps(record["bad_pixel_policy"], indent=2)) - -wht_cut = Cutout2D( - data=wht, - position=WCS(header).world_to_pixel_values(SPEC.ra, SPEC.dec), - size=SPEC.cutout_shape, - mode="strict", -) -print("\n--- weight uniformity, demonstrated standalone ---") -print(f"full mosaic : {weight_uniformity(wht):.3f}") -print(f"cutout : {weight_uniformity(wht_cut.data):.3f} (limit {WEIGHT_UNIFORMITY_LIMIT})") -print(f"pipeline recorded (cutout): " - f"{record['drizzle']['weight_uniformity_cutout']['wht_rms_over_median']:.3f}") - -""" -Finally, `reduction.json` on disk is the returned record plus an envelope — `written_at` and the -software versions of every package in the chain — so a dataset is auditable years later even if -this workspace is long gone. -""" -reduction = json.loads((out_dir / "reduction.json").read_text()) -print("\n--- reduction.json top-level blocks ---") -print(sorted(reduction.keys())) - -""" -__Parity__ - -The pipeline's acceptance method is parity: register a new reduction onto a reference -sub-pixel-accurately and compare bright-pixel data and noise ratios. The public helper -`registered_ratios` implements it — the same statistics behind the SLACS validation numbers -(data ratio ~0.96 against the legacy SLACS dataset; noise ~30% above the legacy maps *by -design*, because the legacy maps do not carry the correlated-noise correction R and ours do). - -Parity needs a reference. If you have run `dials.py` (which writes a second reduction of this -target at different drizzle dials), we compare against it; otherwise this section skips — -honestly, rather than inventing a reference. -""" -reference_dir = OUTPUT_ROOT / "dials" / "pixfrac_0p6" / SPEC.name - -if reference_dir.exists(): - from autoreduce.validation import registered_ratios - - ref_data = fits.getdata(reference_dir / "data.fits").astype(float) - ref_noise = fits.getdata(reference_dir / "noise_map.fits").astype(float) - new_data = fits.getdata(out_dir / "data.fits").astype(float) - new_noise = fits.getdata(out_dir / "noise_map.fits").astype(float) - - parity = registered_ratios(new_data, new_noise, ref_data, ref_noise) - print("\n--- parity vs the dials.py pixfrac-0.6 reduction ---") - print(json.dumps(parity, indent=2)) - print( - "\nExpect data_ratio_median ~= 1 (same photons, different resampling) and the noise\n" - "ratio tracking the ratio of the two Casertano factors." - ) -else: - print( - "\n[parity] no reference reduction found at " - f"{reference_dir}\n" - "[parity] run scripts/hst_acs/dials.py first to create one — skipping the parity leg." - ) - -""" -__Wrap Up__ - -You have now seen every stage of the default ACS pipeline with its evidence: the archive anatomy -and query hygiene, CRDS reference syncing and the offline dial, the transient cache, the explicit -AstroDrizzle keyword set, sky matching, cosmic-ray flagging and its caveat, the noise-map rebuilt -by hand from the run's own mosaic, the PSF summary, and the strict packaging guards. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/dials.py`: the trade study for the dials this script kept at their defaults — - pixfrac, kernel, CR method. -- `scripts/hst_acs/psf.py`: the full PSF story this script deferred. -- `scripts/hst_acs/individual.py`: per-exposure frame products — the un-drizzled alternative. -- `scripts/hst_acs/simulator.py`: end-to-end validation by injecting a synthetic arc into the - real exposures. -- `scripts/guides/noise_maps.py`: the noise recipes across all instruments, and why chi^2 cares. -- `scripts/guides/target_spec.py`: every `TargetSpec` dial in one place. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_wfc3_ir/README.md b/workspace/scripts/hst_wfc3_ir/README.md deleted file mode 100644 index 5b8ddfe..0000000 --- a/workspace/scripts/hst_wfc3_ir/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# hst_wfc3_ir - -HST WFC3/IR reductions — the infrared HgCdTe channel, which rewrites the CCD rules: no -shutter, MULTIACCUM up-the-ramp readout with per-exposure cosmic-ray rejection, `_flt` -products already in e-/s, no CTE correction at all, and 0.128"/pixel undersampling that -makes the drizzle scale the central dial (the adapter recommends 0.065"/pixel). Shared HST -machinery is documented in `scripts/hst_acs/`; the CCD sibling deltas are in -`scripts/hst_wfc3_uvis/`. - -The validation anchor is the WFC3/IR F160W snapshot of SDSS J0252+0039 — including the -recorded zero-weight-speckle finding: pixfrac 0.8 at 0.065"/pixel on few-dither data left -zero-weight pixels and the finite-noise guard refused to ship, establishing the IR rule -"pixfrac -> 1.0 or a coarser scale". - -Read in order: - -1. `start_here.py` — the IR anchor reduction end to end: detector physics, the 0.065" - drizzle choice, the fine-grid Casertano R, and the speckle finding. -2. `step_by_step.py` — the `calwf3` IR calibration chain and every stage's delta, with the - WHT-uniformity reading on few-dither data. - -Both scripts need network access (MAST + CRDS) and the `[hst]` extra (drizzlepac). diff --git a/workspace/scripts/hst_wfc3_ir/__init__.py b/workspace/scripts/hst_wfc3_ir/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/workspace/scripts/hst_wfc3_ir/start_here.py b/workspace/scripts/hst_wfc3_ir/start_here.py deleted file mode 100644 index b3d5ade..0000000 --- a/workspace/scripts/hst_wfc3_ir/start_here.py +++ /dev/null @@ -1,339 +0,0 @@ -""" -Start Here: HST WFC3/IR -======================= - -This script reduces Hubble Space Telescope WFC3/IR imaging of the strong lens SDSS J0252+0039 -into a modeling-ready dataset with **PyAutoReduce** — and in doing so introduces a channel -that is *fundamentally different* from every HST CCD you have reduced so far. - -WFC3/UVIS and ACS/WFC differ in plate scales and reference files; WFC3/IR differs in physics. -It is an infrared HgCdTe detector, not a CCD: it has no shutter, it reads out non- -destructively while photons accumulate, its cosmic rays are rejected *inside each exposure* -by the calibration pipeline, and no charge-transfer correction exists or is needed. Almost -every CCD instinct — `_flc` files, CR rejection needing multiple exposures, electrons as the -native unit — changes here. - -The reduction machinery downstream (AstroDrizzle, the noise recipe, PSF tiers, packaging) is -still the shared HST path, so as in `hst_wfc3_uvis/`, this folder teaches the *deltas* and -defers shared depth to `hst_acs/step_by_step.py`. This script also carries one of the most -instructive findings in the whole workspace: a reduction that **PyAutoReduce** *refused to -ship*, and why that refusal was the pipeline working exactly as designed. - -__Contents__ - -- **A Different Detector:** HgCdTe, no shutter, and up-the-ramp readout. -- **IR Gotchas:** Persistence, blobs and 0.13" undersampling. -- **The Anchor:** The J0252+0039 F160W snapshot — an internal validation target. -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root, absolutely. -- **The Drizzle Choice:** Half-native 0.065"/pixel and the fine-grid Casertano branch. -- **The Zero-Weight-Speckle Finding:** The reduction the finite-noise guard refused to ship. -- **Target Spec:** Declare the IR reduction, including the cutout-size reasoning. -- **The Reduction:** Run the full pipeline with one function call. -- **Provenance:** Walk the diagnostics that matter most on few-dither IR data. -- **Plots:** Visualize the data, noise map and PSF. -- **Load the Dataset in PyAutoLens:** Load the products as an `al.Imaging` dataset. -- **Wrap Up:** Summary of the script and next steps. - -__A Different Detector__ - -The WFC3/IR channel is a HgCdTe photodiode array. Three consequences cascade from that (the -WFC3 Data Handbook, https://hst-docs.stsci.edu/wfc3dhb, is the authoritative reference): - -- **No shutter, MULTIACCUM readout.** The detector is read *non-destructively* many times - while charge accumulates — "up the ramp". Each downloaded exposure is really a fitted - slope: signal per second, from a whole sequence of reads. -- **Cosmic rays are rejected per exposure.** A cosmic-ray hit appears as a discontinuity in - a pixel's ramp, so the `calwf3` ramp-fitting step (CRCORR) identifies and excludes it - *within a single exposure*. IR exposures arrive largely CR-cleaned — unlike CCDs, where CR - rejection needs multiple overlapping exposures at the drizzle stage. (AstroDrizzle's - `driz_cr` still runs on IR stacks, for the residue the ramp fit misses — see - `hst_wfc3_ir/step_by_step.py`.) -- **No CTE correction, no `_flc`.** There is no charge transfer across the detector, so - nothing to correct: the calibrated product is the plain **`_flt`**, and — because the ramp - fit measures a rate — it is already in **electrons per second**. (CCD `_flc` frames are in - electrons; the drizzle stage converts. On IR, the units the lens model wants are there from - the start.) - -The effective saturation full well is ~78 ke-, which feeds the PSF star selection's peak cut -just as on the CCD channels. - -__IR Gotchas__ - -Three IR-specific realities to keep in mind when judging any WFC3/IR dataset: - -- **Persistence:** a bright source observed earlier — even in a previous program — leaves a - slowly-decaying afterglow in the pixels it saturated. A ghost arc in your field may be - someone else's star from two orbits ago. -- **Blobs:** small regions of reduced sensitivity on the channel-select mechanism appear as - soft dark spots; they are flagged in the DQ arrays and drilled out by the drizzle weights, - which means *coverage holes* if the dither pattern never moves the target off them. -- **Undersampling:** the native pixels are 0.128" — badly undersampling HST's infrared PSF. - The remedy is dithering plus drizzle: sub-pixel offsets between exposures let a finer - output grid recover resolution (Fruchter & Hook 2002, PASP 114, 144; - https://ui.adsabs.harvard.edu/abs/2002PASP..114..144F). This is why IR programs dither and - why the drizzle scale choice below is the central dial of any IR reduction. - -__The Anchor__ - -The dataset is the WFC3/IR F160W snapshot of SDSS J0252+0039 — the same lens whose UVIS -F390W reduction is the published Bayer et al. (https://arxiv.org/abs/1803.05952) anchor of -`hst_wfc3_uvis/start_here.py`. Honesty first: unlike the UVIS leg, there is no published IR -reduction of this field to close against, so the IR channel's validation is *internal* — -units, noise closure, weight uniformity and the guard behavior you will meet below. A -snapshot program also means *few dithers*, which is exactly what makes this dataset such an -instructive IR case. -""" - -""" -__Imports__ - -Alongside the two **PyAutoReduce** entry points we import `instruments` — the adapter -registry — to read the IR channel's recommended output scale rather than hard-coding it, and -the public `casertano_r` helper to do the correlated-noise arithmetic by hand. -""" -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np - -from autoreduce import TargetSpec, reduce_target -from autoreduce import instruments -from autoreduce.noise.rms import casertano_r - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -RA, DEC = 43.188375, 0.666222 # SDSS J0252+0039: 02h52m45.21s +00d39m58.4s - -""" -__The Drizzle Choice__ - -Native 0.128" pixels undersample the PSF, so dithered IR programs conventionally drizzle to a -finer grid — deep-field practice sits in the 0.06-0.08" range. The IR adapter recommends -**0.065"/pixel** (half-native), and we read that recommendation off the adapter itself rather -than hard-coding it: -""" -IR_SCALE = instruments.get("wfc3_ir").recommended_final_scale - -print(f"wfc3_ir native scale : {instruments.get('wfc3_ir').native_scale}\"/pix") -print(f"wfc3_ir recommended scale: {IR_SCALE}\"/pix") - -""" -Finer output pixels are not free. The Casertano et al. (2000, AJ 120, 2747) correlated-noise -factor R depends on the pixfrac p and the scale ratio s = output scale / native scale — and -when the output grid is finer than the drizzle drop (s < p), the *fine-grid branch* of the -formula applies and R grows quickly: each shrunken input drop is being spread over several -output pixels, correlating all of them. Compare the two regimes with the public helper — -UVIS at native scale versus IR at half-native: -""" -r_uvis_native = casertano_r(pixfrac=1.0, scale_ratio=1.0) # UVIS start_here dials. -r_ir_fine = casertano_r(pixfrac=1.0, scale_ratio=IR_SCALE / 0.128) # IR: s ~ 0.51 < p -> fine-grid branch. - -print(f"R at native scale, pixfrac 1.0 : {r_uvis_native:.3f}") -print(f"R at 0.065\" from 0.128\", pixfrac 1.0: {r_ir_fine:.3f}") - -""" -R is materially larger on the IR grid — every pixel's noise is inflated by that factor in the -delivered noise map, and the value is recorded per run in `reduction.json`. That is the price -of resolution, paid transparently. If your science prefers uncorrelated noise over sampling, -`final_scale=0.128` at pixfrac 1.0 gets you back to shift-and-add (R = 1.5); the dial is -yours. - -__The Zero-Weight-Speckle Finding__ - -Now the finding this dataset is famous for (in this workspace, anyway). The **PyAutoReduce** -phase-1 default pixfrac is 0.8 — the SLACS-convention value that works beautifully on -well-dithered ACS programs. Reducing *this* snapshot at 0.065"/pixel with pixfrac 0.8 left -230 pixels inside the cutout with **zero drizzle weight**: with only a few dithers, shrunken -drops on a fine grid leave gaps between them that no exposure ever covers — coverage -speckle. Zero weight means infinite noise, and the finite-noise packaging guard *refused to -ship the dataset*. - -That refusal is the design working, not failing. A zero-weight pixel has no data; a pipeline -that quietly interpolated over it would hand your lens model pixels that look like data and -are not. The loud failure forces the dial decision back to you, with the diagnostics to make -it: either widen the drops (pixfrac 1.0 closes coverage, at the cost of the larger R above) -or coarsen the grid. The rule of thumb this finding established: - - On WFC3/IR, few-dither data at sub-native output scales needs pixfrac -> 1.0 - (or a coarser final_scale). - -The spec below therefore sets `final_pixfrac=1.0`. If you want to see the guard fire — a -genuinely educational crash — rerun with pixfrac 0.8; the reduction refuses to package, -reporting the count of non-finite noise pixels and telling you to fix the coverage rather -than patch the noise map. The WHT-uniformity diagnostic tells the same story numerically -(`hst_wfc3_ir/step_by_step.py` reads it in detail). - -__Target Spec__ - -One more IR-specific choice needs explaining: the cutout shape. The default 281x281 cutout -spans 14" at the ACS 0.05" scale, but at 0.065"/pixel it would span 18.3" of sky — and on -this field that wider footprint reaches a zero-coverage detector-defect blob 8.5" from the -lens, which the packaging guard (rightly) also refused. Matching the ACS sky footprint -instead gives 14" / 0.065 -> **215 pixels**. The lesson generalizes: `cutout_shape` is in -*pixels*, so changing `final_scale` changes the sky footprint — size your cutout to the -coverage your dithers actually deliver. -""" -spec = TargetSpec( - name="j0252+0039_f160w", # Output folder name under output/. - ra=RA, # Target right ascension in degrees (cutout centre). - dec=DEC, # Target declination in degrees. - instrument="wfc3_ir", # IR adapter: _flt (already e-/s), no CTE, native 0.128"/pix. - filter_name="F160W", # The IR snapshot's filter (discovered live from MAST; see below). - final_scale=IR_SCALE, # 0.065"/pix — the adapter's half-native recommendation. - final_pixfrac=1.0, # Few-dither snapshot -> full drop, or the coverage guard refuses (see above). - cutout_shape=(215, 215), # 14" at 0.065"/pix — match the ACS footprint, stay inside real coverage. -) - -""" -A note on the filter: this workspace pins F160W for reproducibility, but the integration -script that established this anchor did not — it asked MAST which WFC3/IR filters actually -cover the target and picked from the answer. That live-discovery idiom (query, filter to -direct observations, drop the HAP `"detection"` pseudo-filter) is demonstrated in -`hst_wfc3_uvis/step_by_step.py` and works identically for the IR channel. - -__The Reduction__ - -One call, as always: MAST download of the `_flt` exposures, CRDS (`iref`) reference sync, -alignment, AstroDrizzle onto the fine grid, noise map with the fine-grid R, PSF from field -stars, packaging with the guards you now understand. -""" -print( - "Running the J0252+0039 F160W IR reduction. The first run downloads the " - "exposures from MAST and syncs CRDS references (expect tens of minutes); " - f"re-runs re-use the cache under {CACHE_ROOT}." -) - -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / spec.name - -print(f"Products written to: {out_dir}") - -""" -__Provenance__ - -On few-dither IR data, two diagnostics deserve first attention: - -- `drizzle.weight_uniformity` — RMS/median of the weight map, the numerical form of the - coverage story above. Below ~0.2 is the acceptable regime; pixfrac 0.8 on this data fails - the *finite-noise* guard outright, and even passing values on IR snapshots run higher than - a well-dithered ACS program's. -- `noise.correlated_noise_factor` — should match the fine-grid `casertano_r` computed by - hand above. - -The IR channel has no CTE stage, so unlike UVIS there is no CTE model to audit — one whole -class of systematics simply absent. -""" -print(f"weight_uniformity : {record['drizzle']['weight_uniformity']}") -print(f"correlated_noise_R: {record['noise']['correlated_noise_factor']:.3f} (by hand: {r_ir_fine:.3f})") -print(f"n_exposures : {record['acquire']['n_exposures']}") -print(f"psf : {json.dumps(record['psf'], indent=2)}") - -""" -__Plots__ - -The product triplet, arcsinh-scaled. Note the noise map's structure: on a few-dither IR -mosaic the per-pixel depth genuinely varies across the field (each pixel's weight counts the -exposures that covered it), so a spatially-structured noise map here is *information*, not an -artifact. -""" -from astropy.io import fits - -data = fits.getdata(out_dir / "data.fits").astype(float) -noise = fits.getdata(out_dir / "noise_map.fits").astype(float) -psf = fits.getdata(out_dir / "psf.fits").astype(float) - -sky_rms = record["noise"]["empirical_background_rms"] - -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(1, 3, figsize=(14, 4.5)) - -im0 = axes[0].imshow(np.arcsinh(data / sky_rms), origin="lower", cmap="magma") -axes[0].set_title("data (arcsinh, sky-RMS units)") -fig.colorbar(im0, ax=axes[0], fraction=0.046) - -im1 = axes[1].imshow(noise, origin="lower", cmap="viridis") -axes[1].set_title("noise map (e-/s)") -fig.colorbar(im1, ax=axes[1], fraction=0.046) - -im2 = axes[2].imshow(np.arcsinh(psf / psf.max() * 1e3), origin="lower", cmap="magma") -axes[2].set_title("psf (arcsinh)") -fig.colorbar(im2, ax=axes[2], fraction=0.046) - -fig.suptitle(f"{spec.name}: WFC3/IR F160W at {IR_SCALE}\"/pix, pixfrac 1.0") -fig.tight_layout() - -plot_path = plot_dir / "start_here_products.png" -fig.savefig(plot_path, dpi=120) -plt.close(fig) - -print(f"Plot saved to: {plot_path}") - -""" -__Load the Dataset in PyAutoLens__ - -The IR products obey the same contract as every **PyAutoReduce** dataset — data and matching -RMS noise in e-/s, an odd unit-normalized PSF, pixel scale in the provenance — so the load -is identical. As everywhere, the import is guarded: **PyAutoReduce** never depends on -**PyAutoLens**. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "PyAutoLens is not installed, so the final loading step is skipped. " - "Run `pip install autolens` to model this dataset — the reduction " - "itself is complete and the FITS products are on disk." - ) - -if al is not None: - dataset = al.Imaging.from_fits( - data_path=out_dir / "data.fits", - noise_map_path=out_dir / "noise_map.fits", - psf_path=out_dir / "psf.fits", - pixel_scales=record["package"]["pixel_scale"], - ) - print( - f"Loaded al.Imaging dataset: shape {dataset.data.shape_native} at " - f"{record['package']['pixel_scale']}\"/pix — pair it with the UVIS " - "F390W dataset for multi-wavelength lens modeling " - "(see autolens_workspace/scripts/multi_dataset)." - ) - -""" -__Wrap Up__ - -You have reduced data from HST's infrared channel and met everything that makes it -different: up-the-ramp readout with per-exposure CR rejection, `_flt` files already in e-/s, -no CTE stage at all, undersampled pixels that make the drizzle scale the central dial, the -fine-grid Casertano branch that prices that dial in correlated noise — and the zero-weight- -speckle guard refusal that turned a snapshot's sparse dithers into this channel's most -useful lesson: pixfrac 1.0 or a coarser scale, never a holey dataset. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_wfc3_ir/step_by_step.py`: the IR calibration stages in detail — reference - pixels, ramp fitting, why `driz_cr` still runs, and the WHT-uniformity diagnostic. -- `scripts/hst_wfc3_uvis/start_here.py`: the UVIS half of this same lens, with its published - noise anchor. -- `scripts/hst_acs/step_by_step.py`: the shared HST machinery both WFC3 channels inherit. -- `scripts/guides/noise_maps.py`: the noise recipe and both Casertano branches in full. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_wfc3_ir/step_by_step.py b/workspace/scripts/hst_wfc3_ir/step_by_step.py deleted file mode 100644 index 01c70d2..0000000 --- a/workspace/scripts/hst_wfc3_ir/step_by_step.py +++ /dev/null @@ -1,303 +0,0 @@ -""" -Step By Step: HST WFC3/IR -========================= - -This script walks the WFC3/IR reduction stage by stage, teaching what happens to the data at -each step of the pipeline — as a *delta* against the ACS/WFC reference and the UVIS delta -folder beside this one. The drizzle/noise/PSF/packaging machinery is shared across all HST -channels and documented once in `hst_acs/step_by_step.py`; the UVIS-specific CCD story -(CTE, post-flash, `iref`) is in `hst_wfc3_uvis/step_by_step.py`. Here the focus is the parts -of the pipeline the infrared detector rewrites: the calibration chain that turns -non-destructive ramps into count-rate images, cosmic-ray handling split across two stages, -and the coverage economics of few-dither data on a fine output grid. - -If you have not read `hst_wfc3_ir/start_here.py`, do that first — it establishes the -detector physics, the 0.065"/pixel drizzle choice and the zero-weight-speckle finding this -script keeps building on. - -__Contents__ - -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root, absolutely. -- **Stage 1, Acquisition:** `_flt` products, `iref` references, and no CTE variant to choose. -- **Stage 2, Calibration:** The `calwf3` IR chain — from ramps to count-rate images. -- **Stage 3, Alignment:** Shared with ACS — cross-references only. -- **Stage 4, Drizzle:** Why `driz_cr` still runs on CR-cleaned exposures, and few-dither reality. -- **Stage 5, Noise:** The shared recipe on data that is already e-/s, with the fine-grid R. -- **Stage 6, PSF and Packaging:** Shared with ACS; the 78 ke- saturation delta. -- **The Reduction:** Run the pipeline and produce the evidence to read. -- **Reading the Evidence:** The WHT-uniformity reading and the per-stage provenance audit. -- **Units and Closure:** Confirm the e-/s contract from the headers and close the noise floor. -- **Wrap Up:** Summary of the script and next steps. - -__Imports__ - -Alongside the **PyAutoReduce** entry points we import the adapter registry (to read the IR -channel's constants rather than hard-code them) and the public noise helpers demonstrated -standalone below. -""" -import json -from pathlib import Path - -import numpy as np - -from autoreduce import TargetSpec, reduce_target -from autoreduce import instruments -from autoreduce.noise.rms import casertano_r, empirical_background_rms - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -RA, DEC = 43.188375, 0.666222 # SDSS J0252+0039 — the IR anchor field (see start_here.py). - -IR = instruments.get("wfc3_ir") - -print(f"wfc3_ir adapter: native {IR.native_scale}\"/pix, recommends {IR.recommended_final_scale}\"/pix") - -""" -__Stage 1, Acquisition__ - -The acquire stage is the shared MAST machinery (`hst_acs/step_by_step.py`, Stage 1): query -the archive, filter to direct program observations — dropping the HAP duplicates, exactly as -demonstrated live in `hst_wfc3_uvis/step_by_step.py` — and download the calibrated exposures -into the per-target cache. The IR deltas: - -- The calibrated product is the **`_flt`**, full stop. On the CCD channels you choose the - CTE-corrected `_flc` over the uncorrected `_flt`; on IR *no CTE correction exists* — - a HgCdTe array does not clock charge across the detector, so there is nothing to correct - and no `_flc` variant in the archive. One entire class of systematics (and one entire - decision) is simply absent. -- Reference files come from CRDS (https://hst-crds.stsci.edu) under the WFC3 environment key - `iref`, shared with UVIS. - -__Stage 2, Calibration__ - -The `calwf3` IR chain (WFC3 Data Handbook, https://hst-docs.stsci.edu/wfc3dhb; Python -tooling at https://wfc3tools.readthedocs.io) is where the infrared detector rewrites the -rules. Each raw IR exposure is a MULTIACCUM sequence — the detector read non-destructively -many times up the ramp — and calibration turns that sequence into one count-rate image: - -- **Reference-pixel correction:** the detector's border pixels are blind to light and track - the electronic bias level read by read; subtracting them removes drifts a CCD would handle - with overscan. -- **Zero-read subtraction:** the first read is taken as the ramp's zero point and subtracted, - removing pedestal structure present before integration effectively began. -- **Non-linearity correction:** HgCdTe pixels respond non-linearly as they fill; each read is - corrected up to the saturation flag (the effective full well is ~78 ke-). -- **Dark subtraction:** a MULTIACCUM dark matched to the same read sequence is subtracted. -- **Ramp fitting (CRCORR):** the pipeline fits a slope — counts per second — to each pixel's - reads, and here cosmic rays are caught: a CR hit is a *jump* between two reads, glaringly - inconsistent with a smooth ramp, so the fit detects the discontinuity, splits the ramp at - it, and estimates the rate from the clean segments. - -The output `_flt` is therefore already in **electrons per second**, already largely -CR-cleaned, with an ERR extension and DQ flags — a fundamentally more processed object than -a CCD `_flt`/`_flc`, which is a single destructive read in electrons. - -__Stage 3, Alignment__ - -Identical to the other HST channels — the archive's Gaia-tied WCS is trusted, with a -cross-correlation diagnostic recorded per run. See `hst_acs/step_by_step.py` Stage 3. - -__Stage 4, Drizzle__ - -The combination stage is the shared AstroDrizzle path (https://hst-docs.stsci.edu/drizzpac), -with two IR-flavored points worth understanding: - -- **`driz_cr` still runs on multi-exposure stacks.** If the ramp fit already rejected the - cosmic rays, why keep the median-stack CR flagging? Because ramp fitting is not perfect: - hits in the final read interval, grazing hits, and pixels whose ramps were too short after - splitting all leave *residue*. This is the defaults-first policy — the instrument team's - standard stack processing is kept unless a lensing requirement demands otherwise — with - the caveat recorded that it should be revisited if IR reductions ever show over-flagging. -- **Few-dither reality.** Everything about the IR drizzle is governed by how many - sub-pixel-distinct dither positions the program took. Deep IR programs take four or more, - which is what makes half-native grids work at all. Snapshots (like this anchor) take few — - and on a fine grid the drops must stay wide (pixfrac 1.0) or coverage develops - zero-weight speckle and the reduction refuses to package, the finding documented in - `start_here.py`. - -__Stage 5, Noise__ - -The recipe is the shared one (`guides/noise_maps.py`): - - sigma_i = R * sqrt(N_i / t_exp + 1 / W_i) - -with the Poisson term built from the source counts and the total exposure time, and the -background term from the IVM drizzle weights. Nothing in the formula changes for IR — the -mosaic is in e-/s either way (on CCDs the drizzle's `final_units='cps'` conversion made it -so; on IR the `_flt` arrived that way) — but one input changes a lot: **R**. At the -recommended 0.065"/pixel from 0.128" native, the scale ratio s ~ 0.51 is *smaller than the -pixfrac* (1.0), which puts the reduction on the fine-grid branch of the Casertano et al. -(2000, AJ 120, 2747) formula, where correlation grows steeply as the grid refines. Compute -both branches with the public helper: -""" -s = IR.recommended_final_scale / IR.native_scale - -print(f"scale ratio s = {s:.3f} (fine-grid branch applies, since s < pixfrac = 1.0)") -print(f"R at s = 1.0 (native-scale CCD convention): {casertano_r(pixfrac=1.0, scale_ratio=1.0):.3f}") -print(f"R at s = {s:.3f} (this IR reduction) : {casertano_r(pixfrac=1.0, scale_ratio=s):.3f}") - -""" -The delivered noise map is inflated by that larger R wholesale — the transparent price of -drizzling to a resolving grid. If a modeling analysis is noise-correlation sensitive (e.g. -the surface-brightness-anomaly science behind the UVIS anchor), the recorded R is the number -to propagate. - -__Stage 6, PSF and Packaging__ - -Both shared with ACS: the PSF tiers, star selection and `psf_star_pass` are documented in -`hst_acs/psf.py` (with the star selection's saturation ceiling using the IR adapter's -~78 ke- effective full well), and the packaging contract — strict cutout, masked-by-noise, -finite-noise guard — in `guides/output_contract.py`. On IR the finite-noise guard is not a -formality; it is the enforcement mechanism of the pixfrac rule, as `start_here.py` showed. - -__The Reduction__ - -Run the anchor reduction — the identical spec to `start_here.py`, so a warm cache makes this -fast and the two scripts share one output folder. -""" -spec = TargetSpec( - name="j0252+0039_f160w", # Same name as start_here.py -> shared cache + output. - ra=RA, # Target right ascension in degrees. - dec=DEC, # Target declination in degrees. - instrument="wfc3_ir", # IR adapter: _flt (e-/s), no CTE, native 0.128"/pix. - filter_name="F160W", # The IR snapshot's filter. - final_scale=IR.recommended_final_scale, # 0.065"/pix — half-native (see start_here.py). - final_pixfrac=1.0, # The few-dither pixfrac rule (see start_here.py). - cutout_shape=(215, 215), # 14" footprint at 0.065"/pix, inside real coverage. -) - -print( - "Running the J0252+0039 F160W reduction (cached exposures re-used if " - "start_here.py ran first; otherwise the first run downloads from MAST " - "and takes tens of minutes)..." -) - -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / spec.name - -""" -__Reading the Evidence__ - -The stages are internal to `reduce_target` (a reduction is one declared, reproducible unit), -so the audit runs on the provenance. First the acquisition and instrument blocks — confirm -the `_flt` product path and the exposure count; on a snapshot program expect a *small* -number, which is the root of everything coverage-related above. -""" -provenance = json.loads((out_dir / "reduction.json").read_text()) - -print(f"instrument block : {json.dumps(provenance['instrument'], indent=2)}") -print(f"n_exposures : {provenance['acquire']['n_exposures']}") - -""" -Now the reading this script has been building to: the **WHT-uniformity diagnostic** on a -few-dither IR mosaic. The statistic is RMS/median of the positive drizzle weights; the STScI -rule of thumb calls values above ~0.2 a sign the pixfrac is too small for the dither -pattern. Interpret the recorded value on a sliding scale: - -- A well-dithered ACS program at pixfrac 0.8 measures ~0.07 — smooth, uniform coverage. -- This IR snapshot at pixfrac 1.0 sits *higher* — few dithers genuinely deliver less uniform - depth, and the structured noise map reflects it honestly. -- The same data at pixfrac 0.8 does not merely score badly: it develops outright zero-weight - pixels, and fails the finite-noise guard before this diagnostic even gets its say. - -The uniformity number, the R factor and the guard together are the full coverage story of an -IR reduction — all three recorded, every run. -""" -print(f"drizzle block : {json.dumps(provenance['drizzle'], indent=2)}") - -""" -The noise block: the recorded `correlated_noise_factor` should match the fine-grid -`casertano_r` computed by hand above, and `empirical_background_rms` is the measured sky RMS -in e-/s. Both helpers are public, so we close the loop on the packaged cutout directly (the -cutout value differs slightly from the recorded full-mosaic value — same estimator, -different footprint): -""" -from astropy.io import fits - -data = fits.getdata(out_dir / "data.fits").astype(float) - -print(f"noise block : {json.dumps(provenance['noise'], indent=2)}") -print(f"R by hand (fine-grid branch) : {casertano_r(pixfrac=1.0, scale_ratio=s):.4f}") -print(f"empirical sky RMS (cutout) : {empirical_background_rms(data):.5f} e-/s") - -""" -Finally the PSF block — tier, star count, star-source pass — and the package block, whose -`pixel_scale` is what a **PyAutoLens** load consumes. Both contracts are the shared ones -(`hst_acs/psf.py`, `guides/output_contract.py`). -""" -print(f"psf block : {json.dumps(provenance['psf'], indent=2)}") -print(f"package block : {json.dumps(provenance['package'], indent=2)}") - -""" -__Units and Closure__ - -Two final pieces of evidence, read straight off the products. - -First, units. Stage 2 claimed the IR `_flt` arrives in electrons per second and the pipeline -keeps it that way (`final_units='cps'` is a no-op conversion for IR); the packaged header -should therefore carry `BUNIT` in count-rate units, with the total and per-exposure times -preserved alongside. This is worth checking on *every* dataset you hand to a lens model — -a units mistake at this stage silently rescales every flux the model infers. -""" -with fits.open(out_dir / "data.fits") as hdul: - header = hdul[0].header - -print(f"BUNIT : {header.get('BUNIT', 'missing')}") -print(f"EXPTIME : {header.get('EXPTIME', 'missing')}") -print(f"TEXPTIME: {header.get('TEXPTIME', 'missing')}") - -""" -Second, the noise-floor closure — the same check the UVIS anchor validates against its -published sigma_sky, run here as internal validation (no published IR anchor exists for this -field). In blank sky the noise recipe reduces to approximately R * sigma_sky, so the noise -map's faint floor (5th percentile) over R times the measured sky RMS should sit near 1. On a -few-dither IR mosaic expect the agreement to be looser than the UVIS case — the spatially -varying depth that the weight map records genuinely widens the noise distribution — but a -ratio far from unity would mean the noise map and the sky disagree, and that is a reduction -problem, not a statistic to explain away. -""" -noise = fits.getdata(out_dir / "noise_map.fits").astype(float) - -r_factor = provenance["noise"]["correlated_noise_factor"] -sky_rms = provenance["noise"]["empirical_background_rms"] - -closure = float(np.nanpercentile(noise, 5)) / (r_factor * sky_rms) - -print(f"noise floor / (R * sky RMS): {closure:.3f}") - -""" -__Wrap Up__ - -You have now seen the whole IR pipeline at stage resolution: an acquisition with no CTE -decision to make, a calibration chain that fits ramps instead of reading charge — rejecting -cosmic rays per exposure and delivering `_flt` files already in e-/s — a drizzle stage that -keeps `driz_cr` for the ramp fit's residue and prices fine grids in coverage, a noise stage -whose only IR novelty is the larger fine-grid R, and the diagnostics (WHT uniformity, R, the -finite-noise guard) that together tell you whether a few-dither IR dataset is shippable at -your chosen dials. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_wfc3_ir/start_here.py`: the anchor reduction, the drizzle-choice arithmetic - and the zero-weight-speckle finding in full. -- `scripts/hst_acs/step_by_step.py`: the shared stage-by-stage reference (alignment, - drizzle, PSF, packaging) this script deltas against. -- `scripts/hst_wfc3_uvis/step_by_step.py`: the CCD sibling — CTE, post-flash and the live - MAST discovery idiom that works for both channels. -- `scripts/guides/noise_maps.py`: the noise recipe and both Casertano branches, derived. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_wfc3_uvis/README.md b/workspace/scripts/hst_wfc3_uvis/README.md deleted file mode 100644 index 95a7911..0000000 --- a/workspace/scripts/hst_wfc3_uvis/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# hst_wfc3_uvis - -HST WFC3/UVIS reductions — the blue/optical CCD channel, taught as a *delta* against the -ACS/WFC reference (`scripts/hst_acs/`): same `_flc`/AstroDrizzle machinery, with `calwf3`, -`iref` references, the 0.0396"/pixel native scale, post-flash and a 63 ke- full well as the -channel-specific story. The validation anchor is the published Bayer et al. -(arXiv:1803.05952) F390W reduction of SDSS J0252+0039. - -Read in order: - -1. `start_here.py` — the anchor reduction end to end at the published dials, with the noise - map validated against the published sigma_sky ~ 0.002 e-/s. -2. `step_by_step.py` — every stage's UVIS delta vs ACS, plus the live MAST filter-discovery - idiom. -3. `psf.py` — the UVIS PSF on a star-rich field (Omega Cen F606W): photutils vs STARRED - head to head. -4. `simulator.py` — inject a synthetic arc into the real J0252 frames and measure flux - recovery. - -All scripts need network access (MAST + CRDS) and the `[hst]` extra (drizzlepac). diff --git a/workspace/scripts/hst_wfc3_uvis/__init__.py b/workspace/scripts/hst_wfc3_uvis/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/workspace/scripts/hst_wfc3_uvis/psf.py b/workspace/scripts/hst_wfc3_uvis/psf.py deleted file mode 100644 index 7067cb3..0000000 --- a/workspace/scripts/hst_wfc3_uvis/psf.py +++ /dev/null @@ -1,290 +0,0 @@ -""" -PSF: HST WFC3/UVIS -================== - -This script builds the WFC3/UVIS point spread function on a genuinely star-rich field — a -pointing in the globular cluster Omega Centauri — and compares **PyAutoReduce**'s two -empirical PSF backends head to head: the default photutils effective-PSF builder and the -STARRED joint-reconstruction backend. - -Why a star cluster, in a workspace about lensing? Because PSF validation needs *stars*, and -lens fields rarely provide them. In extragalactic fields most "point sources" that survive a -star-finder's cuts are compact galaxies, which silently broaden any PSF built from them. On a -real stellar field the empirical, sub-pixel-registered stack of the stars *is* the PSF, so -concentration and radial-profile comparisons against it are meaningful truth references. This -is the field on which the UVIS backend comparison was validated, and this script reproduces -that setup. - -For the full PSF architecture — the tier system, `psf_star_pass`, per-frame PSFs, the -drizzled-PSF invariant — see `hst_acs/psf.py`; this script covers what UVIS adds and what a -crowded field changes. - -__Contents__ - -- **Why PSF Accuracy Matters:** PSF errors masquerade as the lensing signal. -- **The Focus Problem:** HST's PSF breathes, so empirical beats optical modeling. -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root, absolutely. -- **Target Spec:** Omega Cen F606W — a clean 6x60s dithered visit. -- **Star Selection in a Crowded Field:** What the selection cuts do when stars are everywhere. -- **The Default Backend:** Reduce with the photutils ePSF (tier 1). -- **The STARRED Backend:** Reduce again with `psf_backend="starred"` (tier 1b). -- **The Head To Head:** The validated comparison numbers and what they mean. -- **Diagnostics and Plots:** Read the psf provenance block and visualize both PSFs. -- **Wrap Up:** Summary of the script and next steps. - -__Why PSF Accuracy Matters__ - -A lens model fits the data *convolved with the PSF*. If the PSF is wrong, the model cannot -reproduce the arcs no matter how good the mass model is, and the misfit appears as structured -residuals along exactly the high-S/N lensed features. Those residuals are indistinguishable -from real astrophysical perturbations — dark substructure, source complexity — so a PSF error -does not just degrade the fit, it *biases the science* by faking or masking the faintest -signals lens modeling looks for. This is why **PyAutoReduce** treats the PSF as a first-class -product with recorded diagnostics, not an afterthought. - -__The Focus Problem__ - -HST's focus changes continuously as the telescope's structure expands and contracts through -each orbit ("breathing"), so the PSF varies from exposure to exposure. The modern response is -*empirical*: measure the PSF from stars in the science data itself, in the tradition of the -effective PSF (ePSF) of Anderson & King (2000, PASP 112, 1360; -https://ui.adsabs.harvard.edu/abs/2000PASP..112.1360A) — the instrumental PSF convolved with -the pixel response, built from dithered star images. STScI maintains focus-diverse ePSF -libraries for WFC3 — Anderson (2016, WFC3 ISR 2016-12) for the IR channel and WFC3 ISR -2018-14 for focus-diverse UVIS models, collected at -https://www.stsci.edu/hst/instrumentation/wfc3/data-analysis/psf — which contextualize what -"the" UVIS PSF even means: a family, indexed by focus. **PyAutoReduce** builds its PSF from -the field's own stars, which bakes the observation's actual focus in automatically. - -The two backends this script compares implement that empirical philosophy differently: - -- **photutils ePSFBuilder** (tier 1, the default; - https://photutils.readthedocs.io/en/stable/user_guide/epsf.html) — the open-source - Anderson & King iterative ePSF construction. -- **STARRED** (tier 1b, opt-in) — JAX-based joint PSF reconstruction from several stars with - starlet-regularized narrow-PSF fitting, from the COSMOGRAIL lensing community - (Michalewicz et al. 2023, JOSS 8(85), 5340; Millon et al. 2024, AJ, - https://arxiv.org/abs/2402.08725). -""" - -""" -__Imports__ - -`dataclasses.replace` is the idiom for spec variants: `TargetSpec` is frozen, so a second -reduction with one dial changed is a `replace(spec, ...)`, never a mutation. -""" -import dataclasses -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np - -from autoreduce import TargetSpec, reduce_target - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -""" -__Target Spec__ - -The field: Omega Centauri (NGC 5139) through WFC3/UVIS F606W, proposal 15733 — six dithered -60s exposures at a single pointing about 10" off the cluster core. Short exposures on a -globular cluster give exactly what PSF work wants: hundreds of bright-but-unsaturated, -well-sampled, genuinely stellar point sources in one frame. - -The drizzle dials are the UVIS-standard native scale with pixfrac 1.0 (see -`hst_wfc3_uvis/start_here.py` for why), and the cutout is enlarged to 401x401 pixels (~16") -to enclose a rich isolated-star sample. -""" -RA, DEC = 201.69283, -47.47906 # Mean pointing of the proposal-15733 F606W exposures. - -spec = TargetSpec( - name="omegacen_f606w", # Output folder name under output/. - ra=RA, # Field centre right ascension in degrees. - dec=DEC, # Field centre declination in degrees. - instrument="wfc3_uvis", # UVIS adapter: _flc, iref, native 0.0396"/pix, sat 63 ke-. - filter_name="F606W", # Broad V-band: high stellar S/N in 60s. - proposal_ids=("15733",), # Pin to one clean 6x60s dithered visit. - final_scale=0.0396, # Native-scale output (UVIS adapter recommendation). - final_pixfrac=1.0, # 6-dither visit -> full drop for guaranteed coverage. - cutout_shape=(401, 401), # ~16": a rich isolated-star sample for the PSF build. -) - -""" -__Star Selection in a Crowded Field__ - -Before either backend runs, **PyAutoReduce** selects PSF stars from the mosaic with a fixed -set of cuts (they are deliberately *not* `TargetSpec` dials — the selection is part of the -pipeline's contract, so its behavior is uniform across datasets): a 10-sigma detection -threshold, point-like sharpness and roundness cuts, a minimum separation of 25 pixels between -candidates, an edge margin, an exclusion radius around the target position, and a saturation -ceiling — candidates whose peak exceeds 70% of full well are rejected. - -On this field those cuts earn their keep in an unusual way. In a globular cluster the -challenge is not finding stars but *rejecting* most of them: crowding violates the minimum -separation for the majority of detections, and the brightest cluster stars saturate — with -the UVIS full well at ~63 ke- and 60s exposures, any star peaking above roughly -0.7 x 63000 / 60 e-/s is excluded. What survives is a subset of isolated, unsaturated, -well-exposed stars — fewer than you might expect from a cluster field, and exactly the ones -you want. - -__The Default Backend__ - -First, the tier-1 reduction with the default `psf_backend="epsf"` (photutils). One recorded -finding from validating this exact field is worth knowing: the photutils builder can fail on -the extended 61x61 `psf_full` product here — background over-subtraction leaves the faint -ePSF wings slightly negative, and the builder rejects the non-positive flux — while the -compact 21x21 core builds cleanly. STARRED, with its regularized wings, delivered both. A -backend difference you only discover on real data, which is why the validation anchors are -real fields. -""" -print( - "Running the Omega Cen F606W reduction (photutils ePSF backend). First " - "run downloads 6 exposures from MAST + CRDS references (expect tens of " - "minutes); re-runs use the cache." -) - -record_epsf = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_epsf = OUTPUT_ROOT / spec.name - -print(f"photutils psf block: {json.dumps(record_epsf['psf'], indent=2)}") - -""" -__The STARRED Backend__ - -Now the same field through tier 1b. STARRED ships as the optional `[starred]` extra -(`pip install "autoreduce[starred]"`) because it pulls in JAX and is GPL-licensed; it is -imported lazily, and if it is missing the pipeline raises rather than silently falling back — -you asked for STARRED, you get STARRED or a loud error. The `try/except` below turns that -into a friendly message so this script degrades gracefully on a minimal install. - -The spec is the same field with two changes: a new `name` (so the two reductions live side by -side under `output/`) and the backend dial. -""" -spec_starred = dataclasses.replace( - spec, - name="omegacen_f606w_starred", # Separate output folder for the side-by-side comparison. - psf_backend="starred", # Tier 1b: STARRED joint PSF reconstruction. -) - -from autoreduce.psf.starred_epsf import StarredUnavailableError - -print("Running the same reduction with psf_backend='starred' (cache warm — fast)...") - -try: - record_starred = reduce_target( - spec_starred, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT - ) -except StarredUnavailableError: - record_starred = None - print( - "STARRED is not installed, so the tier-1b reduction is skipped. " - 'Run `pip install "autoreduce[starred]"` to enable it — the ' - "photutils reduction above is complete and this script continues " - "with it alone." - ) - -out_starred = OUTPUT_ROOT / spec_starred.name - -if record_starred is not None: - print(f"STARRED psf block: {json.dumps(record_starred['psf'], indent=2)}") - -""" -__The Head To Head__ - -How do you score two PSFs against each other? On a stellar field there is a truth reference: -register every selected star to sub-pixel accuracy, stack them, and normalize — the empirical -stack. Then compare each backend's PSF to it on simple, robust statistics: the central 3x3 -concentration (how much flux the core holds) and the RMS deviation of the radial profile. - -The validated comparison on this exact field found: - -- empirical star stack: concentration **0.58** (the truth reference); -- STARRED tier 1b: concentration **0.54** — close to the stack, and the winner on - radial-profile RMS; -- photutils tier 1: concentration **0.39** — a visibly softer core. - -The regime rule **PyAutoReduce** draws from this (and from the matching JWST comparison): -STARRED wins on *well-sampled* data — and UVIS F606W at native scale is well-sampled — while -photutils remains the safe default and wins on undersampled data (e.g. NIRCam SW), where -STARRED's own diagnostics flag the regime (it warns below a sampling FWHM of ~1.6 pixels). -Neither backend is "better"; the sampling regime decides, and the diagnostics recorded in -`reduction.json` tell you which regime you are in. - -__Diagnostics and Plots__ - -The `psf` provenance block records the method, the number of stars used and the star-selection -pass for either backend — compare `n_stars_used` between the two runs above. Now plot the -delivered PSFs side by side on a log stretch, where core sharpness and wing behavior are -visible at once. -""" -from astropy.io import fits - -psf_epsf = fits.getdata(out_epsf / "psf.fits").astype(float) - -panels = [("photutils tier 1", psf_epsf)] - -if record_starred is not None: - psf_starred = fits.getdata(out_starred / "psf.fits").astype(float) - panels.append(("STARRED tier 1b", psf_starred)) - -plot_dir = out_epsf / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(1, len(panels), figsize=(5.5 * len(panels), 4.5), squeeze=False) - -for ax, (title, psf) in zip(axes[0], panels): - im = ax.imshow( - np.log10(np.clip(psf / psf.max(), 1e-6, None)), origin="lower", cmap="magma" - ) - ax.set_title(f"{title} (log10, peak-normalized)") - fig.colorbar(im, ax=ax, fraction=0.046) - -fig.suptitle("omegacen_f606w: UVIS PSF backends") -fig.tight_layout() - -plot_path = plot_dir / "psf_backends.png" -fig.savefig(plot_path, dpi=120) -plt.close(fig) - -print(f"Plot saved to: {plot_path}") - -""" -Both reductions also ship `psf_full.fits` — the 61x61 extended-wings PSF for modeling -workflows that need the far profile (`guides/output_contract.py` explains the two-PSF -contract). And both PSFs obey the drizzled-PSF invariant: they are built from stars in the -*drizzled* mosaic, so they carry the same scale, pixfrac and kernel as the data they will be -convolved with. - -__Wrap Up__ - -You have built the UVIS PSF two ways on a field where the truth is knowable, seen the -validated numbers that make STARRED the recommended backend for well-sampled UVIS data, and -read the diagnostics that let you audit the choice per dataset. When you reduce a lens field -(where the "stars" are scarcer and more suspect), the same machinery applies — with the -star-selection cuts and the `psf_star_pass` dial doing proportionally more of the work. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/psf.py`: the full PSF architecture — tiers, `psf_star_pass`, per-frame - PSFs, the drizzled-PSF invariant. -- `scripts/hst_wfc3_uvis/start_here.py`: the UVIS anchor reduction these dials come from. -- `scripts/jwst_nircam/psf.py`: the same STARRED-vs-photutils story in the JWST sampling - regimes. -- `scripts/guides/output_contract.py`: psf.fits vs psf_full.fits and the delivery contract. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_wfc3_uvis/simulator.py b/workspace/scripts/hst_wfc3_uvis/simulator.py deleted file mode 100644 index b60294f..0000000 --- a/workspace/scripts/hst_wfc3_uvis/simulator.py +++ /dev/null @@ -1,257 +0,0 @@ -""" -Simulator: HST WFC3/UVIS -======================== - -This script validates the WFC3/UVIS pipeline end-to-end by *injection*: it paints a synthetic -lensed arc into the real J0252+0039 F390W `_flc` exposures, runs the identical reduction on -the injected frames, and measures how faithfully the known input flux comes back out of the -packaged dataset. - -Injection is the honest way to simulate reduced data. Rendering a synthetic image from -scratch means inventing the sky, the cosmic rays, the bad pixels, the dither geometry and the -noise correlations; injecting into real frames gets all of them for free, exactly as they are -in the data your lens models will fit. If the recovered flux, the noise map and the PSF are -consistent for a source you put in yourself, you can trust them for the sources nature put in. - -This script is deliberately compact — the full injection methodology (unit handling, the -per-frame rendering path, Poisson realization, seed policy, aperture statistics) is -documented in `hst_acs/simulator.py`, which you should read for depth. Here we cover the -UVIS-specific points and run the loop. - -__Contents__ - -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root, absolutely. -- **The Input Image:** Build a pure-numpy Sersic arc in e-/s — no lensing code required. -- **The Injection Dials:** Declare the injection on top of the anchor spec. -- **Clean and Injected Reductions:** Run the pipeline twice with a shared cache. -- **Recovery:** Difference the two datasets and check aperture flux recovery. -- **Wrap Up:** Summary of the script and next steps. - -__Imports__ - -Everything here is numpy, astropy and **PyAutoReduce** — building the input image needs no -lensing code, by design: the injection contract takes a plain 2-D FITS image. -""" -import dataclasses -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np - -from autoreduce import TargetSpec, reduce_target - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -RA, DEC = 43.188375, 0.666222 # SDSS J0252+0039 — the UVIS anchor field (see start_here.py). - -""" -__The Input Image__ - -The input to injection is a plain 2-D FITS image with three hard requirements: finite, -non-negative pixels; *not* PSF-convolved (the pipeline convolves it with each frame's own -PSF, so blurring it yourself would double-convolve); and surface brightness in the adapter's -injection units — for HST, electrons per second per pixel. - -We build a simple analytic lensed-arc stand-in: a partial ring whose radial cross-section is -a Sersic n=1 (exponential) profile, - - I(r, phi) = exp( -1.678 * |r - r_ring| / r_eff ) * taper(phi) - -with a smooth Gaussian taper in azimuth so the ring becomes an arc. The 1.678 factor makes -r_eff the profile's half-light radius. This is deliberately *not* a ray-traced lensed source -— the point of this script is the pipeline's bookkeeping, and a fully analytic input keeps -the truth exact. The image is normalized so its pixels sum to exactly the total flux we want -to recover, then written to FITS at a chosen pixel scale (which need not match the detector's -— the renderer resamples through each frame's WCS). -""" -ARC_FLUX_CPS = 25.0 # Total injected flux in e-/s: bright enough for a clean measurement. -ARC_PIXEL_SCALE = 0.02 # Input-image pixel scale in arcsec/pix (finer than UVIS native). -ARC_RING_RADIUS = 1.2 # Arc ring radius in arcsec — a typical galaxy-scale Einstein radius. -ARC_R_EFF = 0.15 # Sersic half-light width of the arc cross-section in arcsec. - -shape = (241, 241) -yy, xx = np.mgrid[0 : shape[0], 0 : shape[1]] -cy, cx = shape[0] // 2, shape[1] // 2 - -r = np.hypot(yy - cy, xx - cx) * ARC_PIXEL_SCALE # Radius from image centre in arcsec. -phi = np.arctan2(yy - cy, xx - cx) # Azimuth in radians. - -arc = np.exp(-1.678 * np.abs(r - ARC_RING_RADIUS) / ARC_R_EFF) # Sersic n=1 cross-section. -arc *= np.exp(-0.5 * ((phi - np.pi / 4.0) / 0.8) ** 2) # Azimuthal taper -> an arc, not a ring. -arc = ARC_FLUX_CPS * arc / arc.sum() # Normalize to the truth flux. - -from astropy.io import fits - -input_dir = OUTPUT_ROOT / "j0252+0039_f390w_injected" -input_dir.mkdir(parents=True, exist_ok=True) - -input_path = input_dir / "input_arc.fits" -fits.PrimaryHDU(arc.astype(np.float32)).writeto(input_path, overwrite=True) - -print(f"Input arc written to: {input_path} (total flux {arc.sum():.3f} e-/s)") - -""" -__The Injection Dials__ - -The injection is declared on the `TargetSpec` like every other dial. Starting from the anchor -spec of `start_here.py`, we add the `inject_*` dials and a new `name` so the injected dataset -lives beside the clean one. The injection position is offset ~2" from the lens so the arc -lands on clean sky and the difference measurement is unambiguous. - -The pipeline renders the arc into every `_flc` exposure through that frame's own -full-distortion WCS, convolves with the frame's PSF, draws Poisson noise for the source's own -counts (seeded per frame — reproducible), and updates the ERR extension. The cache is never -mutated: injection works on copies, so your downloaded exposures stay pristine. - -One UVIS note: this adapter also supports `cr_method="deepcr"` — the CNN cosmic-ray flagger -of Zhang & Bloom (2020, ApJ 889, 24; https://ui.adsabs.harvard.edu/abs/2020ApJ...889...24Z) -as an opt-in alternative to `driz_cr` (see e.g. the label-free UVIS retraining of Chen et -al. 2024). Injection runs are a natural place to try it, since you can measure directly -whether either CR method eats injected flux — the trade study lives in `hst_acs/dials.py`. -""" -INJECT_RA = RA + 2.0 / 3600.0 # Inject ~2" east of the lens, on clean sky. - -spec_clean = TargetSpec( - name="j0252+0039_f390w", # Same name as start_here.py -> its output is re-used if present. - ra=RA, # Target right ascension in degrees. - dec=DEC, # Target declination in degrees. - instrument="wfc3_uvis", # UVIS adapter: _flc, iref, native 0.0396"/pix. - filter_name="F390W", # The anchor filter. - final_scale=0.0396, # Bayer anchor dial (see start_here.py). - final_pixfrac=1.0, # Bayer anchor dial. -) - -spec_injected = dataclasses.replace( - spec_clean, - name="j0252+0039_f390w_injected", # Separate output folder for the injected dataset. - inject_image=str(input_path), # The plain-FITS arc built above (absolute path). - inject_pixel_scale=ARC_PIXEL_SCALE, # arcsec/pix of the input image (required with inject_image). - inject_position=(INJECT_RA, DEC), # (ra, dec) degrees to centre the arc on; default is the target. - inject_seed=0, # Seeds the per-frame Poisson draws -> bit-reproducible. - # cr_method="deepcr", # [Optional] CNN CR flagging instead of driz_cr (UVIS-supported). -) - -""" -__Clean and Injected Reductions__ - -Two reductions with a shared cache: once clean, once injected, otherwise identical — so the -recovery measurement is a difference of two identically-processed datasets and everything -except the arc cancels. If you ran `start_here.py`, the clean reduction and all downloads are -already on disk and only the injected run costs time. -""" -print( - "Running the clean reduction (skipped in effect if start_here.py already " - "ran — the cache and output are shared)..." -) - -record_clean = reduce_target(spec_clean, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -print("Running the injected reduction (cache warm — no new downloads)...") - -record_injected = reduce_target( - spec_injected, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT -) - -out_clean = OUTPUT_ROOT / spec_clean.name -out_injected = OUTPUT_ROOT / spec_injected.name - -""" -The provenance of the injected run carries a dedicated `inject` block — the input image, its -units and flux, the position, the per-frame injected counts and the seed — so a -semi-synthetic dataset can never masquerade as real data. -""" -print(f"inject block: {json.dumps(record_injected['inject'], indent=2)}") - -""" -__Recovery__ - -Now the measurement. Difference the injected and clean cutouts (identical processing means -the real sky, lens galaxy and neighbors cancel), locate the injection position through the -cutout's WCS, and sum the difference inside a 3" aperture. Three numbers to compare: - -- the truth: the input arc's total flux in e-/s; -- the recovery: the aperture sum of the difference image; -- the noise prediction: the quadrature sum of the noise map inside the aperture, which sets - the uncertainty on the recovery. - -A recovery ratio consistent with 1.0 within the noise says the pipeline conserves flux -through rendering, PSF convolution, drizzling, CR flagging and packaging — and that the -noise map is a fair uncertainty for aperture photometry on this dataset. -""" -from astropy.wcs import WCS - -data_clean = fits.getdata(out_clean / "data.fits").astype(float) -data_injected = fits.getdata(out_injected / "data.fits").astype(float) -noise_injected = fits.getdata(out_injected / "noise_map.fits").astype(float) -header = fits.getheader(out_injected / "data.fits") - -diff = data_injected - data_clean - -xy = WCS(header).world_to_pixel_values(INJECT_RA, DEC) -yy, xx = np.mgrid[0 : diff.shape[0], 0 : diff.shape[1]] - -pixel_scale = record_injected["package"]["pixel_scale"] -aperture = np.hypot(yy - xy[1], xx - xy[0]) * pixel_scale <= 3.0 - -recovered = float(diff[aperture].sum()) -noise_pred = float(np.sqrt((noise_injected[aperture] ** 2).sum())) - -print(f"injected flux : {ARC_FLUX_CPS:.3f} e-/s") -print(f"recovered flux : {recovered:.3f} e-/s (3\" aperture)") -print(f"recovery ratio : {recovered / ARC_FLUX_CPS:.3f}") -print(f"aperture noise : {noise_pred:.3f} e-/s") - -fig, axes = plt.subplots(1, 3, figsize=(14, 4.5)) - -for ax, (title, image) in zip( - axes, - [("clean", data_clean), ("injected", data_injected), ("difference", diff)], -): - im = ax.imshow(np.arcsinh(image / 0.01), origin="lower", cmap="magma") - ax.set_title(title) - fig.colorbar(im, ax=ax, fraction=0.046) - -fig.suptitle("j0252+0039_f390w: arc injection and recovery") -fig.tight_layout() - -plot_dir = out_injected / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -plot_path = plot_dir / "injection_recovery.png" -fig.savefig(plot_path, dpi=120) -plt.close(fig) - -print(f"Plot saved to: {plot_path}") - -""" -__Wrap Up__ - -You have injected a known synthetic arc into real UVIS exposures, reduced them exactly as the -real data, and recovered the input flux against the noise map's own uncertainty prediction — -the strongest end-to-end validation a reduction pipeline offers, and one you can rerun on any -field with dials of your choosing. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/simulator.py`: the full injection methodology — rendering, units, seeds, - Poisson realization and the aperture statistics in depth. -- `scripts/hst_wfc3_uvis/start_here.py`: the clean anchor reduction differenced against here. -- `scripts/hst_acs/dials.py`: the CR-method trade study (`driz_cr` vs `deepcr`) injection - helps you probe. -- `scripts/jwst_nircam/simulator.py`: the same idea through the JWST path, in Jy units. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_wfc3_uvis/start_here.py b/workspace/scripts/hst_wfc3_uvis/start_here.py deleted file mode 100644 index 8176ab7..0000000 --- a/workspace/scripts/hst_wfc3_uvis/start_here.py +++ /dev/null @@ -1,333 +0,0 @@ -""" -Start Here: HST WFC3/UVIS -========================= - -This script reduces Hubble Space Telescope WFC3/UVIS imaging of the strong gravitational lens -SDSS J0252+0039 into a modeling-ready dataset — `data.fits`, `noise_map.fits`, `psf.fits` and -`psf_full.fits`, plus a full `reduction.json` provenance record — using **PyAutoReduce**. - -WFC3/UVIS shares almost all of its reduction machinery with ACS/WFC: both are CCD channels whose -calibrated exposures are CTE-corrected `_flc` files, and both run through the same AstroDrizzle -combination, noise and PSF stages. This folder therefore teaches only what *changes* for UVIS. -For the full stage-by-stage depth — what drizzle actually does, how the noise map is derived, -how the PSF tiers work — see `hst_acs/start_here.py` and `hst_acs/step_by_step.py`, which this -script cross-references rather than duplicates. - -The first run downloads the exposures from MAST and syncs CRDS reference files, so expect it to -take tens of minutes; re-runs hit the local cache and are much faster. - -__Contents__ - -- **The UVIS Channel:** What WFC3/UVIS is and why lens modelers use it. -- **The Anchor: SDSS J0252+0039:** The published Bayer et al. F390W reduction this script reproduces. -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root, absolutely. -- **CTE and the flc Product:** UVIS exposures are CTE-corrected before we ever touch them. -- **Drizzle Dials:** Why this reduction drizzles to the native scale with pixfrac 1.0. -- **Target Spec:** Declare the reduction as a frozen `TargetSpec`. -- **The Reduction:** Run the full pipeline with one function call. -- **Provenance:** Walk the returned provenance record's key diagnostics. -- **Noise Validation:** Check the noise map against the published sigma_sky of ~0.002 e-/s. -- **Plots:** Visualize the data, noise map and PSF. -- **Load the Dataset in PyAutoLens:** Load the products as an `al.Imaging` dataset. -- **Wrap Up:** Summary of the script and next steps. - -__The UVIS Channel__ - -WFC3/UVIS is the blue/optical CCD channel of Wide Field Camera 3, with a native plate scale of -0.0396"/pixel — finer than the 0.05"/pixel of ACS/WFC. For strong lensing this matters twice -over: the finer pixels sample the PSF better, and blue filters like F390W isolate the lensed -emission of a star-forming background source while the red, quiescent lens galaxy fades — a -high-contrast view of the arcs that redder bands cannot give. - -The authoritative reference for everything the instrument pipeline does to UVIS data is the -WFC3 Data Handbook (https://hst-docs.stsci.edu/wfc3dhb), which documents the `calwf3` -calibration pipeline, the UVIS charge-transfer-efficiency (CTE) correction (its Chapter 6 is -the UVIS CTE reference) and the data products this script consumes. Image combination follows -the DrizzlePac Handbook (https://hst-docs.stsci.edu/drizzpac). - -__The Anchor: SDSS J0252+0039__ - -Every **PyAutoReduce** instrument channel is validated against a published reduction, and for -UVIS that anchor is the Bayer et al. (https://arxiv.org/abs/1803.05952) F390W reduction of the -strong lens SDSS J0252+0039: output at the native 0.0396"/pixel, drizzle pixfrac 1.0, and a -noise map built as sigma = sqrt(N/W + sigma_sky^2) with a measured sky RMS of -sigma_sky ~ 0.002 e-/s (their Section 3.1). This script reduces the same data with the same -dials and closes the loop on their published numbers. - -Why was so much care lavished on this particular noise map? Because this dataset underpins -*surface-brightness-anomaly* science: Bayer et al. (arXiv:1803.05952, and the follow-up -arXiv:2302.00480) constrain the dark-matter substructure content of the lens from the power -spectrum of tiny surface-brightness fluctuations in the arcs. Any error in the noise map — -an underestimated sky RMS, unaccounted correlated noise from drizzling — masquerades as -exactly the signal being measured. A reduction pipeline that gets the noise right, and writes -down how it did so, is the difference between a substructure measurement and a systematics -measurement. That standard is what **PyAutoReduce** aims to inherit for every dataset. -""" - -""" -__Imports__ - -**PyAutoReduce** exposes exactly two names: `TargetSpec` (the frozen declaration of the -reduction) and `reduce_target` (the one function that executes it). Everything else in this -script is plotting and inspection of the products it writes. -""" -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np - -from autoreduce import TargetSpec, reduce_target - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -""" -__CTE and the flc Product__ - -Like ACS, UVIS is a CCD read out by clocking charge across the detector, and years of radiation -damage have filled the silicon with charge traps that smear faint sources into trails during -readout — charge transfer efficiency (CTE) losses. The standard correction is the pixel-based -model of Anderson & Bedin (2010, PASP 122, 1035; -https://ui.adsabs.harvard.edu/abs/2010PASP..122.1035A), which forward-models the trapping -during readout and iteratively inverts it. For WFC3/UVIS the current implementation is the -v2.0 correction of Anderson et al. (2021, WFC3 ISR 2021-09), run inside `calwf3` as the -standalone-capable `wf3cte` step; the corrected product is the `_flc` file, which is what -**PyAutoReduce** downloads from MAST (never the uncorrected `_flt`). - -This is not a cosmetic fix. By ~2021, a faint source far from the readout amplifier on a low -(~20 e-) background loses almost *half* its flux uncorrected — and lensed arcs are exactly -that: faint, extended, often on low backgrounds. Uncorrected CTE would systematically dim and -trail the arcs whose photometry the lens model fits. - -UVIS observations also routinely apply a *post-flash*: an LED illumination that deliberately -raises the background to ~12-20 e-, filling the charge traps so faint signal survives readout. -The CTE correction and the post-flash work together — but the post-flash raises the sky level, -and with it the background noise, which is why the noise map below is *measured* from the data -rather than assumed. See `hst_wfc3_uvis/step_by_step.py` for the full calibration walk. - -__Drizzle Dials__ - -The combination step drizzles (Fruchter & Hook 2002, PASP 114, 144; -https://ui.adsabs.harvard.edu/abs/2002PASP..114..144F) the `_flc` exposures onto a common -output grid. Two dials define the output: `final_scale` (the output pixel size) and -`final_pixfrac` (how much each input pixel is shrunk before being dripped onto the grid). - -Bayer et al. chose the *native* scale (0.0396"/pixel) with pixfrac 1.0 — plain shift-and-add, -no sub-pixel resampling games. The reasoning is about noise: drizzling to a finer grid, or -shrinking the drops, redistributes each input pixel's noise across multiple output pixels and -*correlates* them. A chi-squared likelihood assumes independent pixels, so correlated noise -must either be minimized or accounted for. At native scale with pixfrac 1.0 the correlation is -as simple as it gets, described by the Casertano et al. (2000, AJ 120, 2747) noise-correlation -factor R = 1.5. - -One honest caveat: Bayer et al. do not use the scalar R at all — they propagate the -correlation exactly, by drizzling *blank-sky noise realizations* through the same pipeline and -measuring the result. **PyAutoReduce** applies the scalar R to the per-pixel RMS instead: a -single number that inflates the noise map so its diagonal is correct on average, at the cost -of ignoring the off-diagonal structure. It is the standard DrizzlePac-handbook treatment -(Section 3.4), it is recorded in `reduction.json` on every run, and the comparison below -accounts for it explicitly — but if your science lives in the noise power spectrum, know the -difference between the two approaches. - -__Target Spec__ - -A **PyAutoReduce** reduction is *declared*, not scripted: every dial lives in a frozen -`TargetSpec`, so the pipeline is a pure function of the spec plus the archive, and the same -spec always reproduces the same dataset. Below is the J0252+0039 F390W spec with the published -Bayer et al. dials. Dials left at their defaults (cutout shape, PSF shapes, CR method...) are -documented in `hst_acs/dials.py` and `guides/target_spec.py`. -""" -RA, DEC = 43.188375, 0.666222 # SDSS J0252+0039: 02h52m45.21s +00d39m58.4s - -spec = TargetSpec( - name="j0252+0039_f390w", # Output folder name under output/. - ra=RA, # Target right ascension in degrees (cutout centre). - dec=DEC, # Target declination in degrees. - instrument="wfc3_uvis", # The UVIS adapter: _flc products, iref references, 0.0396"/pix native. - filter_name="F390W", # The blue filter of the published substructure analysis. - final_scale=0.0396, # Bayer dial: drizzle to the native UVIS scale (no upsampling). - final_pixfrac=1.0, # Bayer dial: full drop (shift-and-add) -> simplest noise correlation, R = 1.5. -) - -""" -__The Reduction__ - -One call runs the whole pipeline: MAST query and download of the F390W `_flc` exposures, CRDS -reference sync, alignment on the archive's Gaia-tied WCS, AstroDrizzle combination with -cosmic-ray flagging, noise-map construction, PSF measurement from field stars, and packaging -of the WCS-preserving cutout. The returned dictionary is the full provenance record — the same -content written to `reduction.json` next to the FITS products. -""" -print( - "Running the J0252+0039 F390W reduction. The first run downloads the " - "exposures from MAST and syncs CRDS reference files (~GBs; expect tens " - "of minutes). Re-runs re-use the cache under " - f"{CACHE_ROOT} and are much faster." -) - -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / spec.name - -print(f"Products written to: {out_dir}") - -""" -__Provenance__ - -Every run reports its diagnostics, so the dials are auditable per dataset. The ones worth -reading on every reduction: - -- `acquire.n_exposures`: how many `_flc` exposures went into the stack. -- `drizzle.weight_uniformity`: the STScI rule-of-thumb statistic RMS/median of the drizzle - weight map. Values above ~0.2 mean the pixfrac is too small for the dither pattern - (coverage speckle or holes) — with pixfrac 1.0 this should pass comfortably. -- `noise.correlated_noise_factor`: the Casertano R applied to the noise map — 1.5 for these - dials (native scale, pixfrac 1.0). -- `psf`: the PSF method, the number of stars used, and which drizzle pass the stars came from. -""" -print(f"n_exposures : {record['acquire']['n_exposures']}") -print(f"weight_uniformity : {record['drizzle']['weight_uniformity']}") -print(f"correlated_noise_R : {record['noise']['correlated_noise_factor']}") -print(f"psf : {json.dumps(record['psf'], indent=2)}") - -""" -__Noise Validation__ - -Now the closure check against the published anchor. Bayer et al. measure a blank-sky RMS of -sigma_sky ~ 0.002 e-/s in this reduction. **PyAutoReduce** measures its own sigma-clipped sky -RMS from the mosaic and records it as `noise.empirical_background_rms` — the two should agree, -since they describe the same pixels through the same dials. - -The second check reads the noise *map*: in blank sky, the per-pixel noise recipe -sigma = R * sqrt(N/W + 1/W) reduces to approximately R * sigma_sky, so the noise map's faint -floor (its 5th percentile) divided by R * sigma_sky should be close to 1. If that ratio drifts -from unity, the noise map and the measured sky disagree — the exact failure mode that would -corrupt a surface-brightness-anomaly measurement, caught here in one line. (Remember the R -accounting from `__Drizzle Dials__`: our sky RMS is measured *before* R is applied, while the -noise map carries R — the ratio below divides it back out.) -""" -from astropy.io import fits - -BAYER_SIGMA_SKY = 0.002 # e-/s, published for the F390W reduction (arXiv:1803.05952). - -noise = fits.getdata(out_dir / "noise_map.fits").astype(float) - -r_factor = record["noise"]["correlated_noise_factor"] -sky_rms = record["noise"]["empirical_background_rms"] - -print(f"empirical sky RMS : {sky_rms:.5f} e-/s") -print(f"published Bayer sigma_sky : {BAYER_SIGMA_SKY:.5f} e-/s") -print(f"sky RMS / published : {sky_rms / BAYER_SIGMA_SKY:.3f}") -print( - f"noise floor / (R * sky RMS): " - f"{float(np.nanpercentile(noise, 5)) / (r_factor * sky_rms):.3f}" -) - -""" -__Plots__ - -The three products a lens model consumes, visualized. The data uses arcsinh scaling — linear -near zero so the sky noise is visible, logarithmic on the bright lens galaxy — which is the -standard way to see faint arcs and the sky in one image. -""" -data = fits.getdata(out_dir / "data.fits").astype(float) -psf = fits.getdata(out_dir / "psf.fits").astype(float) - -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(1, 3, figsize=(14, 4.5)) - -im0 = axes[0].imshow(np.arcsinh(data / sky_rms), origin="lower", cmap="magma") -axes[0].set_title("data (arcsinh, sky-RMS units)") -fig.colorbar(im0, ax=axes[0], fraction=0.046) - -im1 = axes[1].imshow(noise, origin="lower", cmap="viridis") -axes[1].set_title("noise map (e-/s)") -fig.colorbar(im1, ax=axes[1], fraction=0.046) - -im2 = axes[2].imshow(np.arcsinh(psf / psf.max() * 1e3), origin="lower", cmap="magma") -axes[2].set_title("psf (arcsinh)") -fig.colorbar(im2, ax=axes[2], fraction=0.046) - -fig.suptitle(f"{spec.name}: WFC3/UVIS F390W at 0.0396\"/pix, pixfrac 1.0") -fig.tight_layout() - -plot_path = plot_dir / "start_here_products.png" -fig.savefig(plot_path, dpi=120) -plt.close(fig) - -print(f"Plot saved to: {plot_path}") - -""" -__Load the Dataset in PyAutoLens__ - -The product set is exactly the input format of **PyAutoLens**: `data.fits` (e-/s, WCS intact), -`noise_map.fits` (matching RMS, correlated-noise corrected), `psf.fits` (21x21, odd, -unit-normalized) and the pixel scale recorded in `reduction.json`. Any pixel the reduction -could not trust carries a noise value of 1e8 with the data zeroed — "masked by noise" — so no -mask surgery is needed on the modeling side. - -**PyAutoReduce** deliberately never imports **PyAutoLens** (it stays releasable on its own), -so the load below is guarded: if **PyAutoLens** is not installed the reduction is still -complete and the message tells you how to proceed. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "PyAutoLens is not installed, so the final loading step is skipped. " - "Run `pip install autolens` to model this dataset — the reduction " - "itself is complete and the FITS products are on disk." - ) - -if al is not None: - dataset = al.Imaging.from_fits( - data_path=out_dir / "data.fits", - noise_map_path=out_dir / "noise_map.fits", - psf_path=out_dir / "psf.fits", - pixel_scales=record["package"]["pixel_scale"], - ) - print( - f"Loaded al.Imaging dataset: shape {dataset.data.shape_native} at " - f"{record['package']['pixel_scale']}\"/pix — ready for lens modeling " - "(see autolens_workspace/scripts/imaging/start_here.py)." - ) - -""" -__Wrap Up__ - -You have reduced HST WFC3/UVIS imaging of a strong lens end-to-end with the published dials of -its literature anchor, validated the noise map against the published sky RMS, and loaded the -result as a **PyAutoLens** dataset. Along the way you saw what makes UVIS UVIS: the `_flc` CTE -correction, the post-flash noise budget, and the native-scale / pixfrac 1.0 drizzle choice -that keeps the noise correlation simple and honest. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_wfc3_uvis/step_by_step.py`: every UVIS reduction stage in detail, as a delta - against the ACS reference, including live MAST filter discovery. -- `scripts/hst_wfc3_uvis/psf.py`: the UVIS PSF story on a star-rich field, including the - STARRED backend. -- `scripts/hst_wfc3_uvis/simulator.py`: inject a synthetic lensed arc into these same frames - to validate the pipeline end-to-end. -- `scripts/hst_wfc3_ir/start_here.py`: the *other* WFC3 channel — an infrared detector that - works completely differently. -- `scripts/hst_acs/start_here.py`: the ACS/WFC reference reduction all HST channels build on. -- `scripts/guides/noise_maps.py`: the noise recipe, the Casertano R and why chi-squared cares. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/hst_wfc3_uvis/step_by_step.py b/workspace/scripts/hst_wfc3_uvis/step_by_step.py deleted file mode 100644 index b9f2e9a..0000000 --- a/workspace/scripts/hst_wfc3_uvis/step_by_step.py +++ /dev/null @@ -1,307 +0,0 @@ -""" -Step By Step: HST WFC3/UVIS -=========================== - -This script walks the WFC3/UVIS reduction stage by stage, teaching what the instrument -pipeline does to the data at each step — but as a *delta* against the ACS/WFC reference. -UVIS is the ACS-like channel: the same `_flc` CTE-corrected products, the same AstroDrizzle -combination, the same noise recipe and PSF tiers. What changes is the calibration pipeline -name (`calwf3` instead of `calacs`), the reference-file environment (`iref` instead of -`jref`), the plate scale (0.0396"/pixel), the saturation level (~63 ke-) and the post-flash -term in the noise budget. - -For everything the two channels *share* — drizzle theory and its dials, the alignment story, -the full noise derivation, the PSF tier system — read `hst_acs/step_by_step.py`, which is the -canonical stage-by-stage reference. This script covers each stage briefly, flags the UVIS -delta, and defers the depth by cross-reference rather than duplication. - -Along the way it demonstrates one genuinely new idiom: *live MAST discovery* — asking the -archive what imaging exists at a position before committing to a filter, and filtering the -answer down to real, directly-reducible observations. - -__Contents__ - -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root, absolutely. -- **Stage 1, Acquisition:** MAST `_flc` downloads, `iref` references and live filter discovery. -- **Stage 2, Calibration:** What `calwf3` already did to the exposures before we downloaded them. -- **Stage 3, Alignment:** Shared with ACS — the archive's Gaia-tied WCS is trusted. -- **Stage 4, Drizzle:** Shared machinery, UVIS-native scale, and the 63 ke- saturation ceiling. -- **Stage 5, Noise:** The shared recipe, with post-flash in the background budget. -- **Stage 6, PSF and Packaging:** Shared with ACS; cross-references only. -- **The Reduction:** Run the pipeline and produce the evidence to read. -- **Reading the Evidence:** Audit each stage from `reduction.json` and the public helpers. -- **Wrap Up:** Summary of the script and next steps. - -__Imports__ - -Alongside `TargetSpec` and `reduce_target`, this script imports two public helpers it -demonstrates standalone: `select_observations` (the MAST query-hygiene filter used in the -discovery idiom below) and `casertano_r` (the correlated-noise factor, computed by hand to -check the pipeline's arithmetic). -""" -import json -from pathlib import Path - -import numpy as np - -from autoreduce import TargetSpec, reduce_target -from autoreduce.acquire.mast import select_observations -from autoreduce.noise.rms import casertano_r, empirical_background_rms - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -RA, DEC = 43.188375, 0.666222 # SDSS J0252+0039 — the UVIS anchor (see start_here.py). - -""" -__Stage 1, Acquisition__ - -The acquire stage queries MAST (via `astroquery.mast`; -https://astroquery.readthedocs.io/en/latest/mast/mast.html) for the target's exposures and -downloads the calibrated products. The UVIS deltas versus ACS are small but load-bearing: - -- The calibrated product is still the CTE-corrected `_flc` (see Stage 2), but the reference - files that produced it live under the WFC3 environment key **`iref`** (ACS uses `jref`), - and **PyAutoReduce** syncs them from CRDS (https://hst-crds.stsci.edu) into its cache under - a `wfc3` subpath. CRDS maps each exposure to its best reference files; syncing them locally - every run keeps the reduction reproducible (`sync_references=False` opts out for offline - re-runs on a warm cache). -- Everything else — the per-target exposure cache with its manifest, the footprint filter, - the query hygiene below — is identical to ACS (`hst_acs/step_by_step.py`, Stage 1). - -The query hygiene deserves demonstration, because MAST will happily mislead a naive query. A -plain coordinate search also returns *Hubble Advanced Products* (HAP; -https://outerspace.stsci.edu/spaces/HAdP/pages/54558799/Improvements+in+HST+Astrometry) — -skycell mosaics and visit-level associations whose members re-reference the same exposures — -and reducing those alongside the direct exposures would ingest everything twice. -`select_observations` keeps only *direct* program observations (numeric proposal IDs, obs_id -not an `hst_*` HAP product). - -Here is the live-discovery idiom: ask the archive which WFC3/UVIS filters actually cover -J0252+0039, before hard-coding one. Note the final subtlety — HAP composite rows carry the -pseudo-filter `"detection"`, which is not a filter you can point a reduction at, so it is -dropped explicitly. -""" -print("Querying MAST for WFC3/UVIS observations of J0252+0039 (network)...") - -from astropy.coordinates import SkyCoord -from astroquery.mast import Observations - -obs = Observations.query_criteria( - coordinates=SkyCoord(RA, DEC, unit="deg"), # The target position. - radius="0.5 arcmin", # Match the acquire stage's default search radius. - obs_collection="HST", # HST only (MAST also serves JWST, TESS, ...). - instrument_name="WFC3/UVIS", # The channel this folder is about. - dataproduct_type="image", # Imaging only — no spectra. -) - -direct = select_observations(obs) # Keep only direct program observations (drop HAP products). - -filters = sorted({str(row["filters"]) for row in direct} - {"detection"}) - -print(f"Direct WFC3/UVIS observations found; filters available: {filters}") - -""" -The list should include F390W — the filter of the published Bayer et al. -(https://arxiv.org/abs/1803.05952) reduction, and the one we reduce below. In your own work -this is the moment to choose: **PyAutoReduce** reduces one filter per `TargetSpec`, so a -multi-band target is simply several specs. - -__Stage 2, Calibration__ - -**PyAutoReduce** never runs the detector calibration itself — MAST serves products already -processed by the instrument pipeline with the current best reference files. For UVIS that -pipeline is **`calwf3`** (Python tooling and docs: https://wfc3tools.readthedocs.io), the -WFC3 counterpart of ACS's `calacs`. Its CCD chain is the same story you know from -`hst_acs/step_by_step.py` Stage 2 — DQ initialization, bias, dark, flat-field, photometric -keywords — with two UVIS-specific wrinkles: - -- **`wf3cte`**: the pixel-based CTE correction runs as its own standalone-capable step inside - `calwf3`, producing the `_flc` product from the trap model of Anderson & Bedin (2010, PASP - 122, 1035; https://ui.adsabs.harvard.edu/abs/2010PASP..122.1035A) as implemented for UVIS - by Anderson et al. (2021, WFC3 ISR 2021-09). The WFC3 Data Handbook Chapter 6 - (https://hst-docs.stsci.edu/wfc3dhb) collects the references. One honest caveat from that - literature: the correction amplifies read noise in low-S/N pixels — another reason the - noise map is measured from the data, not assumed. -- **Post-flash** (the FLSHCORR step): UVIS observations routinely add an LED pre-exposure - that raises the background to ~12-20 e- so faint charge survives readout past the CTE - traps. It is subtracted like a dark, but its Poisson noise stays in the pixels — see - Stage 5. - -The output `_flc` exposures are in electrons, with an ERR extension (read noise + Poisson) -and a DQ extension flagging known-bad pixels. That is the input state for everything below. - -__Stage 3, Alignment__ - -Identical to ACS: **PyAutoReduce** trusts the archive's Gaia-tied a-priori WCS solutions -rather than re-running TweakReg, and records a cross-correlation diagnostic per run so drift -would be caught. The full reasoning — what the archive's WCS solutions are, when re-alignment -would be warranted — is in `hst_acs/step_by_step.py` Stage 3; nothing changes for UVIS. - -__Stage 4, Drizzle__ - -The combination stage is byte-for-byte the ACS machinery: AstroDrizzle -(https://hst-docs.stsci.edu/drizzpac) with sky matching, median-stack cosmic-ray flagging -(`driz_cr`, DQ bit 4096) and IVM weighting, drizzling the `_flc` frames onto a north-up -output grid in e-/s. The UVIS deltas are numerical: - -- The native scale is **0.0396"/pixel**, and the adapter recommends output at native scale — - the Bayer et al. anchor dials (0.0396, pixfrac 1.0) rather than the ACS/SLACS convention - (0.05, pixfrac 0.8). The trade study between these choices — resolution versus noise - correlation versus coverage — is `hst_acs/dials.py`. -- The full well saturates near **63 ke-** (ACS: ~80 ke-). This threshold feeds the PSF star - selection: candidate stars with peaks above a fraction of saturation are rejected, so the - brightest (and most tempting) stars in the field do not corrupt the PSF with bleeding. - -__Stage 5, Noise__ - -The recipe is shared with ACS (`guides/noise_maps.py` derives it in full): - - sigma_i = R * sqrt(N_i / t_exp + 1 / W_i) - -where N_i is the source count rate, W_i the IVM drizzle weight (inverse background variance) -and R the Casertano et al. (2000, AJ 120, 2747) correlated-noise factor for the chosen -pixfrac and scale. The UVIS delta lives inside W_i: the background variance that the weight -encodes includes the *post-flash* Poisson contribution alongside sky, dark and read noise. -A UVIS frame with a 15 e- post-flash is intrinsically noisier than its exposure time -suggests — the price of surviving readout — and because the weights are built from the real -per-frame backgrounds, the noise map inherits this automatically. No hand-tuning, but worth -knowing when comparing depth between UVIS and ACS programs. - -__Stage 6, PSF and Packaging__ - -Both stages are identical to ACS and are covered there: PSF tiers and star selection in -`hst_acs/psf.py` (with the UVIS-specific star-rich walkthrough in `hst_wfc3_uvis/psf.py`), -the packaging contract — strict-mode cutout, masked-by-noise policy, header preservation — -in `hst_acs/step_by_step.py` Stage 6 and `guides/output_contract.py`. - -__The Reduction__ - -Now run the pipeline with the anchor dials. The spec is identical to `start_here.py` — -including the `name` — so if you ran that script first, the exposures and CRDS references are -already cached and this run skips the downloads entirely. -""" -spec = TargetSpec( - name="j0252+0039_f390w", # Same name as start_here.py -> same cache + output folder. - ra=RA, # Target right ascension in degrees. - dec=DEC, # Target declination in degrees. - instrument="wfc3_uvis", # UVIS adapter: _flc, iref, native 0.0396"/pix, sat 63 ke-. - filter_name="F390W", # Chosen from the live-discovered filter list above. - final_scale=0.0396, # Native-scale output (Bayer anchor dial). - final_pixfrac=1.0, # Full drop: simplest noise correlation (Bayer anchor dial). -) - -print( - "Running the J0252+0039 F390W reduction (cached exposures re-used if " - "start_here.py ran first; otherwise the first run downloads from MAST " - "and takes tens of minutes)..." -) - -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / spec.name - -""" -__Reading the Evidence__ - -The stages above are internal to `reduce_target` — you cannot call them one at a time, by -design (a reduction is one declared, reproducible unit). What you *can* do is audit every -stage from the provenance it leaves behind. `reduction.json` carries one block per stage; -here we walk the UVIS-relevant evidence. - -First, acquisition: how many exposures, from which proposal, through which instrument -adapter. Cross-check against the filter discovery above. -""" -provenance = json.loads((out_dir / "reduction.json").read_text()) - -print(f"instrument block : {json.dumps(provenance['instrument'], indent=2)}") -print(f"n_exposures : {provenance['acquire']['n_exposures']}") - -""" -Second, the drizzle block. `weight_uniformity` is the STScI rule-of-thumb statistic -(RMS/median of the weight map, acceptable below ~0.2); with pixfrac 1.0 on a dithered -program it should pass with room to spare. The block also records the CR method and the -dials actually used — the audit trail for every number in this script. -""" -print(f"drizzle block : {json.dumps(provenance['drizzle'], indent=2)}") - -""" -Third, the noise block — and a by-hand check of its arithmetic. The recorded -`correlated_noise_factor` should equal the Casertano R for our dials: pixfrac p = 1.0 at -scale ratio s = 1.0 (output scale / native scale = 0.0396 / 0.0396), which the public -`casertano_r` helper computes directly. At p = 1, s = 1 the variance-reduction factor is -r = 1 - p/(3s) = 2/3, so R = 1/r = 1.5. -""" -r_by_hand = casertano_r( - pixfrac=1.0, # The final_pixfrac dial. - scale_ratio=1.0, # final_scale / native scale = 0.0396 / 0.0396. -) - -print(f"casertano_r(p=1.0, s=1.0) = {r_by_hand:.4f}") -print(f"recorded in provenance = {provenance['noise']['correlated_noise_factor']:.4f}") - -""" -The noise block also records `empirical_background_rms` — the sigma-clipped sky RMS measured -from the mosaic, the number validated against the published sigma_sky ~ 0.002 e-/s in -`start_here.py`. The helper behind it is public too, so you can reproduce it on the packaged -cutout (expect a value close to, though not identical to, the recorded one — the record -measures the full mosaic, this measures the cutout): -""" -from astropy.io import fits - -data = fits.getdata(out_dir / "data.fits").astype(float) - -sky_rms_cutout = empirical_background_rms(data) - -print(f"empirical sky RMS (cutout) : {sky_rms_cutout:.5f} e-/s") -print(f"empirical sky RMS (recorded) : {provenance['noise']['empirical_background_rms']:.5f} e-/s") - -""" -Finally, the PSF block: which tier built the PSF, from how many stars, and — UVIS-relevant — -which drizzle pass the stars came from (`star_source_pass`). The `psf_star_pass="no_cr"` -second-pass option, which recovers stars that `driz_cr` clipped, is an ACS-shared dial -covered in `hst_acs/psf.py`. -""" -print(f"psf block : {json.dumps(provenance['psf'], indent=2)}") - -""" -And the package block, which seals the contract: the pixel scale a **PyAutoLens** load -consumes, the data units, and the product list. If you ever wonder whether a dataset on disk -was reduced the way you think it was, this file — not your memory of the run — is the -answer. -""" -print(f"package block : {json.dumps(provenance['package'], indent=2)}") - -""" -__Wrap Up__ - -You have seen every stage of the UVIS reduction and what distinguishes it from ACS: `calwf3` -and its standalone `wf3cte` CTE step, `iref` references, the post-flash term riding inside -the drizzle weights, the native 0.0396"/pixel scale and the 63 ke- saturation ceiling — plus -the live MAST discovery idiom and the query hygiene that keeps HAP products out of the -stack. Everything else is shared machinery, documented once on the ACS side. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/step_by_step.py`: the full stage-by-stage reference this script deltas - against. -- `scripts/hst_acs/dials.py`: the drizzle dial trade study (scale, pixfrac, kernel, CR - method). -- `scripts/hst_wfc3_uvis/psf.py`: the UVIS PSF walkthrough on a genuinely star-rich field. -- `scripts/hst_wfc3_ir/step_by_step.py`: the IR channel's stages — a different detector - physics entirely. -- `scripts/guides/noise_maps.py`: the noise recipe and the Casertano R in full. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/jwst_nircam/README.md b/workspace/scripts/jwst_nircam/README.md deleted file mode 100644 index 6225b76..0000000 --- a/workspace/scripts/jwst_nircam/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# JWST NIRCam - -Reducing JWST/NIRCam imaging into modeling-ready lens datasets with **PyAutoReduce**: -MAST level-2 `_cal` exposures (calwebb_image2 output, MJy/sr) combined through the -official `jwst` calwebb_image3 pipeline, with noise read from the propagated ERR array -and an empirical ePSF built from the mosaic. - -The anchor dataset is the COSMOS-Web ring (Mercier et al. 2024), the same lens that -`autolens_workspace/scripts/imaging/start_here.py` models. - -Recommended reading order: - -- `start_here.py` — the full pipeline end to end on the COSMOS-Web ring in F277W. -- `step_by_step.py` — what calwebb Detector1 / Image2 / Image3 do to the data, stage by stage. -- `multi_band.py` — all four COSMOS-Web bands (F115W/F150W/F277W/F444W) into a multi-wavelength dataset. -- `psf.py` — the JWST PSF story on the M92 stellar field: ePSFs, STARRED vs photutils, undersampling. -- `individual.py` — per-exposure `_crf` frame products instead of (as well as) the mosaic. -- `simulator.py` — synthetic-source injection into the real `_cal` frames and flux recovery. diff --git a/workspace/scripts/jwst_nircam/__init__.py b/workspace/scripts/jwst_nircam/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/workspace/scripts/jwst_nircam/individual.py b/workspace/scripts/jwst_nircam/individual.py deleted file mode 100644 index b83629f..0000000 --- a/workspace/scripts/jwst_nircam/individual.py +++ /dev/null @@ -1,352 +0,0 @@ -""" -JWST NIRCam: Individual Frames -============================== - -Everything so far has modeled a *mosaic*: one resampled image combining every exposure. -This script produces the alternative — per-exposure **frame products**: a native-pixel -cutout of the target from every calibrated exposure, each with its own noise map, DQ -map and PSF, plus a manifest that records how they relate. You then model the frames -*jointly*, instead of (or as well as) the mosaic. - -Why would you? Two properties the mosaic can never have: - -- **Uncorrelated noise.** Nothing has been resampled, so no Casertano R, no correlated - neighbours — every frame's per-pixel noise is honest as-is, and the joint likelihood - over frames is exact where the mosaic's is an approximation. -- **The sub-pixel information survives.** On the undersampled SW channel, the dither - pattern's sub-pixel phases carry information a resampled mosaic partially destroys - (aliasing + interpolation). Forward-modeling the frames uses those phases directly. - -The precision-measurement literature is moving the same way: the Roman HLIS shear study -of Yamamoto et al. (2022, https://arxiv.org/abs/2203.08845) benchmarked joint -multi-epoch measurement against coadds and found multi-epoch performed better — -avoiding exactly the coadd-PSF discontinuities and correlated noise the mosaic route -accepts. Honest counterweight: published JWST *extended-source* practice is mosaic-based -today — per-frame lens modeling with NIRCam is ahead of the field, not following it. - -__Contents__ - -- **Imports:** Import **PyAutoReduce** and the supporting libraries. -- **Paths:** Anchor the cache and output folders to the workspace root. -- **The Frame Spec:** `frame_products=True` on the COSMOS-Web ring. -- **The _crf Frames:** Why frames come from outlier_detection's flagged products, not raw `_cal` files. -- **The Run:** Reduce with frame products enabled. -- **The DQ Policy — DO_NOT_USE Only:** Why JWST frame masking differs fundamentally from HST's. -- **Manifest Walk — Schema v2:** Units, sky subtraction, source family, per-frame entries. -- **Registration Residuals and the Reliability Flag:** The edge-of-detector story and the honest null. -- **Per-Frame PSFs and the STPSF Fallback:** Tier-1 ePSF per frame, Tier 2b when stars run out. -- **Frame-Level Caveats:** The artifacts that arrive unmitigated. -- **Loading a Frame in PyAutoLens:** Each frame pair is a native-scale `al.Imaging`. -- **Wrap Up:** Where to go next. - -__Imports__ -""" -from pathlib import Path -import json - -import matplotlib.pyplot as plt -import numpy as np - -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target -from autoreduce.instruments import nircam_adapter_for_filter - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: its combine step -changes the working directory internally, so relative paths would break. - -We give the frame reduction its own output tree (`output/frames/`) while keeping the -spec `name` identical to `start_here.py` — the exposure cache is keyed by name, so the -downloaded `_cal` files are shared and only the pipeline stages re-run. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (shared with start_here.py) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target -FRAMES_OUTPUT = OUTPUT_ROOT / "frames" # this script's outputs, separate from the mosaic run - -""" -__The Frame Spec__ - -One dial turns frames on: `frame_products=True`. The mosaic is still produced — frames -are packaged *as well*, into a `frames/` sub-folder beside the four mosaic products. -Each frame cutout is at the **native** detector scale (0.063"/pixel for LW here; -0.031"/pixel on SW), because the entire point is that nothing gets resampled. - -We use the F277W ring data already cached by `start_here.py`. The SW bands are where -the sampling argument bites hardest, but the LW run demonstrates every mechanism at a -quarter of the pixel count — swap the band to F115W/F150W and the same code runs at -0.031"/pixel. -""" -band = "F277W" -adapter = nircam_adapter_for_filter(band) - -spec = TargetSpec( - name=f"cosmos_web_ring_{band.lower()}", # same name as start_here.py -> shares its exposure cache - ra=150.10048, # the COSMOS-Web ring (Mercier et al. 2024) - dec=1.89301, - instrument=adapter.key, # "nircam_lw" - filter_name=band, - proposal_ids=("1727",), # COSMOS-Web only - final_scale=adapter.recommended_final_scale, # mosaic dials still apply — the mosaic is produced too - final_pixfrac=1.0, - cutout_shape=(209, 209), # mosaic cutout; frame cutouts are sized at native scale automatically - frame_products=True, # THE dial: package a native-frame cutout per exposure alongside the mosaic -) - -""" -__The _crf Frames__ - -Which files should the frames be cut from? Not the raw `_cal` products — something -better exists. When calwebb_image3 runs its `outlier_detection` step (comparing each -exposure against the median of all resampled dithers), it can write back the -per-exposure results: the `_crf` files — tweakreg-updated, outlier-flagged calibrated -frames. **PyAutoReduce** runs image3 with `save_results` enabled on that step precisely -so the `_crf` products exist, and packages frames from them. They are the exact JWST -analogue of HST's driz_cr-flagged `_flc` files: single exposures that carry the *stack's* -verdict on their deviant pixels (residual CRs, snowball residue, hot pixels, -persistence) in their DQ arrays. - -If image3 did not run (a single-exposure reduction has no stack to compare against), -packaging falls back to the `_cal` files and the manifest records that absence honestly -— the `source` field below tells you which family you got. - -__The Run__ -""" -print( - """ - Reducing the COSMOS-Web ring (F277W) with frame_products=True. - - With the cache warm from start_here.py this skips the MAST download but re-runs - calwebb_image3 plus the frame packaging — expect several minutes. Each exposure - yields one native-scale frame cutout under frames/. - """ -) - -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=FRAMES_OUTPUT) - -out_dir = FRAMES_OUTPUT / spec.name -frames_dir = out_dir / "frames" - -print("Frames block:", json.dumps(record["frames"], indent=2)) - -""" -The `frames` provenance block summarises the packaging: how many exposures went in, how -many chips were written (one NIRCam `_cal`/`_crf` file carries one detector, so -exposures and chips map one-to-one here), how many were skipped (no overlap with the -target), how many carry a per-frame PSF, and the headline registration residual. - -__The DQ Policy — DO_NOT_USE Only__ - -Every frame ships a `dq.fits` with the full JWST data-quality bitmask — but the -masked-by-noise policy (noise set to 1e8, data zeroed) applies **only to pixels whose -DQ has the DO_NOT_USE bit set**, plus off-chip and non-finite-error pixels. This is a -deliberate divergence from the HST frame policy, where *any* nonzero DQ bit masks the -pixel, and the reason is the ramp architecture from `step_by_step.py`: - -JWST cosmic rays are *removed during ramp fitting* — a jump corrupts groups, the slope -is refit from the clean segments, and the affected pixel emerges with a valid rate and -an informational `JUMP_DET` flag. `JUMP_DET` therefore rides *good* pixels. Masking any -nonzero DQ would throw away swathes of perfectly usable data in every frame. Only -`DO_NOT_USE` — calwebb's considered bad-pixel verdict, which image3's outlier_detection -also sets in the `_crf` products — means bad. The manifest's `dq_semantics` block spells -this out per reduction, including what bits 1 (DO_NOT_USE) and 4 (JUMP_DET) mean, so -the policy travels with the data. - -__Manifest Walk — Schema v2__ - -The frames manifest (`frames/manifest.json`) is the contract for everything in the -folder. The header fields worth knowing: - -- `version: 2` — the schema generation (v2 generalised the HST-era manifest to JWST). -- `data_units` — **"MJy/sr"**: frames keep their native surface-brightness units, - matching the mosaic (defaults-first; the packaging is loud if exposures arrive with - heterogeneous units). -- `sky_subtracted` / `sky_keyword` (per frame) — image3's skymatch *records* each - exposure's background level (`BKGLEVEL`) rather than subtracting it; frame packaging - subtracts that recorded level so every frame shares the mosaic's zero point, and - writes down the number and the keyword it came from. -- `source` — which input family the frames were cut from: the `_crf` outlier-flagged - products (normal case) or the `_cal` fallback with its recorded absence. -- `frame_cutout_shape` / `native_scale` — the per-frame geometry: native pixels, no - resampling. -- `cr_method` — for JWST: ramp-level jump detection plus image3 outlier_detection via - the `_crf` DQ; there is no deepCR model for JWST and none is needed. -""" -manifest = json.loads((frames_dir / "manifest.json").read_text()) - -print("Manifest version:", manifest["version"]) -print("Data units:", manifest["data_units"]) -print("Source family:", manifest["source"]) -print("Frame cutout shape:", manifest["frame_cutout_shape"]) -print("Native scale:", manifest["native_scale"], '"/pixel') -print("CR method:", manifest["cr_method"]) -print("DQ policy:", manifest["dq_semantics"]["policy"]) - -first = manifest["frames"][0] -print("First frame entry keys:", sorted(first.keys())) -print( - f" {first['dir']}: exptime {first['exptime']}s, sky_subtracted " - f"{first['sky_subtracted']} ({first['sky_keyword']}), " - f"{first['n_masked_pixels']} masked px, psf method {first['psf'].get('method')}" -) - -""" -__Registration Residuals and the Reliability Flag__ - -Joint frame modeling needs to know how well the frames' WCS solutions agree — if frame -astrometry disagrees at a significant fraction of a pixel, the model must fit per-frame -offsets. So the packaging *measures* the relative registration: it phase-correlates -each frame's cutout against a reference frame through the shipped WCS and records the -residual, per frame, in the manifest's `registration` blocks (alongside the header's -absolute-catalog metadata — two different things the `registration_note` carefully -distinguishes). - -The JWST validation on this very field taught the measurement some humility, and the -result is **schema v2's reliability flag**. COSMOS-Web's dither pattern routinely puts -the ring near a detector *edge*, so some frame cutouts are mostly off-chip mask. Phase -correlation between two mostly-masked cutouts locks onto the mask geometry, not the -sky — producing spectacular ~200-pixel fake "residuals" that would terrify anyone who -read them as astrometry. The fix, recorded per frame: - -- the reference is the best-covered frame; -- any pair where more than 20% of pixels are masked is flagged - `residual_reliable: false`; -- the headline `max_registration_residual_px` is computed over *reliable* pairs only — - and when no clean pair exists it is an honest **null**, meaning "unmeasured", never a - mask artifact dressed up as a shift. - -The measurement floor is ~0.1–0.3 px where masked pixels bite the source, so sub-0.1 px -values are consistent with zero. Practical stance for modeling: treat the shifts as -known when residuals are far below your modeling scale; otherwise free per-frame -(dy, dx) offsets with priors of the recorded width. -""" -print("Headline max registration residual (native px):", manifest["max_registration_residual_px"]) -for entry in manifest["frames"]: - reg = entry["registration"] - print( - f" {entry['dir']}: reliable={reg.get('residual_reliable')}, " - f"residual=({reg.get('residual_dy_px')}, {reg.get('residual_dx_px')})" - ) - -""" -__Per-Frame PSFs and the STPSF Fallback__ - -Each frame gets its own PSF attempt: the Tier-1 ePSF machinery runs on the frame's own -stars (at native sampling, DQ-patching only DO_NOT_USE pixels — the same policy as the -data). A lens cutout is a small field, though, and a single frame often lacks the -stars a mosaic accumulates; a star-poor frame is a *recorded outcome*, not a fatal -error. - -This is where JWST has a card HST never did: **Tier 2b**. When the frame's star field -cannot support an ePSF, `stpsf` (the STScI optical model, formerly WebbPSF) is -evaluated at that frame's detector and target position, and the detector-sampled, -geometric-distortion-included kernel (`DET_DIST`) ships instead — keeping every frame -modelable. The literature caveat travels in the diagnostics verbatim (empirical PSFs -are consistently preferred over models for decomposition work — see `psf.py`), so a -model-PSF frame is always identifiable. If `stpsf` is not installed, that too is a -recorded outcome, and the frame ships without a PSF. - -Note one subtlety for SW work: a single frame's ePSF is *itself undersampled* — the -sampling recovery happens across frames (the sub-pixel dither phases), which is exactly -the information joint frame modeling exploits. -""" -for entry in manifest["frames"]: - psf_info = entry["psf"] - print(f" {entry['dir']}: psf method = {psf_info.get('method')}") - if "caveat" in psf_info: - print(f" caveat: {psf_info['caveat'][:80]}...") - -""" -__Frame-Level Caveats__ - -Frames trade the mosaic's approximations for the mosaic's *protections*. Artifacts that -mosaic-level processing mitigates arrive at the frame level unmitigated: 1/f banding -(per-amplifier stripes along the slow-read axis), wisps and any snowball residue that -escaped the ramp flags land in your frames raw, because the corrections that remove -them (where teams apply them at all) act on or across the stack. The manifest carries -this honesty; your modeling error budget should too. For routine extended-source work -the mosaic remains the default — frames are the precision option, most compelling for -undersampled SW data and substructure/shear-grade measurements. - -__Loading a Frame in PyAutoLens__ - -Each frame folder is itself a valid `al.Imaging` dataset at the native pixel scale — -`data.fits` + `noise_map.fits` (+ `psf.fits` where viable). Joint fitting then means -one `Analysis` per frame, summed — with the mass model shared and, if the registration -residuals warrant, per-frame offsets free. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "PyAutoLens is not installed (pip install autolens), so the frame-loading demo " - "is skipped — the frame products above are complete." - ) - -if al is not None: - loadable = [e for e in manifest["frames"] if e["psf"].get("method") not in (None, "none")] - if loadable: - entry = loadable[0] - frame_dir = frames_dir / entry["dir"] - frame_dataset = al.Imaging.from_fits( - data_path=frame_dir / "data.fits", - noise_map_path=frame_dir / "noise_map.fits", - psf_path=frame_dir / "psf.fits", - pixel_scales=manifest["native_scale"], # native pixels — no resampling anywhere - ) - print( - f"Loaded frame {entry['dir']} as al.Imaging: shape " - f"{frame_dataset.data.shape_native} at {manifest['native_scale']}\"/pixel." - ) - -""" -Finally, a look at the frames themselves — the same ring, once per exposure, each at -native sampling with its own noise and DQ. -""" -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -entries = manifest["frames"][:4] -fig, axes = plt.subplots(1, max(len(entries), 1), figsize=(4.5 * max(len(entries), 1), 4.5)) -axes = np.atleast_1d(axes) - -for ax, entry in zip(axes, entries): - frame_data = fits.getdata(frames_dir / entry["dir"] / "data.fits").astype(float) - scale = np.nanpercentile(frame_data, 99.5) - ax.imshow(np.arcsinh(frame_data / (0.05 * max(scale, 1e-12))), origin="lower", cmap="magma") - ax.set_title(entry["dir"], fontsize=8) - ax.set_xticks([]) - ax.set_yticks([]) - -fig.tight_layout() -plot_path = plot_dir / "individual_frames.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"Frame gallery saved to: {plot_path.resolve()}") - -""" -__Wrap Up__ - -You now have the ring as the mosaic *and* as its constituent native frames: `_crf` -cutouts carrying the stack's outlier verdicts, masked by the DO_NOT_USE-only policy, -with per-frame skies, PSFs (empirical where possible, flagged STPSF where not), -measured relative registration with an honest reliability flag — and uncorrelated -noise with no Casertano R anywhere, because nothing was resampled. - -The following locations of the workspace are good places to checkout next: - -- `scripts/jwst_nircam/psf.py`: the PSF tiers behind the per-frame kernels. -- `scripts/hst_acs/individual.py`: the HST frame-products sibling — any-bit DQ policy, deepCR, MDRIZSKY. -- `scripts/jwst_nircam/step_by_step.py`: where the `_crf` files come from in the image3 chain. -- `scripts/guides/noise_maps.py`: why uncorrelated frame noise makes the joint likelihood exact. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/jwst_nircam/multi_band.py b/workspace/scripts/jwst_nircam/multi_band.py deleted file mode 100644 index 9e259c0..0000000 --- a/workspace/scripts/jwst_nircam/multi_band.py +++ /dev/null @@ -1,294 +0,0 @@ -""" -JWST NIRCam: Multi Band -======================= - -NIRCam never observes in one filter. Its dichroic splits the beam so the short-wavelength -(SW) and long-wavelength (LW) channels expose *simultaneously*, and surveys like -COSMOS-Web deliver every field in four bands: F115W and F150W (SW) plus F277W and F444W -(LW). For strong lensing this is a gift — the lens galaxy and the lensed source usually -have very different colours, so multi-band imaging separates their light far better than -any single band can, and **PyAutoLens** can fit all four bands simultaneously. - -This script reduces the COSMOS-Web ring in all four bands with **PyAutoReduce**, -assembles the results into a single multi-wavelength dataset folder, and then does the -cross-band audit the parity stance demands: are the internal closures (weight -uniformity, the sky/ERR consistency, the correlated-noise factor) consistent from band -to band? - -Each band is a full calwebb_image3 run, so the first pass takes a while — budget an hour -or more on a cold cache. Bands already reduced by `start_here.py` / `step_by_step.py` -(F277W) re-use their caches. - -__Contents__ - -- **Imports:** Import **PyAutoReduce** and the supporting libraries. -- **Paths:** Anchor the cache and output folders to the workspace root. -- **SW vs LW:** Two channels, two adapters, two pixel scales — and the undersampling trade-off between them. -- **Band Specs:** One `TargetSpec` per band, routed through `nircam_adapter_for_filter`. -- **The Four Reductions:** Loop the bands through `reduce_target`. -- **Multi-Wavelength Dataset Layout:** Assemble a `wavebands//` folder tree for multi-band modeling. -- **Cross-Band Consistency:** Compare weight uniformity, R and the sky/ERR closure across bands. -- **Plots:** The ring in four bands, side by side. -- **Modeling the Multi-Band Dataset:** Where the dataset goes next in **PyAutoLens**. -- **Wrap Up:** Where to go next. - -__Imports__ -""" -from pathlib import Path -import json -import shutil - -import matplotlib.pyplot as plt -import numpy as np - -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target -from autoreduce.instruments import nircam_adapter_for_filter - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: its combine step -changes the working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -""" -__SW vs LW__ - -The channel split is not just plumbing — it changes the character of the data: - -- **SW (F115W, F150W):** native 0.031"/pixel, reduced onto a 0.03"/pixel grid. Twice - the spatial resolution of LW, which for a ~2" Einstein ring means twice the number - of resolution elements across the arcs — more constraining power on the source and - the mass model. The catch: below ~2 microns the NIRCam PSF is *undersampled* — its - FWHM spans less than two native pixels (see the JDox NIRCam imaging pages, - https://jwst-docs.stsci.edu). Resampling dithered exposures onto the finer 0.03" - grid partially recovers the undersampled information, but the mosaic PSF is harder - to characterise (the full story is in `psf.py`). - -- **LW (F277W, F444W):** native 0.063"/pixel, reduced onto 0.06"/pixel. Beyond ~2 - microns the telescope is diffraction-limited (Strehl ~0.8) and the PSF is - well-sampled — cleaner PSF systematics at the price of coarser sampling. Red bands - also favour the lens galaxy's old stellar population and any dusty, high-redshift - source emission. - -For lens modeling the practical upshot: the SW bands carry the sharpest constraints -but the touchiest PSFs; the LW bands are the robust workhorses. Fitting all four -simultaneously gets you both. Depth is not the differentiator — COSMOS-Web reaches -5-sigma point-source depths of roughly 26.7–28.3 AB across its four filters -(Casey et al. 2023; reduction details in Franco et al., -https://arxiv.org/abs/2506.03256) — so the choice of which bands to lean on is driven -by resolution, PSF behaviour and the colours of the deflector and source, not by -signal-to-noise alone. - -The two channels also mean two cutout shapes for the *same* sky area: at 0.03"/pixel a -~12.5" field is 419 pixels across, at 0.06"/pixel it is 209. These shapes match the -COSMOS-Web ring demo dataset convention, so every band covers the identical footprint. - -__Band Specs__ - -One spec per band. `nircam_adapter_for_filter` does the channel routing — F115W/F150W -land on `nircam_sw`, F277W/F444W on `nircam_lw` — from lookup tables covering the full -NIRCam filter complement (wide, medium and narrow bands; the SW/LW boundary sits at -2.4 microns). A filter name the tables do not recognise raises a `KeyError` immediately, -before any download — misrouting a filter to the wrong channel would silently produce a -mosaic at the wrong pixel scale, so the routing is loud by design. -""" -RA, DEC = 150.10048, 1.89301 # the COSMOS-Web ring (Mercier et al. 2024, arXiv:2309.15986) - -BANDS = ("F115W", "F150W", "F277W", "F444W") - - -def spec_for(band: str) -> TargetSpec: - adapter = nircam_adapter_for_filter(band) - shape = (419, 419) if adapter.key == "nircam_sw" else (209, 209) # same ~12.5" footprint per channel - return TargetSpec( - name=f"cosmos_web_ring_{band.lower()}", # one output folder per band - ra=RA, # target right ascension in degrees (J2000) - dec=DEC, # target declination in degrees (J2000) - instrument=adapter.key, # "nircam_sw" or "nircam_lw", routed from the filter - filter_name=band, # the NIRCam filter for this reduction - proposal_ids=("1727",), # COSMOS-Web only — other programs would change depth and noise - final_scale=adapter.recommended_final_scale, # SW 0.03" / LW 0.06" — the COSMOS-Web convention - final_pixfrac=1.0, # full drizzle drop, matching the COSMOS-Web mosaics - cutout_shape=shape, # SW (419, 419) / LW (209, 209) - ) - - -""" -__The Four Reductions__ - -Loop the bands. Each iteration is a complete pipeline run — acquire, calwebb_image3, -noise, PSF, package — and each band's exposures cache independently (SW and LW are -different files even for simultaneous observations). -""" -records = {} - -for band in BANDS: - spec = spec_for(band) - print( - f""" - [{band}] starting reduction ({spec.instrument}, {spec.final_scale}\"/pixel, - cutout {spec.cutout_shape}). A cold-cache band downloads its _cal exposures from - MAST and runs calwebb_image3 — expect tens of minutes per band; warm caches - (e.g. F277W after start_here.py) are much faster. - """ - ) - records[band] = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - print(f"[{band}] done -> {OUTPUT_ROOT / spec.name}") - -""" -__Multi-Wavelength Dataset Layout__ - -Each band reduced into its own `output/cosmos_web_ring_/` folder. For multi-band -modeling it is more convenient to gather them under one dataset root with a -`wavebands//` sub-folder per filter — the same layout the COSMOS-Web ring demo -dataset uses. We copy the four modeling products (plus the provenance record) per band; -nothing is modified, so each `reduction.json` stays valid for its band. -""" -dataset_dir = OUTPUT_ROOT / "cosmos_web_ring_multi_band" - -PRODUCTS = ("data.fits", "noise_map.fits", "psf.fits", "psf_full.fits", "reduction.json") - -for band in BANDS: - band_src = OUTPUT_ROOT / f"cosmos_web_ring_{band.lower()}" - band_dst = dataset_dir / "wavebands" / band - band_dst.mkdir(parents=True, exist_ok=True) - for product in PRODUCTS: - shutil.copy2(band_src / product, band_dst / product) - -print(f"Multi-wavelength dataset assembled at: {dataset_dir.resolve()}") -for path in sorted(dataset_dir.rglob("*.fits")): - print(" ", path.relative_to(dataset_dir)) - -""" -__Cross-Band Consistency__ - -The parity stance (`start_here.py`) accepted that absolute agreement with the bespoke -COSMOS-Web team mosaics is not the bar — internal consistency is. That claim has a -cross-band leg: whatever global offsets exist between this reduction and the team's -(calibration vintage, background treatment) should behave *consistently* from band to -band, and each band's own closures should hold independently. So we tabulate, per band: - -- `n_exposures` — how many exposures the footprint filter admitted. -- `weight_uniformity_cutout` — depth uniformity over the shipped cutout (policy < 0.2). -- `correlated_noise_factor` — the Casertano R; nearly identical for the two channels - because both use pixfrac 1.0 at scale ratios just under 1. -- `sky_over_err_floor` — the blank-sky vs propagated-ERR closure; each band should sit - near 1 independently, and a band that strays flags a band-specific problem (residual - 1/f banding is stronger in some filters, for example). -""" -consistency = {} - -for band, record in records.items(): - consistency[band] = { - "n_exposures": record["acquire"]["n_exposures"], - "pixel_scale": record["package"]["pixel_scale"], - "weight_uniformity_cutout": record["drizzle"]["weight_uniformity_cutout"], - "correlated_noise_factor": record["noise"]["correlated_noise_factor"], - "sky_over_err_floor": record["noise"]["sky_over_err_floor"], - "psf_n_stars": record["psf"].get("n_stars_used"), - } - -print(json.dumps(consistency, indent=2)) - -summary_path = dataset_dir / "cross_band_consistency.json" -summary_path.write_text(json.dumps(consistency, indent=2)) -print(f"Cross-band consistency table saved to: {summary_path.resolve()}") - -""" -Read this table the way a referee would: uniform weights in every band (no band with -ragged coverage over the ring), R values that match the dial settings, and sky/ERR -closures near 1 across the board. If one band misbehaves, that band's `reduction.json` -has the per-stage evidence to chase it down. - -__Plots__ - -The ring in four bands. The colour gradient you see — the arcs brightening relative to -the deflector towards the blue bands — is exactly the lens/source colour separation -that makes multi-band fitting so much more constraining than single-band. -""" -plot_dir = dataset_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(1, 4, figsize=(18, 5)) - -for ax, band in zip(axes, BANDS): - data = fits.getdata(dataset_dir / "wavebands" / band / "data.fits").astype(float) - scale = np.nanpercentile(data, 99.5) - ax.imshow(np.arcsinh(data / (0.05 * scale)), origin="lower", cmap="magma") - ax.set_title(f"{band} ({records[band]['package']['pixel_scale']}\"/pix)") - ax.set_xticks([]) - ax.set_yticks([]) - -fig.tight_layout() -plot_path = plot_dir / "multi_band_ring.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"Four-band plot saved to: {plot_path.resolve()}") - -""" -__Modeling the Multi-Band Dataset__ - -The `wavebands/` tree above is a ready-made multi-wavelength dataset. In **PyAutoLens**, -each band loads as its own `al.Imaging` (with its own pixel scale and PSF), and the -`autolens_workspace/scripts/multi_dataset/` examples show how to fit them -simultaneously — sharing the mass model across bands while letting the lens and source -light vary with wavelength. That is the analysis COWLS ran on more than 100 lens -candidates in these same four bands (Nightingale et al. 2025, -https://arxiv.org/abs/2503.08777). - -Remember the units caveat from `start_here.py` applies per band: every band is in -MJy/sr, so fitted intensities are surface brightnesses, and cross-band flux ratios -(colours) come from integrating model images and converting through each band's pixel -solid angle. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "PyAutoLens is not installed (pip install autolens), so the loading demo is " - "skipped — the multi-band dataset above is complete." - ) - -if al is not None: - datasets = {} - for band in BANDS: - band_dir = dataset_dir / "wavebands" / band - band_record = json.loads((band_dir / "reduction.json").read_text()) - datasets[band] = al.Imaging.from_fits( - data_path=band_dir / "data.fits", - noise_map_path=band_dir / "noise_map.fits", - psf_path=band_dir / "psf.fits", - pixel_scales=band_record["package"]["pixel_scale"], - ) - print( - f"[{band}] loaded al.Imaging: shape {datasets[band].data.shape_native}, " - f"pixel scale {datasets[band].pixel_scales}" - ) - -""" -__Wrap Up__ - -Four bands, four independent pipeline runs, one multi-wavelength dataset — with a -cross-band consistency table certifying that every band's internal closures hold and -that the channel conventions (SW 0.03"/419px, LW 0.06"/209px) cover the same footprint. - -The following locations of the workspace are good places to checkout next: - -- `autolens_workspace/scripts/multi_dataset/`: simultaneous multi-wavelength lens modeling — the consumer of this dataset. -- `scripts/jwst_nircam/psf.py`: why the SW bands' PSFs need more care than LW — undersampling and the ePSF story. -- `scripts/jwst_nircam/individual.py`: frame products, where the SW undersampling argument becomes decisive. -- `scripts/guides/output_contract.py`: the per-band product contract in detail. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/jwst_nircam/psf.py b/workspace/scripts/jwst_nircam/psf.py deleted file mode 100644 index ff69c0b..0000000 --- a/workspace/scripts/jwst_nircam/psf.py +++ /dev/null @@ -1,352 +0,0 @@ -""" -JWST NIRCam: PSF -================ - -No single ingredient of a lens dataset repays care like the PSF. Strong lens modeling -convolves every trial image with it, so PSF errors masquerade as source structure, bias -the inferred mass model, and — for the quasar and substructure work JWST excels at — can -dominate the entire error budget. This script is the JWST PSF story in **PyAutoReduce**: -how the PSF is measured, when each backend wins, and which diagnostics tell you whether -to trust what you got. - -The anchor here is not a lens but a globular cluster: M92 (NGC 6341), observed by -program 1334 — the JWST NIRCam astrometric and flux calibration field. A genuinely -stellar, deep field is the right place to *validate* PSF estimation, because it offers -hundreds of isolated point sources where a lens field offers a handful. We reduce it in -F150W (SW — the undersampled regime) and F277W (LW — well-sampled), with both PSF -backends, and read the verdicts out of the diagnostics. - -__Contents__ - -- **Why the PSF Matters for Lensing:** Convolution errors become science errors. -- **Imports:** Import **PyAutoReduce** and the supporting libraries. -- **Paths:** Anchor the cache and output folders to the workspace root. -- **Undersampling:** The SW channel's defining PSF problem, and why 2 microns is the dividing line. -- **The ePSF Lineage:** Anderson & King's effective PSF, its arrival on NIRCam, and library ePSFs. -- **The M92 Reductions:** Reduce the calibration field in F150W and F277W with the photutils Tier-1 ePSF. -- **Diagnostics Walk:** method, star counts, FWHM, star source pass — the record every PSF ships with. -- **psf vs psf_full and the Drizzled-PSF Invariant:** Two kernels, one resampling history. -- **STARRED — Tier 1b:** The super-sampled alternative, and the M92 regime rule with its undersampled flag. -- **Model PSFs — STPSF as Tier 2b:** Why pure model PSFs are disfavoured, and where **PyAutoReduce** still uses them. -- **Spatially-Varying PSFs — the Roadmap:** PSFEx/ShOpt-style Tier 2 is planned, not implemented. -- **Plots:** The measured kernels, compact and full, in both bands. -- **Wrap Up:** Where to go next. - -__Why the PSF Matters for Lensing__ - -A lens model predicts the sky, but you observe the sky *convolved with the PSF*. During -fitting the model image is convolved with `psf.fits` before comparison with the data, so: - -- An underestimated PSF width leaves compact residuals at the arcs that a flexible - source model will happily absorb as spurious structure. -- PSF wing errors leak deflector light into the arc region, biasing the source - reconstruction and — through the source, the mass model. -- For lensed quasars and substructure searches, percent-level PSF errors are the - systematics floor. - -This is why **PyAutoReduce** treats the PSF as a first-class product with its own -diagnostics block, not a footnote. - -__Imports__ -""" -from dataclasses import replace -from pathlib import Path -import json - -import matplotlib.pyplot as plt -import numpy as np - -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target -from autoreduce.instruments import nircam_adapter_for_filter - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: its combine step -changes the working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -""" -__Undersampling__ - -The JWST optical PSF shrinks with wavelength (diffraction), but the NIRCam pixels do -not. The result, documented on the JDox NIRCam imaging pages -(https://jwst-docs.stsci.edu): beyond ~2 microns the telescope delivers a -diffraction-limited, well-sampled PSF (Strehl ~0.8) on the LW channel's 0.063" pixels — -but *below* ~2 microns, on the SW channel's 0.031" pixels, the PSF FWHM spans less than -two native pixels. The image on the detector no longer Nyquist-samples the PSF. - -Undersampling is the defining problem of SW PSF work. A star's measured shape depends -on where its centre lands within a pixel; naive stacking of stars blurs them by their -subpixel phases; and a mosaic only partially recovers the lost information through the -subpixel dither pattern (this is exactly why STScI designed those patterns, and why -`resample` onto a finer 0.03" grid helps but cannot fully cure it). Every method below -is ultimately a strategy for coping with this fact. - -__The ePSF Lineage__ - -The standard answer is the *effective PSF* (ePSF) of Anderson & King (2000, PASP 112, -1360): model the PSF as seen *by the pixel grid* — the optical PSF convolved with the -pixel response — on an oversampled grid, built iteratively from many stars at different -subpixel phases. The subpixel dithers that sample the star field become the very thing -that beats undersampling. - -On JWST this lineage arrived immediately: Nardiello et al. (2022, MNRAS 517, 484) built -the first NIRCam ePSFs — 5x5 grids of 4x-oversampled library ePSFs per filter and -detector — and the approach extended to NIRISS (Libralato et al. 2023) and MIRI -(Libralato et al. 2024), alongside Anderson's STDPSF library ePSFs for NIRCam. The -existence of *library* ePSFs matters for star-poor fields; **PyAutoReduce**'s Tier 1 -instead builds the ePSF from the mosaic's own stars via the photutils implementation of -the same algorithm, because a PSF measured on your mosaic has automatically been through -your mosaic's resampling — the invariant discussed below. - -One JWST-specific wrinkle in star selection: HST star finding rejects stars near the -full well, but a peak cut in counts is meaningless in MJy/sr surface-brightness units — -and saturated cores arrive from level 2 already blanked. The JWST star finder is -therefore NaN-masked and applies **no peak cut**. - -__The M92 Reductions__ - -Now reduce the calibration field in both channels with the default photutils Tier-1 -backend. The 501x501 cutout (~15" in both channels) spans M92's density gradient, -giving the star finder a rich sample. -""" -RA, DEC = 259.28079, 43.13594 # M92 (NGC 6341) cluster centre - -PSF_BANDS = ("F150W", "F277W") # SW undersampled / LW well-sampled — the two regimes - - -def m92_spec(band: str, psf_backend: str = "epsf") -> TargetSpec: - adapter = nircam_adapter_for_filter(band) - return TargetSpec( - name=f"m92_{band.lower()}", # one output folder per band - ra=RA, # M92 cluster centre, right ascension in degrees - dec=DEC, # M92 cluster centre, declination in degrees - instrument=adapter.key, # "nircam_sw" (F150W) or "nircam_lw" (F277W) - filter_name=band, # the NIRCam filter to reduce - proposal_ids=("1334",), # the JWST NIRCam astrometric/flux calibration program - final_scale=adapter.recommended_final_scale, # SW 0.03" / LW 0.06" - final_pixfrac=1.0, # full drizzle drop - cutout_shape=(501, 501), # ~15" — a rich stellar sample across the density gradient - psf_backend=psf_backend, # "epsf" (photutils Tier 1) | "starred" (Tier 1b) - ) - - -records = {} - -for band in PSF_BANDS: - spec = m92_spec(band) - print( - f""" - [{band}] reducing M92 (program 1334, {spec.instrument}). First run downloads the - _cal exposures from MAST and runs calwebb_image3 — expect tens of minutes per band; - reruns re-use the cache. - """ - ) - records[band] = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - print(f"[{band}] done -> {OUTPUT_ROOT / spec.name}") - -""" -__Diagnostics Walk__ - -Every PSF ships with a diagnostics block in `reduction.json` — the `psf` block — so no -PSF is ever a bare array you must take on faith. The keys to read every time: - -- `method` — which tier produced the kernel ("epsf-tier1" here). -- `n_stars_used` — how many stars survived selection and fed the fit. The Tier-1 - builder requires at least 8 and fails loudly below that; on M92 expect far more. -- `fwhm_pix` — the fitted kernel's FWHM in *output* pixels. Compare against the - expectation for the band: F150W's PSF is ~1.7 native pixels, so an undersampled ~1.7 - on the 0.03" grid; F277W is comfortably above 2. -- `star_source_pass` — which mosaic pass fed the star finder. On JWST this is always - the science mosaic (the CR story lives in the ramps, so no CR-avoiding second - drizzle is ever needed — the reason is recorded rather than assumed). -""" -for band in PSF_BANDS: - print(f"[{band}] psf block:") - print(json.dumps(records[band]["psf"], indent=2)) - -""" -__psf vs psf_full and the Drizzled-PSF Invariant__ - -Each reduction ships two kernels: - -- `psf.fits` (21x21 by default) — compact, sized for the convolution inside model - fitting, where kernel size costs runtime on every likelihood evaluation. -- `psf_full.fits` (61x61) — the extended wings, for flux-sensitive work (aperture - corrections, quasar deblending, checking how much deflector light the compact kernel - ignores). - -Both are odd-shaped and unit-normalised, and both obey the **drizzled-PSF invariant**: -the delivered PSF is measured *from the mosaic*, so it has passed through the identical -resampling — same pixel scale, pixfrac, kernel, rotation — as the data. A PSF from any -other source (a model PSF evaluated on a detector grid, a library ePSF at native -sampling) must be pushed through the same resampling before it is comparable; skipping -that step is one of the classic silent PSF errors in lens modeling. - -__STARRED — Tier 1b__ - -**PyAutoReduce** offers a second empirical backend: STARRED (Millon, Michalewicz et al. -2024, AJ, https://arxiv.org/abs/2402.08725), a JAX-based, wavelet-regularised joint PSF -reconstruction developed in the COSMOGRAIL lensed-quasar tradition and demonstrated on -JWST imaging. It fits all stars simultaneously for a super-sampled PSF with a -starlet-regularised residual channel — a higher-fidelity alternative to the photutils -ePSF for demanding quasar/AGN-grade work. It is optional (GPL-licensed, JAX-dependent): -install the `starred` extra to enable it, and if it is missing the backend raises -loudly rather than silently falling back. - -The M92 validation is exactly why this field is in the workspace, and it produced a -clean **regime rule**: - -- **F277W (LW, well-sampled): STARRED wins.** With the PSF resolved by the pixel grid, - the super-sampled joint fit extracts more information than the classic ePSF. -- **F150W (SW, undersampled): STARRED loses.** The reconstruction *broadens* — the - super-sampled model cannot be constrained by undersampled data and the - regularisation fills the gap with width. - -The pipeline encodes the rule as a diagnostic: any STARRED PSF whose fitted FWHM falls -below 1.6 output pixels is flagged `undersampled` in its diagnostics block, warning you -that you are in the regime where the reconstruction broadens. Practical guidance: -STARRED for well-sampled or crowded fields (LW; also WFC3/UVIS — see the HST leg of -this validation in `hst_wfc3_uvis/psf.py`), photutils Tier 1 for the undersampled SW. - -The code below re-runs both bands with `psf_backend="starred"` — same specs, same -caches, separate output tree — and prints each verdict. It is wrapped in a try/except -so the script survives a missing optional extra. -""" -starred_records = {} - -for band in PSF_BANDS: - spec = replace(m92_spec(band), psf_backend="starred") # frozen dataclass -> replace, not mutate - print(f"[{band}] re-running with the STARRED Tier-1b backend (same cache)...") - try: - starred_records[band] = reduce_target( - spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT / "starred" - ) - except Exception as error: - print( - f"[{band}] STARRED backend unavailable or failed ({error}). Install the " - "optional extra (pip install 'autoreduce[starred]') to run Tier 1b; the " - "Tier-1 reductions above are complete regardless." - ) - break - -for band, record in starred_records.items(): - psf_block = record["psf"] - verdict = "UNDERSAMPLED — expect broadening" if psf_block.get("undersampled") else "well-sampled" - print( - f"[{band}] starred: fwhm {psf_block.get('fwhm_pix')} px, " - f"n_stars {psf_block.get('n_stars_used')} -> {verdict}" - ) - -""" -__Model PSFs — STPSF as Tier 2b__ - -JWST has an outstanding optical model: STPSF (formerly WebbPSF; -https://stpsf.readthedocs.io/; Perrin et al. 2014, Proc. SPIE 9143) simulates the PSF -from the measured, time-dependent wavefront. So why not just use it? - -Because the literature keeps finding that **empirical beats model** for photometric and -morphological work. Zhuang et al. (2024, -https://iopscience.iop.org/article/10.3847/1538-4357/ad1517) find pure STPSF models -consistently disfavoured against empirical PSFs for AGN host decomposition — the model -lacks charge diffusion, interpixel capacitance, the source's spectral energy -distribution and residual wavefront error, and a mosaic PSF is additionally shaped by -the drizzle kernel and dither pattern. A model PSF can only compete after being pushed -through the same resampling as the data (the `spike` tool, published in JOSS, exists -for exactly this), and even then it is the fallback, not the first choice. - -**PyAutoReduce** therefore uses STPSF only as **Tier 2b, for frame products**: when a -single native frame's star field cannot support a per-frame ePSF, `stpsf` evaluates the -model at that frame's detector and target position and ships the detector-sampled, -distortion-included kernel — keeping the frame modelable rather than shipping nothing. -Every such kernel carries the literature caveat verbatim in its diagnostics ("model-PSF -fallback — the JWST literature consistently prefers empirical PSFs for decomposition"), -so a model PSF is flagged, never silent. The mosaic path never uses it. See -`individual.py` for this fallback in action. - -__Spatially-Varying PSFs — the Roadmap__ - -A single ePSF is a *position-independent* model, and the NIRCam PSF does vary across -the field. Zhuang & Shen (2024, https://arxiv.org/abs/2304.13776) quantified it across -8 filters: the spatial FWHM variation shrinks strongly with wavelength, from ~20% max -(~5% RMS) at F070W to ~3% max (~0.6% RMS) at F444W — and among SWarp, photutils and -PSFEx they rank PSFEx's polynomial spatially-varying model best. COSMOS-Web's own -weak-lensing pipeline characterises the PSF with ShOpt (Berman & McCleary, -https://arxiv.org/abs/2401.11625), benchmarked against PSFEx and PIFF. - -This shapes **PyAutoReduce**'s JWST PSF tiering: - -- **Tier 1 (implemented, default):** single mosaic-star ePSF — adequate for - lens-galaxy work in the **LW** bands, where spatial variation is ~1% RMS. -- **Tier 1b (implemented, optional):** STARRED, per the regime rule above. -- **Tier 2 (roadmap — NOT implemented):** a spatially-varying empirical model - (PSFEx/ShOpt-style polynomial) evaluated at the lens position — the quality upgrade - path for the **SW** bands (~5% RMS variation) and any weak-lensing-grade use. Do not - look for it in the current release; this paragraph exists so you know the limitation - a single SW ePSF carries. -- **Tier 2b (implemented, frame products only):** STPSF, flagged, as above. -- **Not a tier:** target-based PSF reconstruction (fitting the PSF jointly with the - lensed quasar images) — that is a modeling-stage technique, out of reduction scope. - -__Plots__ - -The measured kernels: compact and full, both bands, log-stretched to show the wings and -the first Airy structure (crisp in F277W, softened by undersampling + resampling in -F150W). -""" -plot_dir = OUTPUT_ROOT / "m92_psf_plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(2, 2, figsize=(10, 10)) - -for col, band in enumerate(PSF_BANDS): - out_dir = OUTPUT_ROOT / f"m92_{band.lower()}" - for row, product in enumerate(("psf.fits", "psf_full.fits")): - kernel = fits.getdata(out_dir / product).astype(float) - axes[row, col].imshow( - np.log10(np.clip(kernel / kernel.max(), 1e-6, None)), - origin="lower", - cmap="magma", - vmin=-5, - vmax=0, - ) - axes[row, col].set_title(f"{band} {product} ({kernel.shape[0]}x{kernel.shape[1]})") - axes[row, col].set_xticks([]) - axes[row, col].set_yticks([]) - -fig.tight_layout() -plot_path = plot_dir / "m92_psf_kernels.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"PSF kernel plot saved to: {plot_path.resolve()}") - -""" -__Wrap Up__ - -The JWST PSF story in one paragraph: build it empirically from the mosaic's own stars -(Tier 1 ePSF, no peak cut in surface-brightness units), so the drizzled-PSF invariant -holds by construction; upgrade to STARRED where the PSF is well-sampled (LW) and avoid -it where it is not (SW — the 1.6-pixel undersampled flag); treat model PSFs as a -flagged frame-level fallback, never the mosaic default; and know that a single ePSF on -SW carries a ~5%-RMS spatial-variation limitation until the spatially-varying Tier 2 -lands. - -The following locations of the workspace are good places to checkout next: - -- `scripts/jwst_nircam/individual.py`: per-frame ePSFs and the STPSF Tier-2b fallback in action. -- `scripts/hst_acs/psf.py`: the HST PSF story — star passes, tiers and the same invariant on AstroDrizzle. -- `scripts/jwst_nircam/multi_band.py`: how PSF quality differs across the four COSMOS-Web bands. -- `scripts/guides/output_contract.py`: where psf.fits and psf_full.fits sit in the product contract. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/jwst_nircam/simulator.py b/workspace/scripts/jwst_nircam/simulator.py deleted file mode 100644 index a5d28c5..0000000 --- a/workspace/scripts/jwst_nircam/simulator.py +++ /dev/null @@ -1,333 +0,0 @@ -""" -JWST NIRCam: Simulator -====================== - -How do you know a reduction pipeline conserves flux? You feed it a source whose -brightness you know *exactly*, run the identical pipeline, and check what comes out. -This script does that on JWST data with **PyAutoReduce**'s injection mode: a synthetic -lensed-arc image, built in pure numpy with known total flux in janskys, is injected -into the real COSMOS-Web F150W `_cal` exposures — each frame receiving the source -through its own WCS, its own photometric calibration and its own PSF — and the full -calwebb_image3 reduction then runs twice, clean and injected. Differencing the two -mosaics recovers the source, and the recovered flux tests the whole chain end to end. - -This is also the honest way to *simulate* JWST lens data. Injection into real frames -inherits everything a from-scratch simulator must model and always gets slightly wrong: -real cosmic-ray statistics, the real sky, real bad pixels, the real dither geometry, -real PSF wings and real correlated noise downstream. Injection into real *reduced* -imaging is the established pattern in lensing (the COWLS forecast-vs-data comparisons, -https://arxiv.org/abs/2503.08785, are built on synthetic lenses in real image -statistics); pushing the injection one level deeper — into the calibrated exposures, -*before* the combine stage — has no lens-specific published example we know of at the -uncal/ramp level, and pre-launch ramp simulators are not the practical route today. -Injecting at the `_cal` level, through the real pipeline, is the gap **PyAutoReduce** -fills: everything downstream of the injection point is real. - -__Contents__ - -- **Imports:** Import **PyAutoReduce** and the supporting libraries. -- **Paths:** Anchor the cache and output folders to the workspace root. -- **The Input Image:** A pure-numpy lensed arc in Jy per pixel — the formula, the contract, the total flux. -- **The Unit Chain:** Jy per pixel through each frame's own PHOTMJSR and PIXAR_SR — flux-exact in the mean. -- **Noise Bookkeeping:** The source's Poisson noise, the e_per_dn disclosure, and the ERR update before image3. -- **Clean and Injected Reductions:** The same spec twice, sharing one exposure cache. -- **The Inject Provenance Block:** What the reduction record says was done to the data. -- **Recovery:** Difference the mosaics, integrate in an aperture, convert MJy/sr to Jy, compare to truth. -- **Plots:** Clean, injected and difference images. -- **Wrap Up:** Where to go next. - -__Imports__ -""" -from pathlib import Path -import json - -import matplotlib.pyplot as plt -import numpy as np - -from astropy.io import fits -from astropy.wcs import WCS - -from autoreduce import TargetSpec, reduce_target -from autoreduce.instruments import nircam_adapter_for_filter - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: its combine step -changes the working directory internally, so relative paths would break. - -The clean and injected reductions get separate output trees but share one exposure -cache — the cache is never mutated by injection (frames are copied to a scratch area -before the source is added), so the same downloaded `_cal` files serve both runs. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (shared by both runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target -INJECT_ROOT = OUTPUT_ROOT / "inject" # this script's clean/ and injected/ trees live here - -""" -__The Input Image__ - -The injection contract is deliberately spartan: a plain 2-D FITS image — finite, -non-negative, **not** PSF-convolved — whose pixel values are flux per pixel at a stated -pixel scale, oriented north-up, centred at the position you inject at. For JWST the -adapter sets `inject_units="Jy"`: pixel values are janskys per pixel, so the **total -source flux is simply the array sum**. No lensing library is imported to build it — -a simulated *input* should be transparent, so we write the arc as three lines of numpy. - -We build a partial Einstein arc: an exponential profile in the radial distance from a -ring of radius `r_ring`, tapered azimuthally around an angle `phi_0` — the classic -morphology of a lensed source near a fold: - - I(r, phi) ∝ exp( -|r - r_ring| / w_r ) x exp( -(phi - phi_0)^2 / (2 w_phi^2) ) - -normalised so the array sums to `flux_jy`. The input grid is finer than the detector -(0.015"/pixel) so the injector — which deposits flux through each frame's own -distortion WCS with a flux-conserving drizzle-style footprint — resolves the arc's -width properly before the frame PSF is applied. -""" -FLUX_JY = 2.0e-6 # total injected flux: 2 microJy (m_AB ~ 25.6) — bright enough to recover cleanly -INPUT_PIXEL_SCALE = 0.015 # "/pixel of the input image — finer than the SW detector -R_RING = 0.9 # arc radius in arcsec -W_R = 0.12 # radial exponential scale of the arc, arcsec -W_PHI = 0.9 # azimuthal Gaussian width, radians (~50 deg of arc) -PHI_0 = np.pi / 3.0 # arc's central position angle - -shape = (161, 161) -yy, xx = np.mgrid[0 : shape[0], 0 : shape[1]] -cy, cx = shape[0] // 2, shape[1] // 2 - -r = np.hypot(yy - cy, xx - cx) * INPUT_PIXEL_SCALE # radius from the arc centre, arcsec -phi = np.arctan2(yy - cy, xx - cx) # azimuth, radians - -arc = np.exp(-np.abs(r - R_RING) / W_R) * np.exp(-((phi - PHI_0) ** 2) / (2.0 * W_PHI**2)) -arc = FLUX_JY * arc / arc.sum() # normalise: array sum IS the total flux in Jy - -input_dir = INJECT_ROOT -input_dir.mkdir(parents=True, exist_ok=True) -input_path = input_dir / "input_arc_jy.fits" -fits.PrimaryHDU(arc.astype(np.float32)).writeto(input_path, overwrite=True) - -print(f"Input arc written to: {input_path.resolve()}") -print(f"Total input flux: {arc.sum():.3e} Jy over {shape[0]}x{shape[1]} px at {INPUT_PIXEL_SCALE}\"/px") - -""" -__The Unit Chain__ - -Here is where JWST injection earns its keep. The `_cal` frames are in MJy/sr, but their -headers carry the two numbers that connect surface brightness back to detector physics: -`PHOTMJSR` (the calibration factor MJy/sr per DN/s) and `PIXAR_SR` (the pixel solid -angle in steradians). The injector converts your Jy-per-pixel input **through each -frame's own keywords** — not a global constant — so a frame with a slightly different -calibration receives a correspondingly different DN pattern, exactly as a real source -on the sky would. - -The conversion is constructed so that the injected *mean* is flux-exact: the chain -Jy -> electrons -> DN/s -> MJy/sr algebraically reduces to dividing by -(PIXAR_SR x 1e6), with the detector gain cancelling out of the mean entirely. The gain -enters only one place — - -__Noise Bookkeeping__ - -— the *Poisson draw*. A real source arrives as photons, so the injector realises the -source in electrons and draws Poisson noise on it (seeded deterministically from -`inject_seed` and the frame filename, so reruns are bit-identical). Converting -electrons needs a gain, and NIRCam `_cal` headers do not carry a per-frame one, so the -adapter supplies a nominal `e_per_dn = 2.0`. Because the mean is gain-free, this -approximation **shapes only the width of the injected Poisson scatter**, not the -recovered flux — and the provenance discloses exactly that, recording the nominal gain -and the caveat alongside every injection. - -Equally important: the injector updates each frame's **ERR array in quadrature** with -the injected source's variance *before* calwebb_image3 runs. The variance planes then -propagate through resample like every other noise source, so the final `noise_map.fits` -of the injected reduction correctly knows the arc is there. An exposure without an ERR -array refuses injection loudly — silent noise-bookkeeping gaps are how simulations lie. - -__Clean and Injected Reductions__ - -Two runs of the identical spec — the only difference is the three `inject_*` dials on -the second. We inject 3" east of the ring so the recovered arc lands on clean sky in -the same cutout, and use F150W (SW, 0.03"/pixel, 419x419 — the same COSMOS-Web -conventions as `multi_band.py`, whose cache this shares). -""" -RA, DEC = 150.10048, 1.89301 # the COSMOS-Web ring field (Mercier et al. 2024) -band = "F150W" -adapter = nircam_adapter_for_filter(band) - -OFFSET_ARCSEC = 3.0 -inject_ra = RA + OFFSET_ARCSEC / 3600.0 # 3" offset in RA (dec ~ 1.9 deg, cos(dec) ~ 1) - -common = dict( - name=f"cosmos_web_ring_{band.lower()}", # same name as multi_band.py -> shares its exposure cache - ra=RA, # cutout stays centred on the ring; the injection lands 3" away inside it - dec=DEC, - instrument=adapter.key, # "nircam_sw" - filter_name=band, - proposal_ids=("1727",), # COSMOS-Web only - final_scale=adapter.recommended_final_scale, # 0.03"/pixel SW convention - final_pixfrac=1.0, # full drizzle drop - cutout_shape=(419, 419), # ~12.5" at 0.03"/pixel -) - -print( - """ - Run 1/2: the CLEAN reduction (no injection). With a warm cache from multi_band.py - this is a combine-only run; cold, it downloads the F150W _cal exposures first. - Expect tens of minutes per run either way — calwebb_image3 runs twice in this - script. - """ -) - -clean_record = reduce_target( - TargetSpec(**common), cache_root=CACHE_ROOT, output_root=INJECT_ROOT / "clean" -) - -print("Run 2/2: the INJECTED reduction (same spec + inject_* dials).") - -injected_record = reduce_target( - TargetSpec( - **common, - inject_image=str(input_path), # the Jy-per-pixel arc built above (absolute path) - inject_pixel_scale=INPUT_PIXEL_SCALE, # the input image's own pixel scale - inject_position=(inject_ra, DEC), # absolute (ra, dec) where the arc centre lands - inject_seed=0, # deterministic Poisson realisation (per-frame seeds derive from this + filename) - ), - cache_root=CACHE_ROOT, - output_root=INJECT_ROOT / "injected", -) - -clean_dir = INJECT_ROOT / "clean" / common["name"] -injected_dir = INJECT_ROOT / "injected" / common["name"] - -""" -__The Inject Provenance Block__ - -A semi-synthetic dataset that could pass for real data is a scientific hazard, so the -injected reduction's provenance carries an `inject` block stating exactly what was -added: the input image and its units and pixel scale, the total input flux, the -position, the PSF source used for the convolution (each frame's own Tier-1 ePSF unless -you supplied `inject_psf`), the seed, the realised total electrons, and a per-frame -list. The injected FITS frames themselves also carry INJECTED/INJIMG/INJSEED header -keys — the semi-synthetic status is stamped on every layer of the output. -""" -inject_block = injected_record["inject"] - -print("Inject block:") -print(json.dumps({k: v for k, v in inject_block.items() if k != "frames"}, indent=2)) -print(f"Frames injected: {len(inject_block['frames'])}") - -""" -__Recovery__ - -Now the test. The difference of the injected and clean mosaics isolates the arc (plus -the injected Poisson scatter); summing the difference in an aperture around the -injection position gives the recovered surface brightness, and the conversion back to -flux is the MJy/sr bookkeeping from `start_here.py`: - - flux [Jy] = sum(SB [MJy/sr]) x Omega_pixel [sr] x 1e6, Omega_pixel = (scale / 206265)^2 - -The noise prediction for the same aperture comes from the injected reduction's own -noise map, summed in quadrature — so this one number tests flux conservation through -acquisition, injection, image3, the ERR propagation and the packaging all at once. -""" -data_clean = fits.getdata(clean_dir / "data.fits").astype(float) -data_injected = fits.getdata(injected_dir / "data.fits").astype(float) -noise_injected = fits.getdata(injected_dir / "noise_map.fits").astype(float) -header = fits.getheader(injected_dir / "data.fits") - -diff = data_injected - data_clean # MJy/sr — the recovered arc - -pixel_scale = injected_record["package"]["pixel_scale"] -omega = (pixel_scale / 206265.0) ** 2 # pixel solid angle in steradians - -x_inj, y_inj = WCS(header).world_to_pixel_values(inject_ra, DEC) -yy, xx = np.mgrid[0 : diff.shape[0], 0 : diff.shape[1]] -aperture = np.hypot(yy - y_inj, xx - x_inj) * pixel_scale <= 2.0 # 2" radius around the arc - -good = aperture & (noise_injected < 1.0e7) # exclude masked-by-noise pixels from the sums - -recovered_jy = float(diff[good].sum()) * omega * 1e6 -aperture_noise_jy = float(np.sqrt((noise_injected[good] ** 2).sum())) * omega * 1e6 - -report = { - "injected_flux_jy": FLUX_JY, - "recovered_flux_jy_2arcsec": recovered_jy, - "recovery_ratio": recovered_jy / FLUX_JY, - "aperture_noise_jy": aperture_noise_jy, - "total_injected_e": inject_block["total_injected_e"], - "n_frames_injected": len(inject_block["frames"]), -} - -report_path = INJECT_ROOT / "recovery_report.json" -report_path.write_text(json.dumps(report, indent=2)) - -print(json.dumps(report, indent=2)) -print(f"Recovery report saved to: {report_path.resolve()}") - -ok = abs(report["recovery_ratio"] - 1.0) < max(0.05, 3.0 * aperture_noise_jy / FLUX_JY) -print("RECOVERY", "OK" if ok else "DISCREPANT — inspect the report and the inject block") - -""" -A recovery ratio within a few percent of 1 (or within 3x the aperture noise for faint -injections) is the pass condition — the same acceptance the **PyAutoReduce** JWST -injection validation uses. A systematic shortfall would point at flux non-conservation -somewhere in the chain (footprint clipping of the input, aperture too small for the -convolved arc, resample kernel effects); the per-frame `inject` provenance is where -the diagnosis starts. - -__Plots__ - -Clean, injected and difference — the difference panel should show the arc alone, -sitting on (correlated) noise, exactly 3" east of the ring. -""" -plot_dir = INJECT_ROOT / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(1, 3, figsize=(15, 5)) - -scale = np.nanpercentile(data_clean, 99.5) -for ax, (title, image) in zip( - axes, - (("clean", data_clean), ("injected", data_injected), ("difference", diff)), -): - ax.imshow(np.arcsinh(image / (0.05 * scale)), origin="lower", cmap="magma") - ax.set_title(title) - ax.set_xticks([]) - ax.set_yticks([]) - -fig.tight_layout() -plot_path = plot_dir / "injection_recovery.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"Recovery plot saved to: {plot_path.resolve()}") - -""" -__Wrap Up__ - -You built a lensed arc with known flux in pure numpy, injected it into real NIRCam -`_cal` exposures through each frame's own calibration keywords and PSF (with its -Poisson noise realised and its variance propagated into ERR), ran the identical -calwebb_image3 reduction twice, and recovered the flux to within the noise — a -closed-loop test of the entire pipeline, on real detector data, with every synthetic -ingredient disclosed in provenance. - -From here, the natural next step is science-grade simulation: replace the analytic arc -with a ray-traced source image (built in **PyAutoLens** and saved to FITS — still just -a Jy-per-pixel array as far as the injector is concerned) and you have realistic mock -JWST lenses embedded in genuine COSMOS-Web noise, ready for recovery tests of the full -modeling chain. - -The following locations of the workspace are good places to checkout next: - -- `scripts/hst_acs/simulator.py`: the HST injection sibling — e-/s units, driz_cr interactions. -- `scripts/jwst_nircam/start_here.py`: the units and provenance groundwork this script builds on. -- `scripts/jwst_nircam/individual.py`: injected frames also flow into frame products — semi-synthetic frames stay stamped. -- `autolens_workspace/scripts/imaging/simulator.py`: building ray-traced source images to feed this injector. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/jwst_nircam/start_here.py b/workspace/scripts/jwst_nircam/start_here.py deleted file mode 100644 index 564976a..0000000 --- a/workspace/scripts/jwst_nircam/start_here.py +++ /dev/null @@ -1,437 +0,0 @@ -""" -Start Here: JWST NIRCam -======================= - -JWST/NIRCam is, right now, the most powerful imaging instrument in existence for strong -gravitational lens science: diffraction-limited resolution beyond 2 microns, extraordinary -depth in minutes of exposure time, and continuous wavelength coverage from 0.6 to 5 microns -across its short-wavelength (SW) and long-wavelength (LW) channels. - -This script uses **PyAutoReduce** to reduce NIRCam imaging of the COSMOS-Web ring — a -spectacular Einstein ring discovered *during the data reduction* of the COSMOS-Web survey — -from raw archive products into a modeling-ready dataset: `data.fits`, `noise_map.fits`, -`psf.fits`, `psf_full.fits` and a full `reduction.json` provenance record. This is the -exact product set that `autolens_workspace/scripts/imaging/start_here.py` loads to model -this very lens. - -Everything runs through two public names — `TargetSpec` and `reduce_target` — so the -script is short on code and long on explanation. The first run downloads the level-2 -exposures from MAST and runs the official `jwst` calwebb_image3 pipeline, which takes -tens of minutes; reruns re-use the cache and are much faster. - -__Contents__ - -- **The COSMOS-Web Ring:** The lens we reduce, and why it is the perfect JWST anchor. -- **Imports:** Import **PyAutoReduce** and the supporting libraries. -- **Paths:** Anchor the cache and output folders to the workspace root. -- **Instrument Adapter:** Route the F277W filter to the NIRCam LW adapter. -- **Target Spec:** Declare the reduction — every dial explained. -- **Archive Products and Units:** What a `_cal` file is and why its units are MJy/sr. -- **CRDS References:** The reference-file system and how to pin a context for reproducibility. -- **The Reduction:** One function call: MAST download, calwebb_image3, noise, PSF, package. -- **Provenance Walk:** Read the evidence out of the returned reduction record. -- **Noise — Read, Don't Construct:** The resampled ERR array, the Casertano R factor and the blank-sky closure. -- **Parity Stance:** Why "close + internally consistent" is the honest bar against the bespoke COSMOS-Web team pipeline. -- **PSF:** An empirical ePSF from mosaic field stars, and why surface-brightness units change star selection. -- **Plots:** Inspect the data, noise map and PSF. -- **Loading in PyAutoLens:** Load the products with `al.Imaging.from_fits`, with the MJy/sr caveat spelled out. -- **Wrap Up:** Where to go next. - -__The COSMOS-Web Ring__ - -COSMOS-Web (Casey et al. 2023) is the largest JWST Cycle 1 GO program: 0.54 square degrees -of NIRCam imaging in four bands (F115W, F150W, F277W, F444W) over the COSMOS field. While -the team was reducing the very first epoch of data, a complete Einstein ring appeared in -the mosaics around a massive elliptical galaxy — the "COSMOS-Web ring" -(Mercier et al. 2024, A&A 687, A61, https://arxiv.org/abs/2309.15986). It is one of the -most striking strong lenses JWST has observed, alongside "JWST-ER1" -(van Dokkum et al. 2024, Nature Astronomy), whose source lies at redshift 5.1. - -The ring is also the anchor of a much bigger science programme: COWLS, the COSMOS-Web Lens -Survey, has published over 100 high-confidence lens candidates with **PyAutoLens** -reconstructions in all four bands (COWLS I: Nightingale et al. 2025, MNRAS 543, 203, -https://arxiv.org/abs/2503.08777; COWLS II: https://arxiv.org/abs/2503.08782; COWLS III: -https://arxiv.org/abs/2503.08785). Every one of those models consumed a reduced dataset -shaped exactly like the one this script produces — which is why the ring is -**PyAutoReduce**'s JWST validation anchor. - -__Imports__ - -**PyAutoReduce** exposes exactly two names — `TargetSpec` (a frozen dataclass declaring -*what* to reduce) and `reduce_target` (the function that does it). NIRCam adds one routing -helper, `nircam_adapter_for_filter`, which maps a filter name to the right channel adapter. -Everything else here is the standard scientific Python stack. -""" -from pathlib import Path -import json - -import matplotlib.pyplot as plt -import numpy as np - -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target -from autoreduce.instruments import nircam_adapter_for_filter - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: its combine step -changes the working directory internally, so relative paths would break. - -The cache folder holds the downloaded exposures and the CRDS reference files — both are -re-used across runs, so a second reduction of the same target (or a different band of the -same target) skips the download entirely. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -""" -__Instrument Adapter__ - -NIRCam observes through two channels simultaneously via a dichroic: the short-wavelength -(SW) channel (0.6–2.3 microns, eight 2048x2048 H2RG detectors at 0.031"/pixel) and the -long-wavelength (LW) channel (2.4–5.0 microns, two detectors at 0.063"/pixel). See the -JDox NIRCam pages (https://jwst-docs.stsci.edu) for the instrument overview. - -**PyAutoReduce** encodes each channel as an *instrument adapter* — a frozen bundle of -archive, calibration and combination facts (which MAST products to fetch, which CRDS -server to sync from, which combine backend to run, the native and recommended output -pixel scales). You never pick the channel yourself: `nircam_adapter_for_filter` routes -any NIRCam filter name to the right adapter, and raises loudly on a filter it does not -know. - -F277W is an LW filter, so we get the `nircam_lw` adapter: native 0.063"/pixel, with a -recommended output scale of 0.06"/pixel — the COSMOS-Web mosaic convention. -""" -band = "F277W" - -adapter = nircam_adapter_for_filter(band) - -print(f"Adapter key: {adapter.key}") # nircam_lw -print(f"Native pixel scale: {adapter.native_scale}\"/pixel") # 0.063 -print(f"Recommended output scale: {adapter.recommended_final_scale}\"/pixel") # 0.06 -print(f"Combine backend: {adapter.combine_backend}") # jwst_image3 - -""" -__Target Spec__ - -A **PyAutoReduce** reduction is *declared*, not scripted: you build a single frozen -`TargetSpec` and every downstream stage is a pure function of it plus the archive. The -same spec always produces the same dataset — that is what makes the reduction -reproducible and its provenance meaningful. - -Each non-default dial is explained by its trailing comment. Two deserve emphasis: - -- `proposal_ids=("1727",)` restricts the archive query to the COSMOS-Web program. Other - JWST programs have since observed these coordinates; mixing their exposures in would - change the depth and the noise properties of the mosaic in ways the COSMOS-Web - literature comparisons could not track. - -- `final_pixfrac=1.0` uses the full drizzle "drop" — the COSMOS-Web mosaic convention. - Shrinking the drop reduces noise correlation but demands more dither coverage; with - the ring's dither pattern the full drop is the safe, convention-matching choice. -""" -RA, DEC = 150.10048, 1.89301 # the COSMOS-Web ring (Mercier et al. 2024) - -spec = TargetSpec( - name=f"cosmos_web_ring_{band.lower()}", # names the output folder: output/cosmos_web_ring_f277w/ - ra=RA, # target right ascension in degrees (J2000) - dec=DEC, # target declination in degrees (J2000) - instrument=adapter.key, # "nircam_lw" — the adapter routed from the filter above - filter_name=band, # the NIRCam filter to reduce - proposal_ids=("1727",), # COSMOS-Web only: other programs at these coords would change the depth - final_scale=adapter.recommended_final_scale, # 0.06"/pixel output grid — the COSMOS-Web LW convention - final_pixfrac=1.0, # full drizzle drop, matching the COSMOS-Web mosaics - cutout_shape=(209, 209), # 209 px x 0.06" ~ 12.5" — generous margin around the ~2" ring -) - -""" -__Archive Products and Units__ - -**PyAutoReduce** enters the JWST calibration chain at level 2: it downloads the `_cal` -products of the `calwebb_image2` pipeline from MAST. These are individual exposures that -have already been through detector-level corrections (stage 1) and per-exposure -calibration (stage 2) at STScI — flat-fielded, WCS-assigned and flux-calibrated. The -stage-by-stage story of what those pipelines do lives in `step_by_step.py`; the short -version is that stages 1 and 2 are pure STScI defaults, and **PyAutoReduce** only takes -over at the combination stage where the lensing-specific choices live. - -One consequence matters immediately: the `photom` step of calwebb_image2 converts every -`_cal` file to **megajanskys per steradian (MJy/sr)** — a *surface brightness*, not a -flux. HST reductions ship in electrons per second (a count rate per pixel); JWST products -instead tell you the brightness per unit solid angle, independent of pixel size. -**PyAutoReduce** follows its defaults-first principle and keeps the native MJy/sr units -all the way into `data.fits` (with `BUNIT` recorded in the header). - -To recover a flux from these units you multiply by the pixel solid angle, which the JWST -headers carry as `PIXAR_SR` (steradians per pixel): - - flux [Jy] = surface_brightness [MJy/sr] x 1e6 x PIXAR_SR - -and AB magnitudes follow as m_AB = -2.5 log10(flux / 3631 Jy). We return to this when -loading the dataset into **PyAutoLens** below, because it changes how you interpret -every fitted intensity. - -__CRDS References__ - -JWST calibration reference files come from the CRDS system (https://jwst-crds.stsci.edu). -Unlike the HST path — where **PyAutoReduce** syncs best references explicitly before -running — the `jwst` pipeline fetches what it needs *lazily* through the `CRDS_PATH` -environment variable, which **PyAutoReduce** points into the cache folder. The references -persist there across runs and are never evicted. For byte-for-byte reproducibility you -can additionally pin `CRDS_CONTEXT` to a fixed context; the software versions actually -used are recorded in `reduction.json` either way. - -__The Reduction__ - -Everything happens in the single call below. Internally it runs, in order: - -1. **acquire** — query MAST for COSMOS-Web `_cal` exposures overlapping the target - footprint in F277W, download them into the cache. -2. **combine** — build a level-3 association and run the official `jwst` - calwebb_image3 pipeline (tweakreg / skymatch / outlier_detection / resample), with - the lensing dials mapped onto the resample step. -3. **noise** — read the resampled ERR array and apply the correlated-noise correction. -4. **psf** — find stars on the mosaic and build an empirical ePSF. -5. **package** — cut out the target region, apply the bad-pixel policy, write the four - FITS products plus `reduction.json`. -""" -print( - f""" - Starting the {band} reduction of the COSMOS-Web ring. - - First run: this downloads the program-1727 _cal exposures from MAST (a few GB) and - runs the jwst calwebb_image3 pipeline — expect tens of minutes, and a large CRDS - reference download on a cold cache. Reruns re-use the exposure cache and are much - faster. - """ -) - -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / spec.name - -print(f"Reduction complete. Products written to: {out_dir}") - -""" -__Provenance Walk__ - -`reduce_target` returns the full provenance record (the same content written to -`reduction.json`). Nothing about the reduction is hidden: every stage writes a block -recording what it did and the diagnostics it measured. Walking this record after every -reduction is a habit worth building — it is how you catch a marginal reduction before -you spend GPU-hours modeling it. - -The `drizzle` block records the combine backend and the exact resample keywords that -were mapped from your spec, and two diagnostics worth reading every time: - -- `weight_uniformity` — the RMS/median of the drizzle weight map. A uniform weight map - means every output pixel saw comparable total exposure; values creeping toward the - 0.2 policy limit flag ragged dither coverage. -- `weight_uniformity_cutout` — the same statistic restricted to the shipped cutout, - which is the number that actually matters for the lens. -""" -print("Backend:", record["drizzle"]["backend"]) # "jwst_image3" -print("Resample kwargs:", json.dumps(record["drizzle"]["resample_kwargs"], indent=2)) -print("Exposures combined:", record["acquire"]["n_exposures"]) -print("Weight uniformity (mosaic):", record["drizzle"]["weight_uniformity"]) -print("Weight uniformity (cutout):", record["drizzle"]["weight_uniformity_cutout"]) - -""" -The `psf` block records how the PSF was built (method, number of stars used, the FWHM in -output pixels and which mosaic pass fed the star finder), and the `package` block records -the pixel scale and data units your modeling code needs. -""" -print("PSF:", json.dumps(record["psf"], indent=2)) -print("Pixel scale:", record["package"]["pixel_scale"]) -print("Data units:", record["package"]["data_units"]) # MJy/sr - -""" -__Noise — Read, Don't Construct__ - -On HST, **PyAutoReduce** *constructs* the noise map from first principles: Poisson counts -over exposure time plus the inverse of the drizzle weight. On JWST it deliberately does -the opposite — it **reads** the noise, because the `jwst` pipeline already propagates a -full per-pixel error budget. Each `_cal` exposure carries `VAR_POISSON`, `VAR_RNOISE` -and `VAR_FLAT` variance planes; the resample step drizzles each plane separately onto -the output grid and recombines them into the mosaic `ERR` array. Re-deriving that budget -from scratch would discard information the pipeline already has (per-pixel read noise, -flat-field uncertainty, ramp-fit weighting). - -One correction is still required. Resampling correlates neighbouring output pixels — -each input pixel's flux is shared among several output pixels — so the *per-pixel* ERR -values underestimate the uncertainty of any quantity summed over more than one pixel -(Fruchter & Hook 2002, PASP 114, 144). **PyAutoReduce** applies the standard scalar -correction factor R of Casertano et al. (2000, AJ 120, 2747), computed from the same -pixfrac and scale ratio the resample step used: - - noise_map = R x ERR - -This is exactly the factor the HST path applies, because resample correlates pixels -exactly as drizzle does. R is recorded as `correlated_noise_factor`. - -The `noise` block then closes the loop with an internal consistency check: it compares -the sigma-clipped RMS of blank sky in the mosaic against the 5th percentile of the ERR -array (the ERR "floor", pre-R). If the pipeline's error model and the actual sky -fluctuations agree, `sky_over_err_floor` sits near 1. A large disagreement means the -upstream error model and the data disagree — something to investigate, never to absorb -silently. -""" -print("Noise recipe:", record["noise"]["recipe"]) -print("Correlated-noise factor R:", record["noise"]["correlated_noise_factor"]) -print("Empirical sky RMS:", record["noise"]["empirical_sky_rms"]) -print("ERR 5th percentile (pre-R):", record["noise"]["err_5th_percentile_pre_R"]) -print("sky / ERR floor:", record["noise"]["sky_over_err_floor"]) - -""" -__Parity Stance__ - -An honest caveat, stated up front because it frames how to interpret this dataset. - -The published COSMOS-Web mosaics were *not* made by running calwebb with defaults. The -team's reduction (Franco et al., https://arxiv.org/abs/2506.03256) layers survey-grade -corrections on top of the official pipeline: custom 1/f-noise destriping, wisp -subtraction, snowball handling, a custom background model and astrometry tied to a -Gaia-registered COSMOS reference frame. Every major deep survey does something similar -(CEERS: Bagley et al. 2023, https://arxiv.org/abs/2211.02495; JADES: Rieke et al. 2023, -https://arxiv.org/abs/2306.02466), because wide-area depth uniformity demands it. - -**PyAutoReduce** deliberately does not chase those corrections. Its bar for JWST is -**"close + internally consistent," not reproduction**: strong lens modeling needs a -reduction whose noise map matches its data, whose weights are uniform over the cutout, -and whose masked-pixel policy is explicit — all properties this pipeline checks itself, -on your cutout, every run. Order-unity data/noise ratios against the team mosaics are -expected and acceptable; what must hold are the internal closures you just read -(`sky_over_err_floor`, weight uniformity) and cross-band consistency of any global -offset (which `multi_band.py` demonstrates). At lens-cutout scale (~12"), the survey-wide -corrections mostly matter through slightly elevated large-scale noise — which the -noise-map closure would reveal if it mattered. - -__PSF__ - -The PSF ships in two forms: `psf.fits` (21x21 pixels) sized for convolution during -model fitting, and `psf_full.fits` (61x61) capturing the extended wings for -flux-sensitive work. Both are odd-shaped, unit-normalised, and — crucially — built -*from the mosaic itself*, so the delivered PSF has been through the identical -resampling (same pixel scale, pixfrac, kernel and rotation) as the data it will be -convolved with. A PSF that has not shared the data's resampling history is subtly -wrong everywhere. - -The builder is the empirical "effective PSF" (ePSF) estimator of Anderson & King -(2000, PASP 112, 1360) as implemented in photutils, fed by stars found on the mosaic. -One JWST-specific detail: on HST, star selection rejects stars whose peaks approach -the detector full well, because their cores are non-linear. In MJy/sr units a peak -threshold in counts is meaningless — and saturated cores arrive from level 2 already -blanked to NaN — so the JWST star finder is NaN-masked and applies **no peak cut**. -The full JWST PSF story (undersampling, STARRED, model PSFs) lives in `psf.py`. - -__Plots__ - -Now inspect what was produced. We use an arcsinh stretch for the data — linear near -zero so the sky noise is visible, logarithmic on the bright lens galaxy — and a linear -stretch for the noise map, where structure (depth variations, masked pixels at 1e8) -is what you are looking for. -""" -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -data = fits.getdata(out_dir / "data.fits").astype(float) -noise = fits.getdata(out_dir / "noise_map.fits").astype(float) -psf = fits.getdata(out_dir / "psf.fits").astype(float) - -fig, axes = plt.subplots(1, 3, figsize=(15, 5)) - -scale = np.nanpercentile(data, 99.5) -axes[0].imshow(np.arcsinh(data / (0.05 * scale)), origin="lower", cmap="magma") -axes[0].set_title(f"data.fits ({band}, MJy/sr, arcsinh)") - -axes[1].imshow(np.where(noise > 1.0e7, np.nan, noise), origin="lower", cmap="viridis") -axes[1].set_title("noise_map.fits (masked px blanked)") - -axes[2].imshow(np.arcsinh(psf / psf.max() * 1000.0), origin="lower", cmap="magma") -axes[2].set_title("psf.fits (21x21, arcsinh)") - -for ax in axes: - ax.set_xticks([]) - ax.set_yticks([]) - -fig.tight_layout() -plot_path = plot_dir / "start_here_products.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"Product plot saved to: {plot_path.resolve()}") - -""" -In the data panel you should see the lens galaxy at the centre with the Einstein ring -wrapped around it — the same morphology as Figure 1 of Mercier et al. 2024. In the noise -panel, depth structure follows the dither coverage; any isolated blanked pixels are the -masked-by-noise policy at work (bad pixels carry noise 1e8 and zeroed data, so a chi^2 -simply ignores them — no separate mask file needed downstream). - -__Loading in PyAutoLens__ - -The products are designed to drop straight into `al.Imaging.from_fits`. The pixel scale -comes from the provenance record, never from memory. - -**The MJy/sr caveat, spelled out.** **PyAutoLens** is unit-agnostic: it fits whatever -units the data arrives in, and the model's `intensity` parameters inherit those units. -With this dataset: - -- Fitted intensities are in **MJy/sr** — surface brightnesses, not counts. This is - actually natural for galaxy profiles (a Sersic is a surface-brightness profile), but - it means you cannot compare intensity values against an HST fit in e-/s without - converting. -- To quote a **flux**, integrate the model image and multiply by the pixel solid angle: - `flux_jy = model_image.sum() * (pixel_scale / 206265.0)**2 * 1e6`, then - m_AB = -2.5 log10(flux_jy / 3631). -- The noise map is in the same units, so signal-to-noise and chi^2 are unit-free — the - *fit* is unaffected. Only the physical interpretation of intensities needs care. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "PyAutoLens is not installed (pip install autolens), so the final loading step " - "is skipped — the reduction itself is complete and the products above are " - "ready for any modeling tool." - ) - -if al is not None: - dataset = al.Imaging.from_fits( - data_path=out_dir / "data.fits", - noise_map_path=out_dir / "noise_map.fits", - psf_path=out_dir / "psf.fits", - pixel_scales=record["package"]["pixel_scale"], - ) - print( - f"Loaded al.Imaging: shape {dataset.data.shape_native}, " - f"pixel scale {dataset.pixel_scales} — ready for lens modeling." - ) - -""" -__Wrap Up__ - -You have reduced JWST/NIRCam imaging of the COSMOS-Web ring from MAST archive products -to a modeling-ready dataset, and audited the reduction through its provenance record: -the calwebb_image3 backend and its resample dials, weight uniformity, the read-don't- -construct noise map with its Casertano R factor and blank-sky closure, and the mosaic -ePSF. - -The following locations of the workspace are good places to checkout next: - -- `scripts/jwst_nircam/step_by_step.py`: what Detector1, Image2 and Image3 actually do to the data, stage by stage, with the pipeline documentation. -- `scripts/jwst_nircam/multi_band.py`: reduce all four COSMOS-Web bands into a multi-wavelength dataset. -- `scripts/jwst_nircam/psf.py`: the JWST PSF story in depth — undersampling, ePSF lineage, STARRED vs photutils. -- `scripts/jwst_nircam/individual.py`: per-exposure frame products instead of a mosaic. -- `scripts/jwst_nircam/simulator.py`: inject a synthetic source into the real exposures and test flux recovery. -- `scripts/guides/output_contract.py`: the four-file + reduction.json contract in full detail. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/jwst_nircam/step_by_step.py b/workspace/scripts/jwst_nircam/step_by_step.py deleted file mode 100644 index ffde792..0000000 --- a/workspace/scripts/jwst_nircam/step_by_step.py +++ /dev/null @@ -1,377 +0,0 @@ -""" -JWST NIRCam: Step By Step -========================= - -The `start_here.py` example reduced the COSMOS-Web ring in one `reduce_target` call. -This script slows down and teaches what actually happens to a NIRCam exposure on its way -from detector ramps to your `data.fits` — the three stages of the JWST Science -Calibration Pipeline, which of them **PyAutoReduce** re-runs (only the third), and how to -read the evidence of every choice out of `reduction.json` afterwards. - -The canonical references are the `jwst` package documentation -(https://jwst-pipeline.readthedocs.io/en/stable/, stage index at -https://jwst-pipeline.readthedocs.io/en/stable/jwst/pipeline/main.html) and the JDox -pipeline pages (https://jwst-docs.stsci.edu/jwst-science-calibration-pipeline). Links to -the specific pages accompany each stage below — this workspace's whole purpose is that -every reduction step points at the document that defines it. - -The reduction at the end re-uses the cache from `start_here.py`, so run that first if you -want the fast path. - -__Contents__ - -- **The Three Stages:** The calwebb pipeline architecture and where **PyAutoReduce** enters it. -- **Imports:** Import **PyAutoReduce** and the supporting libraries. -- **Paths:** Anchor the cache and output folders to the workspace root. -- **Stage 1 — Detector1:** Up-the-ramp fitting, jump detection, snowballs, 1/f noise, superbias and reference pixels. -- **Stage 2 — Image2:** WCS assignment, flat fielding and the photometric calibration to MJy/sr. -- **Stage 3 — Image3, As PyAutoReduce Runs It:** Association building, tweakreg, skymatch, outlier detection, and the resample dial mapping. -- **CRDS:** Lazy reference syncing through CRDS_PATH, and context pinning for reproducibility. -- **The Reduction:** Run the ring reduction and collect the record. -- **Reading the Evidence:** The drizzle and noise blocks of reduction.json, stage by stage. -- **The Casertano Factor, Standalone:** Compute R yourself with the public helper. -- **Wrap Up:** Where to go next. - -__The Three Stages__ - -Every JWST image passes through three pipeline stages at STScI before you ever see it: - -1. **calwebb_detector1** — detector-level corrections on the raw up-the-ramp readouts - (`_uncal.fits` -> `_rate.fits`): one file per exposure, units of DN/s. -2. **calwebb_image2** — per-exposure calibration (`_rate.fits` -> `_cal.fits`): WCS, - flat field, flux calibration to MJy/sr. -3. **calwebb_image3** — ensemble combination (`_cal.fits` -> `_i2d.fits` mosaic): - alignment, background matching, outlier rejection, drizzle-style resampling. - -**PyAutoReduce** enters at level 2: it downloads the `_cal` products from MAST and -re-runs only stage 3. Stages 1 and 2 are therefore *pure STScI defaults* — the archive's -own processing, with the calibration reference files current at retrieval time. This is -the defaults-first principle doing real work: the detector-level corrections are exactly -what every other JWST paper uses, and the lensing-specific decisions all live in the one -stage **PyAutoReduce** controls. You still need to understand stages 1 and 2, though, -because their fingerprints (removed cosmic rays, error budgets, surface-brightness -units) are all over the products you model. - -__Imports__ -""" -from pathlib import Path -import json - -import matplotlib.pyplot as plt -import numpy as np - -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target -from autoreduce.instruments import nircam_adapter_for_filter -from autoreduce.noise.rms import casertano_r, empirical_background_rms -from autoreduce.drizzle.diagnostics import weight_uniformity - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: its combine step -changes the working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded exposures + CRDS references (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -""" -__Stage 1 — Detector1__ - -Documentation: -https://jwst-pipeline.readthedocs.io/en/stable/jwst/pipeline/calwebb_detector1.html - -JWST's near-infrared detectors do not take a single exposure the way a CCD does. Each -pixel is read out *non-destructively* many times as charge accumulates — an -"up-the-ramp" sequence of groups — and the flux is the fitted *slope* of that ramp. -calwebb_detector1 turns the raw ramp cube (`_uncal.fits`) into a slope image -(`_rate.fits`) through a chain of NIR steps: group_scale, dq_init, saturation, ipc, -superbias, refpix, linearity, persistence, dark_current, jump, ramp_fit and gain_scale. -The ones whose consequences you will meet downstream: - -- **superbias / refpix** — subtract the fixed bias structure per group, then use the - 4-pixel border of light-insensitive reference pixels around each 2048x2048 sensor to - track and remove bias drifts during the integration. - -- **jump** — detects cosmic-ray hits as discontinuities between consecutive group - differences (two-point difference statistics). This is why JWST frames need no - L.A.Cosmic-style CR rejection: a hit corrupts one *group*, not the exposure. - -- **ramp_fit** — fits an optimally-weighted (Fixsen et al. 2000, PASP 112, 1350) slope - to each pixel's ramp, split into segments at flagged jumps. A pixel hit by a cosmic - ray mid-ramp still yields a valid rate from its clean segments; even pixels that - saturate late in the ramp yield rates from the early groups. Cosmic rays are thus - *removed*, not merely flagged — a fact that shapes the DQ policy for frame products - (see `individual.py`). - -- **Snowballs** — the spectacular failure mode of jump detection. Large cosmic-ray - events (likely secondary particle showers) saturate a core and splash charge over - hundreds to thousands of pixels - (https://jwst-docs.stsci.edu/data-artifacts-and-features/snowballs-and-shower-artifacts). - The jump step's `expand_large_events` option fits enclosing circles around big events - and expands the flagged region to cover the halo. Early-cycle data (including the - first COSMOS-Web and CEERS epochs) needed manual snowball masking; modern pipeline - builds handle most of it automatically. - -- **1/f noise** — correlated readout noise from the SIDECAR ASIC readout electronics, - appearing as faint banding along the slow-read axis, different per amplifier - (https://jwst-docs.stsci.edu/known-issues/1-f-noise). The community fix — Chris - Willott's `image1overf.py` (https://github.com/chriswillott/jwst), applied by CEERS - and many survey teams to the rate files — worked well enough that the algorithm was - adopted into the official pipeline as the `clean_flicker_noise` step. A community - correction graduating into the pipeline is the JWST calibration story in miniature, - and it is why **PyAutoReduce** is comfortable trusting archive defaults: the defaults - keep absorbing the community's fixes. (The `_cal` files this workspace's anchor - reductions use predate routine 1/f cleaning — the residual banding is part of the - parity stance discussed in `start_here.py`.) - -__Stage 2 — Image2__ - -Documentation: -https://jwst-pipeline.readthedocs.io/en/stable/jwst/pipeline/calwebb_image2.html - -calwebb_image2 calibrates each slope image individually into a `_cal.fits` file: - -- **assign_wcs** — attaches the full distortion-aware WCS (a gwcs object serialised in - the file, alongside a FITS-approximation SIP solution), tied to the spacecraft - pointing solution. -- **flat_field** — divides by the CRDS flat for the detector/filter. -- **photom** — converts DN/s to **MJy/sr** using the CRDS photometric calibration - (the `PHOTMJSR` scale factor recorded in the header). From this point on, the data - are surface brightnesses; the pixel solid angle `PIXAR_SR` rides in the header so - fluxes remain recoverable (flux [Jy] = SB [MJy/sr] x 1e6 x PIXAR_SR). - -The `_cal` file also carries the per-pixel error budget as separate variance planes — -`VAR_POISSON`, `VAR_RNOISE`, `VAR_FLAT` — with `ERR = sqrt(sum)`. These planes are the -foundation of the JWST noise story: they propagate through resampling, which is why -**PyAutoReduce** reads its noise map rather than constructing one. - -__Stage 3 — Image3, As PyAutoReduce Runs It__ - -Documentation: -https://jwst-pipeline.readthedocs.io/en/stable/jwst/pipeline/calwebb_image3.html - -Stage 3 is where **PyAutoReduce** takes over, because the ensemble-combination choices -(output grid, drop size, kernel, orientation) are exactly the ones lens modeling cares -about. What it does, in order: - -1. **Association building.** Image3 consumes an ASN file — a JSON manifest declaring - which `_cal` exposures form one product. **PyAutoReduce** builds this from the - cached exposures that survived its footprint filter and hands the pipeline the list. - -2. **tweakreg** — relative alignment of the exposures via source catalogs, then - absolute alignment against Gaia DR3 (the default `abs_refcat`). A deep-field - caveat worth knowing: in fields like COSMOS, few Gaia stars land on a single - NIRCam pointing, which is why survey teams align to Gaia-*tied* external catalogs - instead (the COSMOS astrometric frame; CEERS aligns to HST catalogs tied to Gaia). - **PyAutoReduce** runs tweakreg with defaults — the MAST `_cal` headers already - carry good a-priori pointing, and residual relative alignment is what matters at - lens-cutout scale. - -3. **skymatch** — measures and equalises background levels between exposures. By - default it *records* the matched levels (the `BKGLEVEL` header key) rather than - subtracting them from your data. - -4. **outlier_detection** — the stack-based second line of cosmic-ray defence: resample - all exposures, median them, blot the median back to each frame, and flag deviant - pixels (residual CRs, snowball residue, hot pixels, persistence). Only overlapping - dithers can be cleaned this way. The flagged per-exposure files are the `_crf` - products, which the frame-products mode packages (`individual.py`). - -5. **resample** — the drizzle analogue (Fruchter & Hook 2002, PASP 114, 144), combining - all exposures onto one output grid. This is where your `TargetSpec` dials land. - -The dial mapping is one-to-one and recorded in provenance: - -| TargetSpec dial | resample argument | value here | -|----------------------|-------------------|----------------------------------| -| `final_scale` | `pixel_scale` | 0.06 (LW COSMOS-Web convention) | -| `final_pixfrac` | `pixfrac` | 1.0 (full drop) | -| `final_kernel` | `kernel` | "square" | -| (always) | `rotation` | 0.0 — north-up output | -| (adapter default) | `weight_type` | "ivm" — inverse-variance weights | - -Two implementation details you would otherwise discover the hard way: **PyAutoReduce** -runs the pipeline with `in_memory=False` (image3 holds every resampled model in RAM -otherwise, and exhausts it on large dither sets), and the multi-extension `_i2d` output -(SCI/ERR/CON/WHT/VAR_*) is normalised into standalone `sci`/`wht`/`err` files so every -downstream stage (noise, PSF, package) stays backend-agnostic — the same three files an -HST AstroDrizzle run produces. - -__CRDS__ - -Every pipeline step above pulls its reference files (flats, distortion maps, photometric -calibrations...) from CRDS (https://jwst-crds.stsci.edu). The `jwst` package syncs -references *lazily*: when a step needs a file it checks the local cache under -`CRDS_PATH` and downloads on miss. **PyAutoReduce** points `CRDS_PATH` into its cache -root, so references download once and persist — they are never evicted, even when -exposure caches are. - -Reproducibility note: CRDS is versioned by *context* (a pmap file naming every reference -in force). By default you get the latest operational context, which advances as STScI -delivers new calibrations. For strictly reproducible reruns, pin `CRDS_CONTEXT` in your -environment; either way `reduction.json` records the software versions used. Relatedly, -the `jwst` package itself is pinned at 1.14.0 in this stack — the provenance records the -version, so a future upgrade is a visible, auditable event rather than silent drift. - -__The Reduction__ - -Now run it: the same F277W ring spec as `start_here.py`, so a warm cache makes this a -combine-only rerun. -""" -band = "F277W" -adapter = nircam_adapter_for_filter(band) - -spec = TargetSpec( - name=f"cosmos_web_ring_{band.lower()}", # same name as start_here.py -> shares its exposure cache - ra=150.10048, # the COSMOS-Web ring (Mercier et al. 2024) - dec=1.89301, - instrument=adapter.key, # "nircam_lw" - filter_name=band, - proposal_ids=("1727",), # COSMOS-Web only - final_scale=adapter.recommended_final_scale, # 0.06"/pixel — maps to resample pixel_scale - final_pixfrac=1.0, # maps to resample pixfrac - final_kernel="square", # maps to resample kernel (the default, stated here for the mapping table) - cutout_shape=(209, 209), # ~12.5" at 0.06"/pixel -) - -print( - """ - Running calwebb_image3 on the COSMOS-Web ring (F277W). - - With a warm cache from start_here.py this skips the MAST download and runs the - combine + noise + psf + package stages — expect several minutes. On a cold cache, - add the download time (a few GB). - """ -) - -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / spec.name - -""" -__Reading the Evidence__ - -Every claim made above is checkable in the record. First the `drizzle` block — the -backend, the exact resample kwargs, and the `_crf` bookkeeping from outlier_detection: -""" -drizzle = record["drizzle"] - -print("Backend:", drizzle["backend"]) # "jwst_image3" -print("Resample kwargs as run:", json.dumps(drizzle["resample_kwargs"], indent=2)) -print("Normalised ERR mosaic:", drizzle["err_path"]) -print(f"Outlier-flagged _crf exposures: {len(drizzle['crf_paths'])}") -for path in drizzle["crf_paths"][:3]: - print(" ", Path(path).name) - -""" -The `crf_paths` list is the receipt that outlier_detection ran and saved its -per-exposure flagged products — `individual.py` turns those into modelable native-frame -cutouts. The `err_path` is the resampled ERR mosaic the noise stage read. - -Next the `noise` block. The recipe string states the JWST policy in one line — read the -propagated ERR, multiply by R — and the consistency numbers close the loop against the -actual sky in the mosaic: -""" -noise_block = record["noise"] - -print("Recipe:", noise_block["recipe"]) # "R * ERR (propagated by calwebb_image3 resample)" -print("R:", noise_block["correlated_noise_factor"]) -print("Empirical sky RMS:", noise_block["empirical_sky_rms"]) -print("ERR floor (5th percentile, pre-R):", noise_block["err_5th_percentile_pre_R"]) -print("sky_over_err_floor:", noise_block["sky_over_err_floor"]) - -""" -`sky_over_err_floor` near 1 means the pipeline's propagated error budget and the -measured blank-sky fluctuations agree — the reduction is internally consistent. A value -far from 1 would mean the upstream variance planes and the data disagree; that is -investigated (wrong exposures mixed in? background structure?), never absorbed into a -fudge factor. - -__The Casertano Factor, Standalone__ - -R is not a black box — it is a closed-form function of the two resampling dials, from -Casertano et al. (2000, AJ 120, 2747; also the DrizzlePac handbook). You can compute it -yourself with the same public helper the pipeline uses, and check it against the record: -""" -scale_ratio = spec.final_scale / adapter.native_scale # s: output / native pixel size - -R = casertano_r(pixfrac=spec.final_pixfrac, scale_ratio=scale_ratio) - -print(f"scale_ratio s = {scale_ratio:.4f}, pixfrac p = {spec.final_pixfrac}") -print(f"casertano_r -> R = {R:.4f}") -print(f"recorded -> R = {noise_block['correlated_noise_factor']:.4f}") - -""" -Note the regime: here s < p (0.06/0.063 ~ 0.95 output pixels per native pixel, with a -full drop), which puts us on the fine-grid branch of the formula where correlation — -and therefore R — grows as the output grid gets finer than the drop. Shift-and-add -(p = 1 at s = 1) gives the textbook R = 1.5; our slightly finer grid gives a little -more. This is the price of resampling: adjacent output pixels share input flux, so -per-pixel errors understate aperture errors by R. - -You can also re-derive the two mosaic diagnostics from the normalised products the -combine stage left in the work directory — the same public helpers the pipeline calls: -""" -sci = fits.getdata(out_dir / "data.fits").astype(float) -wht_files = sorted((out_dir / "work").rglob("*_wht.fits")) - -print("Empirical background RMS of the cutout:", empirical_background_rms(sci)) -if wht_files: - wht = fits.getdata(wht_files[0]).astype(float) - print("Weight uniformity of the full mosaic:", weight_uniformity(wht)) - -""" -Finally, a picture of the evidence: the ERR mosaic's distribution against the sky RMS — -the closure check drawn rather than tabulated. -""" -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -noise_map = fits.getdata(out_dir / "noise_map.fits").astype(float) -good = noise_map[np.isfinite(noise_map) & (noise_map < 1.0e7)] - -fig, ax = plt.subplots(figsize=(7, 5)) -ax.hist(good, bins=100, color="steelblue", log=True) -ax.axvline( - noise_block["correlated_noise_factor"] * noise_block["err_5th_percentile_pre_R"], - color="k", ls="--", label="R x ERR floor", -) -ax.axvline( - noise_block["correlated_noise_factor"] * noise_block["empirical_sky_rms"], - color="crimson", ls=":", label="R x empirical sky RMS", -) -ax.set_xlabel("noise_map value (MJy/sr)") -ax.set_ylabel("pixels") -ax.set_title("Noise-map distribution vs the blank-sky closure") -ax.legend() -fig.tight_layout() -plot_path = plot_dir / "step_by_step_noise_closure.png" -fig.savefig(plot_path, dpi=150) -plt.close(fig) - -print(f"Noise-closure plot saved to: {plot_path.resolve()}") - -""" -__Wrap Up__ - -You now know what each calwebb stage does to a NIRCam exposure, which one -**PyAutoReduce** re-runs (Image3, with the lensing dials mapped onto resample), and how -to audit every choice from the provenance record: the resample kwargs, the `_crf` -receipts, the read-don't-construct noise recipe, the Casertano R you can recompute -yourself, and the blank-sky closure that certifies internal consistency. - -The following locations of the workspace are good places to checkout next: - -- `scripts/jwst_nircam/multi_band.py`: run all four COSMOS-Web bands and compare their closures. -- `scripts/jwst_nircam/individual.py`: package the `_crf` exposures as native-frame products. -- `scripts/jwst_nircam/psf.py`: the PSF stage in the same depth as the combine stage here. -- `scripts/guides/noise_maps.py`: the noise recipes across all instruments, and why chi^2 cares. - -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/scripts/keck_nirc2/README.md b/workspace/scripts/keck_nirc2/README.md deleted file mode 100644 index 97c26cb..0000000 --- a/workspace/scripts/keck_nirc2/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Keck NIRC2 AO - -Reducing Keck II NIRC2 laser-guide-star AO imaging — the first ground-based path, where -calibration, sky subtraction and registration are the pipeline's own job. The validation -anchor is B1938+666, the SHARP Einstein ring in which Vegetti et al. 2012 detected a -10^8 solar-mass dark satellite. - -Run the scripts in this order (each later script reads the cache/products of the first): - -- `start_here.py` — the full pipeline end to end on B1938+666: KOA discovery, pinned frame - ids, the ground stages, provenance, noise closure, and loading into PyAutoLens with the - plate-scale correction. -- `step_by_step.py` — every ground stage demonstrated standalone on the cached frames: - calibration matching, master flats/darks, the scaled running sky, phase-correlation - registration, the one-pixmap combine, and the detector noise budget. -- `psf.py` — the AO PSF problem: tier-A PSF-star epochs, the vetting gates, candidates, - and the `psf_provisional` contract. -- `simulator.py` — synthetic-ring injection into the real prepared frames, with flux - recovery and registration-invariance checks. diff --git a/workspace/scripts/keck_nirc2/__init__.py b/workspace/scripts/keck_nirc2/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/workspace/scripts/keck_nirc2/psf.py b/workspace/scripts/keck_nirc2/psf.py deleted file mode 100644 index ddd772d..0000000 --- a/workspace/scripts/keck_nirc2/psf.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -PSF: Keck NIRC2 AO -================== - -The PSF is where AO lens modeling is won or lost. A lensed arc is a ~0.1"-wide feature -convolved with a ~0.07" PSF: get the PSF wrong and the smooth-model residuals it leaves are -exactly the kind of localized flux a substructure search would claim as a dark subhalo. SHARP -III (Chen et al. 2016, MNRAS 462, 3457, arXiv:1601.01321) devotes itself to this problem — -the AO PSF is *the* dominant systematic of AO lens modeling. - -This script tells the AO PSF story end to end as **PyAutoReduce** implements it: why no -reduction-time AO PSF can be final, the tier-A PSF-star-epoch design and its cosmic-ray -vetting gates, the `psf_provisional` contract, and the fallback tiers. It reads the products -of `start_here.py`, so run that first. - -__Contents__ - -- **The AO PSF Problem:** Time-variable, field-variable, and no stable model library exists. -- **The SHARP Practice:** Dedicated PSF stars, and final PSF selection during modeling — not reduction. -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root. -- **Tier A — PSF-Star Epochs:** Each epoch reduced pipeline-identically, every epoch shipped as a candidate. -- **Grouping Into Epochs:** MJD-gap grouping of the PSF-star frames, demonstrated on the cache. -- **The Vetting Gates:** Coherence and diffraction-sharpness — how a cosmic ray fails to impersonate a star. -- **The Candidates:** Load and plot every `psf_candidate_.fits`, with the selection diagnostics. -- **Provisional By Contract:** `psf_provisional: true`, always — final selection belongs to modeling. -- **SHARP Parity:** The candidate FWHMs land in the ~65-70 mas range SHARP reports. -- **The FWHM Definition Wart:** Tier A and tier B measure FWHM differently — never cross-compare. -- **Tier B And Tier C:** The in-field ePSF fallback, and why target-based reconstruction is out of scope. -- **Wrap Up:** Summary and where to go next. - -__The AO PSF Problem__ - -An AO PSF is not a property of the instrument; it is a property of the *moment*. The -correction quality tracks the seeing and the AO loop performance, so the PSF changes from -visit to visit and frame to frame. It also changes *across the field* — anisoplanatism -elongates the PSF away from the guide star (the AIROPA project models exactly this -spatial variability for NIRC2; Witzel et al., JATIS 8, 038007, arXiv:2210.10940 and -arXiv:2207.00548). On typical lens fields the delivered Strehl is ~10-30% and the FWHM -~60-90 mas (van Dam et al. 2006, PASP 118, 310, measure 30-40% Strehl at K only for bright, -on-axis stars). - -Contrast this with HST or JWST, where optical models (TinyTim, STPSF) and stable ePSF -libraries exist because the optics barely change. For AO there is no stable library to look -up — and the narrow camera's ~10" field of view rarely contains a usable star, so you cannot -simply measure the PSF in-field either. (Telescope-telemetry PSF reconstruction — Ragland et -al., SPIE 9909 — is an active alternative, but not a reduction-pipeline commodity.) The PSF -must come from dedicated observations or from the science data itself. - -__The SHARP Practice__ - -SHARP's answer, inherited here, has two halves: - -1. **Observe a dedicated PSF star**, interleaved in time with the science dithers — for - B1938+666 the tip-tilt star itself, ~20" away, visited repeatedly through the sequence - (Lagattuta et al. 2012, MNRAS 424, 2800). Each visit samples the PSF *at that moment*. - -2. **Defer the final choice to modeling.** No reduction-time statistic can know which epoch - best matches the PSF that was in force during the science frames — but the lens model - can: fit the same data with each candidate PSF and let the Bayesian evidence decide - (the SHARP I practice). Reduction's job is to deliver *all* the candidates, reduced - honestly, and to be explicit that none of them is final. - -The logical extreme of this philosophy is to infer the PSF from the lensed images -themselves, iteratively, during modeling — SHARP III (Chen et al. 2016) showed this reaches -precision comparable to or better than HST, and Chen et al. 2021 (MNRAS 508, 755, -arXiv:2106.11060) validated AO PSF reconstruction against the astrometric requirements of -time-delay cosmography. That is a *modeling-stage* technique (tier C below) — but it only -works because the reduction hands modeling clean, pipeline-consistent starting candidates. - -__Imports__ -""" -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits - -from autoreduce.sky import group_by_time_gaps - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). We read the products and cache of `start_here.py`. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" -OUTPUT_ROOT = WORKSPACE / "output" - -out_dir = OUTPUT_ROOT / "b1938+666" -record_path = out_dir / "reduction.json" - -if not record_path.exists(): - raise FileNotFoundError( - f"No reduction record at {record_path} — run scripts/keck_nirc2/start_here.py " - f"first; this script reads its products." - ) - -record = json.loads(record_path.read_text()) -psf_diag = record["psf"] - -""" -__Tier A — PSF-Star Epochs__ - -The default Keck PSF path (`tier A`) turns the interleaved PSF-star visits into candidates: - -- The PSF-star frames are grouped into **epochs** — contiguous visits, split wherever the - MJD gap between frames exceeds 600 s. -- Each epoch is reduced **pipeline-identically**: the same calibration masters, the same - scaled running sky, the same one-pixmap `nirc2_native` combine at the same `final_scale`, - `pixfrac` and kernel as the science mosaic. This is the drizzled-PSF invariant — the - delivered PSF has been through *exactly* the resampling the science data have, so its - pixel-level correlations and sub-pixel structure match what convolution with the model - requires. -- **Every surviving epoch ships** as `psf_candidate_.fits` (full `psf_full_shape` - stamps), and `psf.fits` / `psf_full.fits` are cut from the *sharpest* candidate — highest - peak fraction, a Strehl proxy, since at fixed total flux a higher peak means a better - correction. - -__Grouping Into Epochs__ - -The epoch grouping is the same `group_by_time_gaps` helper the sky stage uses, with the -600 s PSF-visit gap. We can reproduce it from the cached PSF-star frame headers directly. -""" -star_paths = sorted( - p for p in (CACHE_ROOT / "b1938+666" / "psf").rglob("*.fits*") if "download" not in p.name -) - -if star_paths: - star_mjds = sorted(float(fits.getheader(p)["MJD-OBS"]) for p in star_paths) - epochs = group_by_time_gaps(star_mjds, gap_s=600.0) - print(f"{len(star_paths)} PSF-star frames -> {len(epochs)} epochs; sizes {[len(e) for e in epochs]}.") -else: - print("No cached PSF-star frames found; skipping the epoch-grouping demonstration.") - -""" -__The Vetting Gates__ - -A single-frame epoch has no outlier protection: with one frame there is nothing to -median-reject against, so a cosmic-ray hit can sit right where the star should be, and the -combine will faithfully resample it. Two physical gates catch impostors before they become -candidates, with every rejection recorded (reason and all) in provenance: - -- **Coherence**: a real AO PSF is spatially coherent — a bright 3x3 core well above the - local background scatter, with positive total flux. A hot pixel on empty sky is neither. - Starless epochs (telescope offsets, failed acquisitions) fail here with a recorded reason; - only an *all*-epochs-starless result is fatal. - -- **Diffraction sharpness**: a real PSF cannot be narrower than the telescope's diffraction - core (~45-50 mas at K' on Keck). Any "PSF" with FWHM below max(2 output pixels, 25 mas) - is rejected as a cosmic ray. This gate earned its keep on the B1938 validation run: the - CR-contaminated epochs measured 11-16 mas — far below anything optics can produce — while - the real star measured ~72 mas. -""" -for rej in psf_diag.get("rejected_epochs", []): - print(f"Rejected epoch {rej['epoch']} ({rej['n_frames']} frames): {rej['reason']}") -if not psf_diag.get("rejected_epochs"): - print("No epochs were rejected in this run.") - -""" -__The Candidates__ - -Now load what tier A shipped. The candidate list in provenance carries, per epoch: the -frame count, start MJD, peak fraction (the Strehl proxy used for selection) and the FWHM. -""" -print(f"Method: {psf_diag['method']}") -print(f"Candidates: {psf_diag['n_candidates']}, selected epoch {psf_diag['selected_epoch']}") -print(f"Selection rule: {psf_diag['selection']}") - -candidate_paths = sorted( - out_dir.glob("psf_candidate_*.fits"), - key=lambda p: int(p.stem.rsplit("_", 1)[1]), -) -candidates = [fits.getdata(p).astype(float) for p in candidate_paths] -stats = psf_diag["candidates"] - -selected_index = int(np.argmax([s["peak_fraction"] for s in stats])) - -fig, axes = plt.subplots(1, max(len(candidates), 1), figsize=(4 * max(len(candidates), 1), 4.5)) -axes = np.atleast_1d(axes) -for i, (ax, cand, stat) in enumerate(zip(axes, candidates, stats)): - ax.imshow(np.arcsinh(cand / cand.max() * 100.0), origin="lower", cmap="magma") - marker = " <- psf.fits" if i == selected_index else "" - ax.set_title( - f"candidate {i} (epoch {stat['epoch']})\n" - f"FWHM {stat['fwhm_arcsec'] * 1000:.0f} mas, peak {stat['peak_fraction']:.3f}{marker}", - fontsize=9, - ) - ax.set_xticks([]) - ax.set_yticks([]) -plt.tight_layout() -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) -cand_plot = plot_dir / "psf_candidates.png" -plt.savefig(cand_plot, dpi=150) -plt.close() -print(f"Saved candidate gallery to: {cand_plot.resolve()}") - -""" -Look at the gallery: even between epochs of the same night the core width and the halo -structure differ — that visible epoch-to-epoch variation *is* the AO PSF problem, rendered. - -__Provisional By Contract__ - -The provenance marks the Keck PSF `psf_provisional: true` — always, with no code path that -sets it false. This is deliberate contract, not a temporary limitation: - -- The peak-fraction selection is a reasonable *reduction-time* heuristic, but the question - "which candidate matches the PSF during the science frames?" is only answerable *against - the science data* — by fitting the lens model once per candidate and comparing Bayesian - evidence (the SHARP I practice). The information needed for the decision simply does not - exist at reduction time. -- Marking the PSF provisional keeps a semi-trustworthy product from masquerading as a final - one — the same honesty principle as the `injected:` block in simulated reductions. Your - modeling workflow should treat `psf.fits` as the *default starting candidate*, and the - `psf_candidate_.fits` family as the decision to make. -""" -print(f"psf_provisional: {psf_diag['psf_provisional']} (always true on the Keck path)") - -""" -__SHARP Parity__ - -The published SHARP reference for this dataset puts the PSF-candidate core FWHM at -~65-70 mas. The acceptance validation for this pipeline checks each candidate lands in a -generous 45-120 mas window (diffraction floor to poor-correction ceiling) and expects the -selected candidate near the SHARP value. -""" -for stat in stats: - fwhm_mas = stat["fwhm_arcsec"] * 1000 - in_range = 45.0 < fwhm_mas < 120.0 - print(f"Candidate epoch {stat['epoch']}: FWHM {fwhm_mas:.0f} mas — {'within' if in_range else 'OUTSIDE'} 45-120 mas.") - -""" -__The FWHM Definition Wart__ - -One honest wart: the FWHM printed above is tier A's *equivalent-area* definition — the -diameter of a circle with the same area as the above-half-maximum region. The tier-B/tier-1 -ePSF machinery measures FWHM from a radial profile instead. The two definitions agree only -for a perfectly circular PSF, which an AO PSF is not. Never compare a tier-A FWHM against a -tier-B one, or against a differently-defined literature value, at face precision — this is a -recorded open item, not a hidden inconsistency. - -__Tier B And Tier C__ - -- **Tier B — in-field ePSF (the fallback).** If a spec pins no `koa_psf_star_ids`, the Keck - path falls through to the same photutils ePSF machinery the HST path uses, building the - PSF from whatever stars the ~10" field offers (usually few to none — which is exactly why - tier A exists). A tier-B Keck PSF is *still* flagged provisional. The STARRED ePSF backend - (Michalewicz et al. 2023, JOSS; Millon et al. 2024, AJ) is directly applicable to AO data - in this same field-star role, though see the HST/JWST examples for where each backend wins. - -- **Tier C — target-based reconstruction (out of scope).** For lensed AGN, the sharpest PSF - source is the lensed quasar images themselves, fit iteratively during modeling (Chen et - al. 2016; the PSFr and STARRED two-channel deconvolution tools work in this regime). That - is a modeling-stage concern by construction — it requires the lens model — and is - permanently outside reduction scope. The reduction's job ends at honest candidates. - -__Wrap Up__ - -The AO PSF is time-variable, field-variable and unmodellable from first principles at -reduction time — so **PyAutoReduce** ships *every* pipeline-identically-reduced PSF-star -epoch as a candidate, vets each against the physics a cosmic ray cannot fake, selects a -default by Strehl proxy, and marks the result provisional so the final, evidence-based -choice happens where it belongs: against your science data, in the lens model. - -The following locations of the workspace are good places to checkout next: - -- `scripts/keck_nirc2/start_here.py`: the reduction that produced these candidates. -- `scripts/keck_nirc2/simulator.py`: injection with an epoch-specific candidate PSF as `inject_psf`. -- `scripts/hst_acs/psf.py`: the HST PSF story — star selection, ePSF tiers, `psf_star_pass`. -- `scripts/guides/output_contract.py`: `psf.fits` vs `psf_full.fits` and the drizzled-PSF invariant. - -__Env__ (Developer Only) - -Not user documentation: this section configures the automated test harness. This script -reads products and cache written by `start_here.py`, so the smoke runner skips it. - -ENV: network -""" diff --git a/workspace/scripts/keck_nirc2/simulator.py b/workspace/scripts/keck_nirc2/simulator.py deleted file mode 100644 index 1c4cfdc..0000000 --- a/workspace/scripts/keck_nirc2/simulator.py +++ /dev/null @@ -1,376 +0,0 @@ -""" -Simulator: Keck NIRC2 AO Injection -================================== - -How do you know a reduction pipeline — or the lens model you run on its products — recovers -flux faithfully? You inject a source of *known* brightness into the real data, reduce again, -and check what comes out. This script injects a synthetic Einstein-ring into the real SHARP -B1938+666 frames, immediately after the ground stages have prepared them, and runs the -identical pipeline on the injected set. - -Injection into real frames beats simulating an observation from scratch, because everything -hard about the data comes for free: the real sky and its variations, the real bad pixels, -the real dither geometry, the real AO PSF and its halo, the real correlated noise of the -combine. This is also the field's precedent for AO: Chen et al. 2021 (MNRAS 508, 755, -arXiv:2106.11060) validate AO PSF reconstruction on simulated lensed-quasar observations -built from realistic, *empirical* Keck AO PSFs, and the AIROPA project simulates NIRC2 -scenes the same way (arXiv:2207.00548) — empirical-PSF injection plus a noise model, not -end-to-end atmospheric simulation. - -Run `start_here.py` first: this script reuses its warm cache, and borrows a tier-A PSF -candidate from its output as the injection PSF. - -__Contents__ - -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root. -- **The Input Image:** A Sersic ring in pure numpy — the formula, the units, the contract. -- **Placement Without WCS:** Why injection never trusts the raw header WCS, and what it uses instead. -- **The Spec, From YAML:** `TargetSpec.from_yaml` plus `dataclasses.replace` — the injection idiom. -- **Clean Reduction:** The reference run, and the PSF candidate it donates. -- **Injected Reduction:** The same pipeline with the `inject_*` dials set. -- **No ERR Bookkeeping:** Why the Keck path updates no error extensions — noise is constructed downstream. -- **Recovery Check:** Difference the mosaics and compare recovered flux against what went in. -- **Registration Unchanged:** The measured offsets must agree between the clean and injected runs. -- **Plots:** Clean, injected, difference. -- **Wrap Up:** Summary and where to go next. - -__Imports__ -""" -import dataclasses -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import yaml -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target -from autoreduce import instruments -from autoreduce.acquire import koa - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). The clean and injected runs get separate output roots (each -run writes `//`), while the cache — which injection never mutates — is -shared with `start_here.py`. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" -OUTPUT_ROOT = WORKSPACE / "output" - -sim_root = OUTPUT_ROOT / "simulator_keck" -sim_root.mkdir(parents=True, exist_ok=True) - -""" -__The Input Image__ - -The input to injection is deliberately dumb: a plain 2-D FITS image of the source *before -the telescope* — finite, non-negative, **not** PSF-convolved (the pipeline convolves with -the epoch PSF per frame), at its own pixel scale, in the Keck adapter's injection units of -**e-/s per pixel**. No lensing code is imported to make it; a ring is just geometry. - -We build a Sersic ring: an exponential (n = 1) Sersic profile in the radial distance from a -circle of radius `r_ring`, mimicking a thin Einstein ring like B1938's: - - rho(y, x) = sqrt(y^2 + x^2) radial distance from the ring centre - I(y, x) = A * exp( -b_1 * |rho - r_ring| / r_e ) with b_1 = 1.678 (n = 1) - -(the general Sersic exponent is b_n ~ 2n - 1/3; n = 1 gives an exponential fall-off either -side of the ring crest). The image is normalised so its *total* flux is a known number of -e-/s — that number is what the recovery check hunts for. We render it at 5 mas/pixel, finer -than the native 9.942 mas, so the pipeline's flux-conserving resampling does the down-binning. -""" -INPUT_SCALE = 0.005 # arcsec/pixel of the input image (finer than native — deliberately). -RING_RADIUS = 0.30 # arcsec — the ring crest radius. -RING_R_E = 0.05 # arcsec — Sersic effective radius across the ring. -TOTAL_FLUX_EPS = 200.0 # e-/s — the known truth the recovery check compares against. - -shape = (241, 241) -yy, xx = np.mgrid[0 : shape[0], 0 : shape[1]] -cy, cx = (shape[0] - 1) / 2.0, (shape[1] - 1) / 2.0 -rho = np.hypot(yy - cy, xx - cx) * INPUT_SCALE -ring = np.exp(-1.678 * np.abs(rho - RING_RADIUS) / RING_R_E) -ring = TOTAL_FLUX_EPS * ring / ring.sum() - -input_path = sim_root / "input_sersic_ring_eps.fits" -fits.PrimaryHDU(ring.astype(np.float32)).writeto(input_path, overwrite=True) -print(f"Wrote input ring image ({TOTAL_FLUX_EPS:.0f} e-/s total) to: {input_path.resolve()}") - -""" -__Placement Without WCS__ - -On HST or JWST, injection renders the input through each exposure's full-distortion WCS — -those WCS solutions are Gaia-tied and milliarcsecond-grade. A raw NIRC2 header WCS is -**arcsecond-grade**: at 9.942 mas/pixel, trusting it would misplace the source by *tens to -hundreds of native pixels*, and by a different amount in every frame. - -So the Keck path never touches the header WCS. It reuses the same measured-offset arithmetic -the combine itself uses: a pre-pass measures `offsets_to_reference` on the prepared frames -(phase cross-correlation), reproduces the combine's deterministic mosaic geometry, places the -target at the mosaic centre (the Keck path's WCS convention), and inverts the frame<->mosaic -mapping to find each frame's injection centre. Because the injected content is placed -consistently with the measured offsets, it reinforces — never biases — the registration when -the combine re-measures it (checked empirically below). - -One consequence to internalise: **`inject_position` is honoured as an offset from the -target, not as an absolute position.** You still write it as `(ra, dec)` degrees, but only -the *difference* from `(spec.ra, spec.dec)` is used, applied by offset arithmetic on the -mosaic grid (whose axes follow the detector frame, x ~ -RA / y ~ +Dec — our circular ring is -orientation-agnostic, so this does not matter here). We offset the ring 0.8" in Dec so the -difference image lands on clean sky, well inside the 2.81" cutout. - -__The Spec, From YAML__ - -A reduction is declared, not scripted — and the cleanest injection workflow declares the -*base* spec once (here, written to YAML and loaded back with `TargetSpec.from_yaml`, the -file format you would keep per-target in a real project), then derives the injected variant -with `dataclasses.replace`. `TargetSpec` is frozen, so `replace` is the only way to vary it -— which is exactly what makes clean-vs-injected pairs trustworthy: everything not explicitly -replaced is *guaranteed* identical. - -First, pin the frame set exactly as `start_here.py` did (KOA has no association tables; see -that script for the full discovery story). -""" -RA, DEC = 294.60496, 66.81450 -FILTER = "Kp" - -adapter = instruments.get("nirc2_narrow") -query_dir = sim_root / "koa_queries" -query_dir.mkdir(parents=True, exist_ok=True) - -print("Querying KOA to pin the B1938+666 frame set (metadata only; downloads hit the warm cache)...") - -science_table = koa.query_science_frames( - RA, DEC, adapter, FILTER, query_dir, proposal_ids=None, koa_ids=None -) - -from collections import Counter - -separation_arcsec = ( - np.hypot( - (np.asarray(science_table["ra"], float) - RA) * np.cos(np.radians(DEC)), - np.asarray(science_table["dec"], float) - DEC, - ) - * 3600.0 -) -lens_table = science_table[separation_arcsec < 12.0] -star_table = science_table[separation_arcsec >= 12.0] - -setups = Counter((float(r["itime"]), int(r["coadds"])) for r in lens_table) -(modal_itime, modal_coadds), _ = setups.most_common(1)[0] -science_ids = [ - str(r["koaid"]) - for r in lens_table - if float(r["itime"]) == modal_itime and int(r["coadds"]) == modal_coadds -] -star_itimes = np.asarray(star_table["itime"], float) -short = star_itimes <= 60.0 -star_rows = star_table[short] if short.any() else star_table -star_ids = [str(k) for k in star_rows["koaid"]][:12] - -spec_yaml = { - "name": "b1938+666", - "ra": RA, - "dec": DEC, - "instrument": "nirc2_narrow", - "filter_name": FILTER, - "final_scale": 0.010, - "final_pixfrac": 1.0, - "cutout_shape": [281, 281], - "koa_science_ids": science_ids, - "koa_psf_star_ids": star_ids, -} -spec_path = sim_root / "b1938_spec.yaml" -spec_path.write_text(yaml.safe_dump(spec_yaml, sort_keys=False)) -print(f"Wrote base spec YAML to: {spec_path.resolve()}") - -base_spec = TargetSpec.from_yaml(spec_path) - -""" -__Clean Reduction__ - -The reference run. Its cache is warm from `start_here.py`, so no downloads happen — but the -full calibrate/sky/combine/noise/psf chain runs again into its own output root. -""" -print( - """ - Starting the CLEAN reduction (reference run). With a warm cache this is - reduction-only — expect minutes, not the first-run download time. - """ -) - -clean_record = reduce_target(base_spec, cache_root=CACHE_ROOT, output_root=sim_root / "clean") - -clean_dir = sim_root / "clean" / base_spec.name - -""" -The clean run also donates the injection PSF. **`inject_psf` is required on the Keck path**: -the AO PSF varies per epoch, no stable model library exists, and the tier-A candidates are -not built until after the combine — so no automatic per-frame PSF source exists at injection -time. You must hand injection an epoch-specific PSF explicitly, and the natural choice is a -`psf_candidate_.fits` from a previous reduction of the same data. We take the selected -(sharpest) candidate. -""" -psf_stats = clean_record["psf"]["candidates"] -selected_index = int(np.argmax([s["peak_fraction"] for s in psf_stats])) -inject_psf_path = clean_dir / f"psf_candidate_{selected_index}.fits" -print(f"Injection PSF: {inject_psf_path.name} (epoch {psf_stats[selected_index]['epoch']}, " - f"FWHM {psf_stats[selected_index]['fwhm_arcsec'] * 1000:.0f} mas).") - -""" -__Injected Reduction__ - -Now the injected variant via `dataclasses.replace`. The injection runs *after* the ground -stages have prepared the frames (calibrated, sky-subtracted) and *before* the combine — into -work-directory copies, never the cache. Per frame, the pipeline renders the input through -the measured-offset placement, convolves with your `inject_psf`, converts e-/s to that -frame's total electrons (x ITIME x COADDS), and adds a Poisson draw of those counts — the -injected source carries its own shot noise, like a real one. -""" -injected_spec = dataclasses.replace( - base_spec, - inject_image=str(input_path), # the plain-FITS ring, e-/s per pixel, un-convolved. - inject_pixel_scale=INPUT_SCALE, # arcsec/pixel of the input image — required with inject_image. - inject_position=(RA, DEC + 0.8 / 3600.0), # honoured as an OFFSET from the target: +0.8" in Dec. - inject_psf=str(inject_psf_path), # REQUIRED on Keck: the epoch-specific tier-A candidate. - inject_seed=0, # deterministic Poisson draws (per-frame streams derive from this + the filename). -) - -print( - """ - Starting the INJECTED reduction — identical pipeline, identical cache, - plus the synthetic ring. Expect a similar runtime to the clean run. - """ -) - -injected_record = reduce_target(injected_spec, cache_root=CACHE_ROOT, output_root=sim_root / "injected") - -injected_dir = sim_root / "injected" / base_spec.name - -""" -__No ERR Bookkeeping__ - -On HST and JWST, injection must update each exposure's ERR extension in quadrature — those -pipelines *read* propagated errors downstream. The Keck prepared frames carry **no ERR -extension**, and injection deliberately keeps it that way: the Keck noise map is -*constructed* downstream from the mosaic counts and the IVM weights -(`R x sqrt(sci/exptime + 1/wht)`), so the injected source's Poisson noise flows into the -shipped `noise_map.fits` through the mosaic counts themselves — naturally, with no -bookkeeping to get wrong. The provenance `inject` block records everything that went in: -""" -print(f"Injected total: {injected_record['inject']['total_injected_e']:.0f} e- " - f"across {len(injected_record['inject']['frames'])} frames.") -print(f"Placement: {injected_record['inject']['placement']}") -print(f"PSF source: {injected_record['inject']['psf_source']}") - -""" -__Recovery Check__ - -Difference the two mosaics — everything real cancels, leaving the injected ring plus noise — -then sum the difference in an aperture around the ring and compare with the known input -flux. The aperture centre comes from the flux-weighted centroid of the significant positive -difference pixels (a ring has no central peak, so a naive brightest-pixel centre would land -on the crest). -""" -data_clean = fits.getdata(clean_dir / "data.fits").astype(float) -data_inj = fits.getdata(injected_dir / "data.fits").astype(float) -noise_inj = fits.getdata(injected_dir / "noise_map.fits").astype(float) - -diff = data_inj - data_clean -pixel_scale = injected_record["package"]["pixel_scale"] - -significant = np.nan_to_num(diff, nan=0.0) -significant = np.where(significant > 3.0 * np.nanmedian(noise_inj), significant, 0.0) -yy, xx = np.mgrid[0 : diff.shape[0], 0 : diff.shape[1]] -centroid_y = float((significant * yy).sum() / significant.sum()) -centroid_x = float((significant * xx).sum() / significant.sum()) - -aperture = np.hypot(yy - centroid_y, xx - centroid_x) * pixel_scale <= RING_RADIUS + 3.0 * RING_R_E -recovered = float(np.nansum(diff[aperture])) -aperture_noise = float(np.sqrt(np.nansum(noise_inj[aperture] ** 2))) - -recovery_ratio = recovered / TOTAL_FLUX_EPS -print(f"Injected {TOTAL_FLUX_EPS:.0f} e-/s; recovered {recovered:.0f} e-/s in the aperture " - f"(ratio {recovery_ratio:.3f}; aperture noise {aperture_noise:.1f} e-/s).") - -""" -Expect the ratio within a few percent of 1.0 (the injected Poisson noise and the aperture -noise set the scatter). A systematic deficit would mean flux is being lost somewhere in the -chain — which is precisely what this test exists to catch. - -__Registration Unchanged__ - -The design promise from the placement section, checked: because the injected content is -placed consistently with the measured offsets, the combine's re-measured registration must -agree between the clean and injected runs. Disagreement would mean the injection biased the -phase correlation — invalidating the whole "identical pipeline" premise. -""" -off_clean = np.asarray(clean_record["drizzle"]["registration_offsets_native_pix"]) -off_inj = np.asarray(injected_record["drizzle"]["registration_offsets_native_pix"]) -max_shift = float(np.max(np.abs(off_clean - off_inj))) -print(f"Max registration shift clean vs injected: {max_shift:.3f} native pixels (expect < 0.1).") - -report = { - "injected_flux_eps": TOTAL_FLUX_EPS, - "recovered_flux_eps": recovered, - "recovery_ratio": recovery_ratio, - "aperture_noise_eps": aperture_noise, - "registration_max_offset_shift_pix": max_shift, - "total_injected_e": injected_record["inject"]["total_injected_e"], -} -report_path = sim_root / "recovery_report.json" -report_path.write_text(json.dumps(report, indent=2)) -print(f"Wrote recovery report to: {report_path.resolve()}") - -""" -__Plots__ - -Clean, injected, and the difference — the injected ring should sit 0.8" north of the real -lens, and the difference panel should show *only* the ring on a field of noise. -""" -fig, axes = plt.subplots(1, 3, figsize=(15, 5)) -scale = np.nanpercentile(data_clean, 90) -axes[0].imshow(np.arcsinh(data_clean / scale), origin="lower", cmap="magma") -axes[0].set_title("clean data.fits") -axes[1].imshow(np.arcsinh(data_inj / scale), origin="lower", cmap="magma") -axes[1].set_title("injected data.fits") -axes[2].imshow(np.arcsinh(diff / np.nanmedian(noise_inj)), origin="lower", cmap="magma") -axes[2].set_title("difference (injected ring)") -for ax in axes: - ax.set_xticks([]) - ax.set_yticks([]) -plt.tight_layout() -plot_path = sim_root / "injection_recovery.png" -plt.savefig(plot_path, dpi=150) -plt.close() -print(f"Saved injection-recovery plot to: {plot_path.resolve()}") - -""" -__Wrap Up__ - -You injected a synthetic Einstein-ring of known flux into the real SHARP B1938+666 frames — -placed by measured-offset arithmetic (never the arcsecond-grade header WCS), convolved with -an epoch-specific tier-A PSF candidate, carrying its own Poisson noise — reduced through the -byte-identical pipeline, and verified both flux recovery and that the injection left the -registration untouched. The provenance `inject` block keeps the semi-synthetic dataset from -ever masquerading as real. - -The following locations of the workspace are good places to checkout next: - -- `scripts/keck_nirc2/start_here.py`: the clean reduction this script builds on. -- `scripts/keck_nirc2/psf.py`: where the injection PSF candidates come from, and their contract. -- `scripts/hst_acs/simulator.py`: the HST injection path — WCS-based placement and ERR bookkeeping, for contrast. -- `scripts/alma/simulator.py`: the visibility-domain analogue, via CASA simobserve. - -__Env__ (Developer Only) - -Not user documentation: this section configures the automated test harness. This script -needs network access (KOA metadata queries) plus the cache and products of `start_here.py`, -so the smoke runner skips it. - -ENV: network -""" diff --git a/workspace/scripts/keck_nirc2/start_here.py b/workspace/scripts/keck_nirc2/start_here.py deleted file mode 100644 index 5156bbf..0000000 --- a/workspace/scripts/keck_nirc2/start_here.py +++ /dev/null @@ -1,464 +0,0 @@ -""" -Start Here: Keck NIRC2 AO -========================= - -Adaptive optics from the ground can beat Hubble. The sharpest published images of the strong -lens B1938+666 — the system in which Vegetti et al. 2012 (Nature 481, 341) detected a -(1.9 +/- 0.1) x 10^8 solar-mass dark satellite at z = 0.881 — were taken not from space but -from Mauna Kea, with the NIRC2 camera behind the Keck II laser-guide-star adaptive optics -system (the SHARP programme; Lagattuta et al. 2012, MNRAS 424, 2800). - -This script runs the **PyAutoReduce** Keck NIRC2 pipeline end to end on exactly that dataset: -it discovers the SHARP B1938+666 K'-band frames in the Keck Observatory Archive, pins the -frame set, reduces it from raw detector counts to a modeling-ready dataset, and closes by -loading the products into **PyAutoLens**. - -Expect the first run to take tens of minutes: it downloads the raw science frames, the -night's calibrations and the PSF-star frames from KOA, plus the geometric-distortion lookup -tables, then runs the full ground-based reduction. Re-runs use the warm cache and are much -faster. - -__Contents__ - -- **Why Adaptive Optics:** The atmosphere limits ground-based imaging to ~0.5-1.0" — AO restores the 45-50 mas diffraction limit. -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root, because the pipeline requires absolute paths. -- **KOA Discovery:** Query the Keck Observatory Archive for the SHARP B1938+666 NIRC2 frames. -- **Pinning The Frame Set:** KOA has no association tables, so the spec pins the exact KOA identifiers. -- **Target Spec:** Build the frozen `TargetSpec` that declares the whole reduction. -- **The Ground Stages:** What `reduce_target` does to raw NIRC2 frames: calibrate, sky, register, combine. -- **Distortion Epochs:** The NIRC2 distortion solution is epoch-matched: Yelda et al. 2010 before 2015-04-13, Service et al. 2016 after. -- **Run The Reduction:** One function call: `reduce_target`. -- **Provenance:** Walk the returned `reduction.json` record: weights, offsets, sky levels, PSF candidates. -- **Noise Closure:** Check the noise map against the blank-sky scatter of the mosaic itself. -- **PSF Candidates:** Every PSF-star epoch ships as a candidate, and the PSF is provisional by contract. -- **Plots:** Visualize the data, noise map and PSF. -- **Loading In PyAutoLens:** Load the products with `al.Imaging.from_fits`. -- **The Plate-Scale Caveat:** The narrow-camera plate scale in the adapter is under revision — correct it at load time. -- **Open Items:** What this pipeline honestly does not do yet. -- **Wrap Up:** Summary and where to go next. - -__Why Adaptive Optics__ - -A ground-based telescope does not deliver its diffraction limit: atmospheric turbulence -smears every image to the seeing, typically 0.5-1.0" — worse than a 10 cm telescope's -theoretical resolution, no matter how large the mirror. Adaptive optics measures the -atmospheric wavefront in real time and cancels it with a deformable mirror, restoring the -10 m Keck aperture's diffraction limit of ~45-50 mas at K' (2.1 um) — about twice as sharp -as HST in the near-infrared, because resolution scales with aperture. - -For lens fields there is rarely a natural star bright enough to measure the wavefront, so -Keck II uses a sodium laser guide star (Wizinowich et al. 2006, PASP 118, 297). The laser -cannot sense image motion, so LGS AO still needs a natural *tip-tilt star* (R ~ 18-19 or -brighter) within about a minute of arc of the target — a hard requirement that decides which -lenses can be observed at all. B1938+666 has a usable tip-tilt star (R ~ 15; Lagattuta et -al. 2012), which is part of why it became the SHARP flagship. - -The delivered correction is partial: on typical lens fields the Strehl ratio is ~10-30% and -the PSF FWHM ~60-90 mas (van Dam et al. 2006, PASP 118, 310, report 30-40% at K on bright, -on-axis stars). NIRC2's *narrow* camera samples this PSF at 9.942 mas/pixel — comfortably -Nyquist-sampling the ~50 mas diffraction core, which is exactly why lens programmes use it -(https://www2.keck.hawaii.edu/inst/nirc2/). The scientific payoff is resolution-limited -science: the Vegetti et al. 2012 substructure detection, SHARP's flux-ratio-anomaly studies -(Hsueh et al. 2016, MNRAS-L 463, L51; Hsueh et al. 2017, MNRAS 469, 3713) and AO-based -time-delay cosmography (Chen et al. 2019, MNRAS 490, 1743, arXiv:1907.02533) all rest on -Keck AO imaging reduced with care. - -__Imports__ - -**PyAutoReduce** exposes exactly two names — a frozen `TargetSpec` and the `reduce_target` -function. The KOA query helpers used for discovery live in `autoreduce.acquire.koa`. -""" -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np - -from autoreduce import TargetSpec, reduce_target -from autoreduce.acquire import koa -from autoreduce import instruments - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths: its combine step -changes the working directory internally, so relative paths would break. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" # downloaded KOA frames + distortion tables (re-used across runs) -OUTPUT_ROOT = WORKSPACE / "output" # reduced datasets, one folder per target - -""" -__KOA Discovery__ - -All Keck data flows through the Keck Observatory Archive (KOA, https://koa.ipac.caltech.edu/), -and **PyAutoReduce** talks to it through PyKOA -(https://github.com/KeckObservatoryArchive/PyKOA), the archive's own Python client — the -KOA analogue of `astroquery.mast`. KOA serves *raw level-0 frames only* for NIRC2: unlike -HST or JWST there are no archive-calibrated products, so every calibration step below is the -pipeline's own job. - -The SHARP B1938+666 K' data were taken UT 2010 June 29-30 with the narrow camera -(14,760 s total; Lagattuta et al. 2012). We cone-query KOA around the lens position for -narrow-camera K' object frames. The query is cheap metadata — the frame downloads happen -later, inside `reduce_target`, against the cache. -""" -RA, DEC = 294.60496, 66.81450 # B1938+666 (SHARP I) -FILTER = "Kp" - -adapter = instruments.get("nirc2_narrow") - -work_dir = OUTPUT_ROOT / "b1938+666" / "koa_queries" -work_dir.mkdir(parents=True, exist_ok=True) - -print("Querying KOA for SHARP B1938+666 NIRC2 narrow-camera K' frames (metadata only)...") - -science_table = koa.query_science_frames( - RA, # target RA (degrees) for the cone query. - DEC, # target Dec (degrees). - adapter, # the nirc2_narrow adapter selects the camera in the ADQL query. - FILTER, # K' — KOA's filter string for the NIRC2 K-prime filter. - work_dir, # where the raw query VOTables are written. - proposal_ids=None, # a cone query is enough here; a program ID would narrow it further. - koa_ids=None, # no pinned ids yet — this IS the discovery pass that finds them. -) - -print(f"KOA returned {len(science_table)} object frames at the B1938+666 pointing.") - -""" -__Pinning The Frame Set__ - -KOA has **no association tables**: nothing in the archive says "these 39 frames are one -science stack". An HST reduction can hand MAST an association and trust it; a Keck reduction -must *pin the exact frame set itself*, which is what the `koa_science_ids` and -`koa_psf_star_ids` dials on `TargetSpec` are for. Pinning makes the reduction reproducible — -the same spec always reduces the same frames. - -Two wrinkles in the SHARP data, both worth knowing about for your own targets: - -1. The cone result contains *two pointing clusters*, and both carry the tip-tilt star's name - in `targname` (a SHARP convention): the lens dithers themselves, and interleaved visits to - the PSF/tip-tilt star ~20" away — SHARP's dedicated PSF-star strategy (Lagattuta et al. - 2012). We split them by angular separation from the lens. - -2. The pipeline refuses mixed ITIME/COADDS within one science stack (a single calibration - set must match every frame), so we pin the *modal* setup — for SHARP B1938 that is the - 180 s x 1 science frames, which drops the short acquisition frames. For the PSF star we - prefer the *shortest* integrations: a K ~ 13-14 star saturates the narrow camera in - 180 s, and the short frames exist precisely to keep its core unsaturated. -""" -from collections import Counter - -separation_arcsec = ( - np.hypot( - (np.asarray(science_table["ra"], float) - RA) * np.cos(np.radians(DEC)), - np.asarray(science_table["dec"], float) - DEC, - ) - * 3600.0 -) -lens_table = science_table[separation_arcsec < 12.0] -star_table = science_table[separation_arcsec >= 12.0] - -setups = Counter((float(r["itime"]), int(r["coadds"])) for r in lens_table) -(modal_itime, modal_coadds), _ = setups.most_common(1)[0] -print(f"Science setups found {dict(setups)}; pinning the modal {modal_itime}s x {modal_coadds}.") - -science_ids = tuple( - str(r["koaid"]) - for r in lens_table - if float(r["itime"]) == modal_itime and int(r["coadds"]) == modal_coadds -) - -star_itimes = np.asarray(star_table["itime"], float) -short = star_itimes <= 60.0 -star_rows = star_table[short] if short.any() else star_table -star_ids = tuple(str(k) for k in star_rows["koaid"])[:12] - -print(f"Pinned {len(science_ids)} science frames and {len(star_ids)} PSF-star frames.") - -""" -__Target Spec__ - -A **PyAutoReduce** reduction is *declared, not scripted*: everything about it lives in one -frozen `TargetSpec`, and the pipeline is a pure function of the spec plus the archive. The -non-default dials below are the Keck-specific ones. -""" -spec = TargetSpec( - name="b1938+666", # output folder name under OUTPUT_ROOT. - ra=RA, # target RA (degrees) — the mosaic WCS reference and cutout centre. - dec=DEC, # target Dec (degrees). - instrument="nirc2_narrow", # the 9.942 mas/pixel camera; nirc2_wide is registered but fails loudly at combine. - filter_name=FILTER, # K' — selects the flats and labels the products. - final_scale=0.010, # output pixel scale in arcsec: 10 mas, the SHARP convention (Chen et al. 2019). - final_pixfrac=1.0, # drizzle drop size; 1.0 (shift-and-add) is robust for the near-native resampling here. - cutout_shape=(281, 281), # 2.81" on a side at 10 mas — generous around the ~0.45" Einstein radius. - koa_science_ids=science_ids, # the pinned raw science frames (KOA has no association tables). - koa_psf_star_ids=star_ids or None, # the pinned PSF-star frames — the tier-A PSF strategy (see psf.py). - sky_window=9, # running-sky window: sky structure from the 9 temporally nearest frames. -) - -""" -__The Ground Stages__ - -Space-based archives hand you calibrated exposures; KOA hands you raw detector reads. The -pipeline therefore runs two stages that HST/JWST reductions never see, before anything is -combined. There is no maintained community pipeline to wrap — KAI, the standard NIRC2 DRP -(Lu et al. 2022, https://doi.org/10.5281/zenodo.6677744), is Python 2.7 + IRAF/PyRAF — so -**PyAutoReduce** implements the ground stages natively in numpy/astropy: - -- **Calibrate** — each raw frame (stored as the per-coadd average in DN) is converted to - total electrons (gain 4.0 e-/DN x COADDS), dark-subtracted when the night has matched - darks, flat-fielded (lamp-on minus lamp-off dome flats, unit median), and bad pixels (hot - in the dark, dead in the flat) are carried as NaN into zero combine weight. The SHARP - recipe is flat + sky only — darkless nights are recorded, never silently ignored. - -- **Sky** — the defining ground-based NIR step. At K' the sky at Mauna Kea runs - ~13-13.5 mag/arcsec^2, orders of magnitude brighter per pixel than a lensed arc, and it - varies on minutes timescales — faster than any calibration. The pipeline estimates a - *scaled running sky* from the temporally adjacent, object-masked, dithered frames of the - field itself (see `step_by_step.py` for the mechanics, and Vaduvescu & McCall 2004, PASP, - astro-ph/0404337, for the method background). - -- **Register + Combine** — NIRC2 header pointing is only arcsecond-grade, so relative frame - offsets are *measured* by phase cross-correlation, then distortion correction, - registration and the native-to-10-mas rescale enter the `drizzle` resampler as **one - pixel mapping** — the data are resampled exactly once. Per-frame weights are the inverse - background variance, so the accumulated weight map is the IVM the shared noise recipe - expects. - -- **Noise + PSF** — the same noise recipe as HST/JWST, `R x sqrt(sci/exptime + 1/wht)`, - with the Casertano et al. 2000 (AJ 120, 2747) correlated-noise factor R carried over - unchanged because the resampling engine is identical; and the tier-A PSF-star treatment, - described below. - -__Distortion Epochs__ - -NIRC2's geometric distortion is corrected with published lookup tables -(https://www2.keck.hawaii.edu/inst/nirc2/dewarp.html), and the right table depends on *when* -your data were taken: the AO bench was realigned on 2015-04-13, so the pipeline -epoch-matches automatically — Yelda et al. 2010 (ApJ 725, 331; ~1 mas accuracy) before the -boundary, Service et al. 2016 (PASP 128, 095004; ~1.1 mas) after. The B1938+666 data are -from 2010, so this reduction uses the Yelda solution; the tables are downloaded once, -checksummed, and recorded in provenance. Frame sets spanning the boundary are rejected -loudly rather than mixed. - -__Run The Reduction__ - -One call runs everything: acquisition (science + calibrations + PSF stars + distortion -tables), calibrate, sky, register, combine, noise, PSF, package. -""" -print( - f""" - Starting reduce_target for {spec.name}. - - First run: downloads {len(science_ids)} science frames, the night's calibrations and - {len(star_ids)} PSF-star frames from KOA (~hundreds of MB), then reduces — expect - tens of minutes. Re-runs hit the warm cache under {CACHE_ROOT} and skip the downloads. - """ -) - -record = reduce_target(spec, cache_root=CACHE_ROOT, output_root=OUTPUT_ROOT) - -out_dir = OUTPUT_ROOT / spec.name -print(f"Reduction complete. Products in: {out_dir}") - -""" -__Provenance__ - -`reduce_target` returns the full `reduction.json` record — every decision the pipeline made, -so the reduction can be audited (and re-derived) later. The highlights for a Keck run: -""" -print(f"Frames combined: {record['drizzle']['n_exposures']}") -print(f"Total exposure time: {record['drizzle']['total_exptime']:.0f} s") -print(f"Distortion epoch: {record['acquire']['distortion_epoch']} (Yelda et al. 2010 expected)") -print(f"Casertano R: {record['noise']['correlated_noise_factor']:.3f}") -print(f"Weight uniformity: {record['drizzle']['weight_uniformity_cutout']}") -print(f"Calibration: {record['calibrate']}") -print(f"Per-frame sky levels (e-), first 5: {record['sky']['sky_levels_e'][:5]}") - -""" -The measured registration offsets are provenance too — they are the astrometric truth of -this reduction (the headers are not), and the frame-products and injection machinery reuse -them. -""" -offsets = record["drizzle"]["registration_offsets_native_pix"] -print(f"Registration offsets (native pixels), first 5 frames: {offsets[:5]}") - -""" -__Noise Closure__ - -Is the shipped noise map *right*? The pipeline's internal check compares the blank-sky -scatter measured off the mosaic itself against the noise map's background floor. One subtlety -makes the comparison honest: the shipped map is the *decorrelated-equivalent* noise (x R, the -chi^2-correct value for model fitting), while the measurable pixel-to-pixel RMS of a drizzled -mosaic is correlation-*suppressed* by ~1/R (Casertano et al. 2000; Fruchter & Hook 2002). The -apples-to-apples statistic is therefore - - empirical RMS x R^2 / predicted floor ~ 1 - -This closure is what keeps the detector constants honest rather than trusted: the per-frame -budget (sky + dark + MCDS read noise) predicts 62.0 e- per frame on these MCDS-32 180 s K' -frames versus 62-64 e- measured empirically, and the mosaic-level closure lands at ~0.84 on -B1938 — order-unity, where a unit error (gain, coadds, cps) would miss by x6-x40. -""" -from astropy.io import fits - -from autoreduce.noise.rms import empirical_background_rms - -data = fits.getdata(out_dir / "data.fits").astype(float) -noise = fits.getdata(out_dir / "noise_map.fits").astype(float) - -r_factor = float(record["noise"]["correlated_noise_factor"]) -empirical = empirical_background_rms(data) -predicted_floor = float(np.nanmedian(noise[noise < np.nanpercentile(noise, 50)])) -closure = empirical * r_factor**2 / predicted_floor - -print(f"Blank-sky closure (empirical x R^2 / predicted): {closure:.2f} (~1 expected; 0.84 on the reference run)") - -""" -__PSF Candidates__ - -The AO PSF changes with the atmosphere from visit to visit, so no single reduction-time PSF -can be trusted as final. The tier-A treatment reduces each PSF-star *epoch* through the -identical calibrate/sky/combine path as the science and ships **every epoch** as -`psf_candidate_.fits`; `psf.fits` is cut from the sharpest one (a peak-fraction Strehl -proxy), and the provenance carries `psf_provisional: true` — final PSF selection belongs to -lens modeling, by Bayesian evidence over the candidates (the SHARP I practice). The full -story, including the cosmic-ray vetting gates, is in `psf.py` in this folder. -""" -psf_diag = record["psf"] -print(f"PSF method: {psf_diag['method']}") -print(f"PSF provisional: {psf_diag['psf_provisional']} (always true on the Keck path)") -print(f"Candidates: {psf_diag.get('n_candidates')}, selected epoch {psf_diag.get('selected_epoch')}") -for cand in psf_diag.get("candidates", []): - print( - f" epoch {cand['epoch']}: {cand['n_frames']} frames, " - f"FWHM {cand['fwhm_arcsec'] * 1000:.0f} mas, peak fraction {cand['peak_fraction']:.3f}" - ) - -""" -__Plots__ - -Now look at what you produced. The data are plotted with arcsinh scaling (linear near zero, -logarithmic at the bright lens galaxy), the noise map linearly, and the PSF with arcsinh to -bring up the faint AO halo around the diffraction core. -""" -psf = fits.getdata(out_dir / "psf.fits").astype(float) - -plot_dir = out_dir / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -fig, axes = plt.subplots(1, 3, figsize=(15, 5)) - -axes[0].imshow(np.arcsinh(data / np.nanpercentile(data, 90)), origin="lower", cmap="magma") -axes[0].set_title("data.fits (arcsinh, e-/s)") - -axes[1].imshow(noise, origin="lower", cmap="viridis", vmax=np.nanpercentile(noise, 99)) -axes[1].set_title("noise_map.fits (RMS e-/s)") - -axes[2].imshow(np.arcsinh(psf / psf.max() * 100.0), origin="lower", cmap="magma") -axes[2].set_title("psf.fits (arcsinh)") - -for ax in axes: - ax.set_xticks([]) - ax.set_yticks([]) - -plt.tight_layout() -plot_path = plot_dir / "start_here_products.png" -plt.savefig(plot_path, dpi=150) -plt.close() -print(f"Saved product summary plot to: {plot_path.resolve()}") - -""" -You should see the B1938+666 Einstein ring — a near-complete ~0.9"-diameter infrared ring -around the lens galaxy. This is the image in which a 10^8 solar-mass dark subhalo is -detectable; everything the reduction did (sky, distortion, one-pass resampling, honest -noise, candidate PSFs) exists to keep that signal trustworthy. - -__Loading In PyAutoLens__ - -The products load directly into **PyAutoLens** — that is the whole point of the output -contract. The import is guarded so this reduction script stands alone if **PyAutoLens** is -not installed. - -__The Plate-Scale Caveat__ - -One honest correction is applied at load time. The `nirc2_narrow` adapter carries the -9.942 mas/pixel plate scale, but the best pre-2015 narrow-camera calibration is -9.952 mas/pixel (Yelda et al. 2010 — the raw headers agree). The mosaic was resampled -assuming 9.942, so one output pixel is truly 10 mas x (9.952 / 9.942) = 10.010 mas. Feeding -the corrected scale to **PyAutoLens** makes the Einstein radius physical without touching -the shipped FITS products. The adapter value is under revision (an epoch-aware fix is a -gated source change); until it lands, apply this correction whenever absolute angular scales -matter. -""" -record_from_disk = json.loads((out_dir / "reduction.json").read_text()) - -pixel_scale_asbuilt = record_from_disk["package"]["pixel_scale"] # what the reduction assumed (10 mas). -pixel_scale_true = pixel_scale_asbuilt * (9.952 / 9.942) # Yelda et al. 2010 plate-scale correction. - -print(f"As-built pixel scale: {pixel_scale_asbuilt}\" -> corrected: {pixel_scale_true:.6f}\"") - -try: - import autolens as al -except ImportError: - al = None - print( - "PyAutoLens is not installed (pip install autolens), so the loading demonstration " - "is skipped — the reduction itself is complete and the products are on disk." - ) - -if al is not None: - dataset = al.Imaging.from_fits( - data_path=out_dir / "data.fits", # the drizzled K' cutout, e-/s. - noise_map_path=out_dir / "noise_map.fits", # matching RMS map (already x R). - psf_path=out_dir / "psf.fits", # the sharpest tier-A candidate — provisional, see psf.py. - pixel_scales=pixel_scale_true, # the plate-scale-corrected value, NOT the packaged one. - ) - print(f"Loaded al.Imaging with shape {dataset.data.shape_native} at {pixel_scale_true:.6f}\"/pixel.") - -""" -For real modeling of this system the SHARP practice is to fit at 20 mas via 2x2 binning -(Chen et al. 2019's efficiency trick) and to select among the PSF candidates by Bayesian -evidence — see `autolens_workspace` for the modeling side. - -__Open Items__ - -Honesty about what this path does not do yet, so you are never surprised: - -- **Wide camera**: the published distortion solutions are narrow-camera only, so - `nirc2_wide` is registered for spec completeness but *fails loudly at combine*. -- **No cosmic-ray rejection at combine**: the drizzle accumulates without an outlier pass. - The ~39-frame science stack dilutes CRs by the weight sum and the bad-pixel policy covers - the cutout, but a median/blot-style rejection is the principled fix and remains open. - (Single-frame PSF epochs are protected instead by the sharpness vetting — see `psf.py`.) -- **Detector-frame orientation**: the output WCS is TAN at the target with the detector's - orientation — there is no north-up resampling yet; rotator-angle handling awaits the - astrometric-parity validation. -- **Plate scale**: as above — corrected at load, adapter fix pending. - -__Wrap Up__ - -You reduced the SHARP B1938+666 Keck AO dataset from raw KOA frames to a modeling-ready -**PyAutoLens** dataset: pinned frame set, native ground-based calibration and running-sky -subtraction, epoch-matched distortion in a single resampling pass, a closure-checked noise -map and a candidate-based provisional PSF. - -The following locations of the workspace are good places to checkout next: - -- `scripts/keck_nirc2/step_by_step.py`: every ground stage above, demonstrated standalone on the cached frames with the literature. -- `scripts/keck_nirc2/psf.py`: the AO PSF problem end to end — candidates, vetting gates, the provisional contract. -- `scripts/keck_nirc2/simulator.py`: injecting a synthetic lensed source into the real prepared frames to test recovery. -- `scripts/guides/noise_maps.py`: the shared noise recipe, Casertano R and the closure statistics across instruments. -- `scripts/guides/output_contract.py`: the 4-file + `reduction.json` contract every instrument path emits. - -__Env__ (Developer Only) - -Not user documentation: this section configures the automated test harness. This script -needs network access (KOA, distortion tables) and the heavy `[keck]` extras, so the smoke -runner skips it. - -ENV: network -""" diff --git a/workspace/scripts/keck_nirc2/step_by_step.py b/workspace/scripts/keck_nirc2/step_by_step.py deleted file mode 100644 index c1d7bac..0000000 --- a/workspace/scripts/keck_nirc2/step_by_step.py +++ /dev/null @@ -1,408 +0,0 @@ -""" -Step By Step: Keck NIRC2 AO -=========================== - -`start_here.py` reduced the SHARP B1938+666 K' dataset with one call to `reduce_target`. -This script opens that call up: every ground-based stage — calibration, sky subtraction, -registration, the single-pass combine, the detector noise budget — demonstrated standalone -on the cached raw frames, with the literature that motivates each step. - -Run `start_here.py` first: this script works off the frames it cached under -`cache/b1938+666/` (science, calibrations, PSF stars), so it re-downloads nothing. - -The ground stages exist because KOA hands you *raw detector reads*. An HST reduction starts -from `_flc` files that STScI has already bias-corrected, dark-subtracted, flat-fielded and -CR-flagged; a Keck reduction starts from voltages. Everything a space-telescope archive does -for you, this script does in front of you. - -__Contents__ - -- **Imports:** Import the required Python libraries. -- **Paths:** Anchor every path to the workspace root. -- **Raw Frames And KOA:** What a KOA level-0 NIRC2 frame is, and what its header carries. -- **Calibration Matching:** Darks matched to ITIME/COADDS, flats within +/-14 days, and the SHARP flat+sky-only honesty. -- **Building Master Calibrations:** `build_calibrations` — master flat, optional master dark, bad-pixel mask. -- **Calibrating A Frame:** `calibrate_frame` — DN to total electrons, flat-fielded, bad pixels as NaN. -- **Why The Sky Dominates:** The K' sky at Mauna Kea is orders of magnitude brighter per pixel than a lensed arc. -- **Grouping By Time:** `group_by_time_gaps` splits the frame sequence into contiguous runs before any sky is estimated. -- **Running Sky Subtraction:** The scaled running sky, and why it takes two passes. -- **Registration:** `phase_offset` — measured sub-pixel offsets, because header pointing is arcsecond-grade. -- **One Pixmap, One Resampling:** Distortion + registration + rescale as a single drizzle mapping. -- **The Detector Noise Budget:** Gain, MCDS read noise, dark current — and the per-frame budget checked empirically. -- **Casertano R And The Closure:** The correlated-noise factor and the mosaic-level closure statistic. -- **Why Not KAI:** The community pipeline is Python 2.7 + IRAF — why the stages are implemented natively. -- **Wrap Up:** Summary and where to go next. - -__Imports__ -""" -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits - -from autoreduce import instruments -from autoreduce.calibrate import build_calibrations, calibrate_frame, load_calibration_sets -from autoreduce.sky import group_by_time_gaps, running_sky_subtract -from autoreduce.align.registration import phase_offset -from autoreduce.noise.rms import casertano_r, empirical_background_rms - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). We point at the cache `start_here.py` populated. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" -OUTPUT_ROOT = WORKSPACE / "output" - -cache_dir = CACHE_ROOT / "b1938+666" -science_paths = sorted( - p for p in (cache_dir / "science").rglob("*.fits*") if "download" not in p.name -) -calib_paths = sorted( - p for p in (cache_dir / "cals").rglob("*.fits*") if "download" not in p.name -) - -if not science_paths: - raise FileNotFoundError( - f"No cached science frames under {cache_dir / 'science'} — run " - f"scripts/keck_nirc2/start_here.py first; this script teaches off its cache." - ) - -print(f"Cached science frames: {len(science_paths)}; calibration frames: {len(calib_paths)}.") - -""" -__Raw Frames And KOA__ - -Every frame here came from the Keck Observatory Archive (https://koa.ipac.caltech.edu/) via -PyKOA (https://github.com/KeckObservatoryArchive/PyKOA), queried through its TAP/ADQL -service. KOA serves NIRC2 as *raw level-0* frames — the level-1 quick-look products are not -science grade — so the header is the reduction's ground truth. The keywords that drive -everything downstream: - -- `ITIME` — integration time per coadd (seconds). -- `COADDS` — number of coadds averaged on chip; the stored pixel value is the *per-coadd - average* in DN, so total electrons = DN x gain x COADDS. -- `MJD-OBS` — observation time; temporal order defines the running sky and PSF epochs, and - the date routes the distortion solution (Yelda et al. 2010 before 2015-04-13, Service et - al. 2016 after). -- `SAMPMODE` / `MULTISAM` — the readout mode; MCDS/Fowler sampling cuts the effective read - noise (the noise-budget section below). -- `KOAIMTYP` — frame type (`object`, `dark`, `flatlamp`, `flatlampoff`, `domeflat`), which - is how calibration frames are sorted. -""" -facts = [] -for path in science_paths: - header = fits.getheader(path) - facts.append( - { - "path": path, - "mjd": float(header["MJD-OBS"]), - "itime": float(header["ITIME"]), - "coadds": int(header["COADDS"]), - "sampmode": int(header.get("SAMPMODE", 2)), - "multisam": int(header.get("MULTISAM", 1)), - } - ) -facts.sort(key=lambda f: f["mjd"]) - -science_itime = facts[0]["itime"] -science_coadds = facts[0]["coadds"] -print( - f"Science setup: ITIME {science_itime}s x COADDS {science_coadds}, " - f"SAMPMODE {facts[0]['sampmode']} / MULTISAM {facts[0]['multisam']} (MCDS)." -) - -""" -__Calibration Matching__ - -Ground-based NIR calibration has its own matching rules, and the acquire stage applied them -when it filled the cache: - -- **Darks must match ITIME and COADDS.** Dark current and bias structure scale with the - integration pattern, so a 180 s x 1 science frame needs 180 s x 1 darks — nothing else - subtracts cleanly. Darks are *optional by design*: the running sky is estimated from - frames that carry the same dark signal, so sky subtraction removes dark and sky together. - That is the SHARP recipe — flat + sky only — and the pipeline records a darkless - calibration honestly (`dark_subtraction: false` in provenance) rather than failing or - staying silent. - -- **Flats are mandatory, within +/-14 days.** NIR flat fields are stable over weeks, and - SHARP's own science nights routinely carry only darks — so when the night has no flats, - the nearest flat-bearing night inside a 14-day window is used and recorded. K-band dome - flats carry a thermal pedestal (the warm dome glows at 2 um), so the master flat is - *lamp-on minus lamp-off* when off-frames exist: the difference isolates the lamp - illumination and removes the thermal print-through. - -- **Bad pixels come from the calibrations themselves**: hot pixels stand out in the dark, - dead pixels are unresponsive in the flat. They are *propagated, not interpolated* — NaN in - the calibrated frame becomes zero weight in the combine, exactly how HST handles - CR-flagged pixels. -""" -darks, flat_on, flat_off = load_calibration_sets( - calib_paths, # every cached calibration frame; sorted by KOAIMTYP internally. - science_itime=science_itime, # only darks matching the science ITIME are kept. - science_coadds=science_coadds, # ... and the science COADDS. -) -print(f"Calibration sets: {len(darks)} matched darks, {len(flat_on)} lamp-on flats, {len(flat_off)} lamp-off flats.") - -""" -__Building Master Calibrations__ - -`build_calibrations` median-stacks each set into the masters and derives the bad-pixel mask. -Its provenance dict is what `reduce_target` records under `calibrate` in `reduction.json`. -""" -calib = build_calibrations( - dark_frames=darks, # median-stacked into the master dark (kept in DN; scaled by gain at use). - flat_on_frames=flat_on, # median lamp-on stack. - flat_off_frames=flat_off, # median lamp-off stack, subtracted to remove the thermal pedestal. - hot_sigma=5.0, # pixels > 5 sigma above the dark median are flagged hot. - dead_flat_threshold=0.5, # pixels below 50% response in the unit-median flat are flagged dead. -) -print(f"Master calibrations built: {calib.provenance}") - -""" -__Calibrating A Frame__ - -`calibrate_frame` converts one raw frame to *total electrons*: DN x gain x COADDS, dark -subtracted (when a master dark exists), divided by the unit-median flat, with bad pixels set -to NaN so no downstream stage can use them by accident. Working in total electrons is what -keeps Poisson statistics computable downstream — the noise stage needs real counts. - -The gain is an adapter-owned detector constant (4.0 e-/DN), validated by the noise closure -rather than trusted blindly. -""" -adapter = instruments.get("nirc2_narrow") -detector = adapter.ground_detector() - -n_demo = min(12, len(facts)) -raw_demo = [fits.getdata(f["path"]).astype(np.float64) for f in facts[:n_demo]] - -calibrated = [ - calibrate_frame( - raw, # the raw per-coadd-average DN array. - calib, # the master calibrations built above. - gain_e_per_dn=detector.gain_e_per_dn, # 4.0 e-/DN — adapter-owned, closure-validated. - coadds=science_coadds, # total e- needs the coadd count back. - ) - for raw in raw_demo -] - -print( - f"Calibrated {n_demo} frames: median level {np.nanmedian(calibrated[0]):.0f} e- " - f"(sky pedestal), {int(np.isnan(calibrated[0]).sum())} bad pixels as NaN." -) - -""" -__Why The Sky Dominates__ - -That median level *is the problem this stage exists for*. The K' sky at Mauna Kea runs -~13-13.5 mag/arcsec^2 — orders of magnitude brighter than a lensed arc in every pixel. Below -~2 um the sky is OH airglow, varying on minutes timescales; beyond ~2.2 um thermal emission -takes over. Either way it changes faster than any calibration you could take before or -after, so it must be estimated *from the science sequence itself*: the temporally adjacent, -dithered frames of the same field, with sources masked (the standard craft; see Vaduvescu & -McCall 2004, PASP, astro-ph/0404337, for a methods treatment). Dithering is what makes this -possible — the target lands on different pixels each frame, so a running median over -neighbours sees mostly sky — and it simultaneously averages down bad pixels and flat errors. - -__Grouping By Time__ - -Window adjacency is positional, so the frame set must be *temporally contiguous* before any -running sky is estimated — a window spanning a gap would silently borrow sky from a -different night or visit. `group_by_time_gaps` splits the MJD-sorted sequence at gaps: the -pipeline uses 3600 s for science sequences and 600 s for the interleaved PSF-star visits -(the same 600 s gap that defines PSF epochs — see `psf.py`). -""" -mjds = [f["mjd"] for f in facts[:n_demo]] -groups = group_by_time_gaps(mjds, gap_s=3600.0) -print(f"{n_demo} frames -> {len(groups)} contiguous group(s); sizes {[len(g) for g in groups]}.") - -""" -__Running Sky Subtraction__ - -`running_sky_subtract` implements the *scaled* running sky, in two passes: - -1. For each frame, take the `window` temporally nearest other frames (a frame never - contributes to its own sky), mask their sources, and normalise each to unit median. The - median of that stack is the sky *structure* — fringes, illumination gradients — free of - the overall level. - -2. Multiply the structure by the frame's *own* masked median — the sky *level*. This scaling - is the difference between a running sky and a *scaled* running sky: the K' level drifts - on minutes timescales, so borrowing the level from neighbours biases the frames at the - edges of the sequence. Structure from neighbours, level from yourself. - -The two passes refine the object mask: once the sky pedestal is gone, fainter source wings -emerge in the first-pass residuals, so the mask is rebuilt and the sky re-estimated. The -per-frame sky levels are returned in the provenance — the noise stage's background variance -is built from exactly these numbers. -""" -group = groups[0] -group_frames = [calibrated[i] for i in group] - -print(f"Running scaled sky subtraction over {len(group_frames)} frames (window 9, two passes)...") - -subtracted, sky_prov = running_sky_subtract( - group_frames, # calibrated frames in electrons, NaN bad pixels, temporal order. - window=9, # sky structure from the 9 nearest neighbours (the TargetSpec sky_window dial). - n_sigma=3.0, # object-mask threshold above the sky, in MAD-sigmas. -) - -print(f"Recipe: {sky_prov['recipe']}") -print(f"Per-frame sky levels (e-): {[f'{s:.0f}' for s in sky_prov['sky_levels_e']]}") - -fig, axes = plt.subplots(1, 2, figsize=(12, 6)) -scale = np.nanpercentile(group_frames[0], 99) -axes[0].imshow(np.arcsinh(group_frames[0] / scale), origin="lower", cmap="magma") -axes[0].set_title("calibrated (sky pedestal in)") -axes[1].imshow(np.arcsinh(subtracted[0] / np.nanstd(subtracted[0])), origin="lower", cmap="magma") -axes[1].set_title("sky-subtracted") -for ax in axes: - ax.set_xticks([]) - ax.set_yticks([]) -plt.tight_layout() -plot_dir = OUTPUT_ROOT / "b1938+666" / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) -sky_plot = plot_dir / "step_by_step_sky.png" -plt.savefig(sky_plot, dpi=150) -plt.close() -print(f"Saved sky-subtraction comparison to: {sky_plot.resolve()}") - -""" -__Registration__ - -NIRC2 header pointing is *arcsecond-grade* — at 9.942 mas/pixel, an arcsecond is a hundred -native pixels. The headers tell you roughly where the telescope pointed; they cannot align -frames. Relative offsets are therefore *measured* from the data by phase cross-correlation -(`phase_offset`, pure numpy): the cross-power spectrum of two frames, amplitude-normalised -(whitened) so the correlation peak is sharp and contrast-independent, with a parabolic fit -refining the peak to sub-pixel precision. Bad pixels (NaN) are zero-filled for the transform -only — a fraction of dead pixels does not move the peak. - -The measured offsets are the astrometric truth of a Keck reduction, and the pipeline records -them in provenance (`registration_offsets_native_pix`); the injection machinery -(`simulator.py`) reuses the same arithmetic rather than ever trusting the header WCS. -""" -dy, dx = phase_offset( - subtracted[0], # the reference frame. - subtracted[1], # the frame to align; returns the (dy, dx) that shifts it onto the reference. -) -print(f"Measured dither offset frame 1 -> 0: ({dy:.2f}, {dx:.2f}) native pixels (~{np.hypot(dy, dx) * 9.942:.0f} mas).") - -""" -__One Pixmap, One Resampling__ - -The combine backend (`nirc2_native`) now has three geometric transforms to apply to each -frame: the distortion correction (the Yelda/Service lookup tables), the measured -registration offset, and the native-to-final rescale (9.942 mas -> 10 mas). The crucial -design decision is that they enter the `drizzle` resampler as **one pixel mapping** — each -input pixel is mapped straight to its output location through all three at once, and the -data are resampled exactly once. - -Why it matters: every resampling correlates the noise between neighbouring pixels and blurs -the PSF a little. Dewarp-then-shift-then-rescale as three separate interpolations would pay -that price three times, and the correlated-noise bookkeeping would be intractable. One -pixmap means the standard Casertano et al. 2000 (AJ 120, 2747) analysis applies verbatim — -exactly how drizzlepac treats ACS distortion inside AstroDrizzle, with the same resampling -engine (the standalone `drizzle` package is the engine inside drizzlepac and the jwst -pipeline). Distortion correction is where correlated noise enters a NIRC2 reduction; doing -it once, jointly, keeps it accountable. - -Each frame's weight in the combine is its *inverse background variance* (sky + dark + read -noise, in cps^2) — so the accumulated weight map is exactly the IVM the shared noise recipe -`R x sqrt(sci/exptime + 1/wht)` expects, and the noise stage needs no Keck-specific branch -at all. - -__The Detector Noise Budget__ - -The weights and the noise map both rest on the per-frame background variance, built from -three adapter-owned detector constants — gain 4.0 e-/DN, CDS read noise 38 e-, dark current -0.1 e-/s — and the frame's own header facts: - - var_bkg [e-^2] = sky_e + dark_e_per_s x ITIME x COADDS + RN_eff^2 x COADDS - -The read noise is *sampling-mode aware*: in MCDS/Fowler-M mode (SAMPMODE 3) the detector -averages MULTISAM read pairs, cutting the read-noise variance by ~1/MULTISAM, so -RN_eff = 38 / sqrt(MULTISAM). For the SHARP B1938 K' frames (MCDS-32, 180 s) the budget -predicts 62.0 e- per frame where plain CDS would predict 72 — and the empirical frame -scatter measures 62-64 e-. That agreement is not luck; it is the check that keeps the -constants honest. -""" -sky_e = float(sky_prov["sky_levels_e"][0]) -itime, coadds = science_itime, science_coadds -sampmode, multisam = facts[0]["sampmode"], facts[0]["multisam"] - -rn_eff = detector.read_noise_e(sampmode, multisam) # 38 / sqrt(MULTISAM) for MCDS. -dark_e = detector.dark_e_per_s * itime * coadds -budget_e = np.sqrt(sky_e + dark_e + rn_eff**2 * coadds) - -empirical_e = empirical_background_rms(subtracted[0]) - -print(f"Effective read noise (SAMPMODE {sampmode}, MULTISAM {multisam}): {rn_eff:.1f} e- (CDS would be {detector.read_noise_e_cds:.0f}).") -print(f"Per-frame budget: sqrt({sky_e:.0f} sky + {dark_e:.0f} dark + {rn_eff**2 * coadds:.0f} read) = {budget_e:.1f} e-") -print(f"Empirical per-frame background RMS: {empirical_e:.1f} e- (reference run: budget 62.0 vs empirical 62-64)") - -""" -__Casertano R And The Closure__ - -Drizzling correlates the noise of neighbouring output pixels, so a per-pixel RMS map that -ignored the correlation would make every chi^2 in your lens fit wrong. The scalar correction -R of Casertano et al. 2000 (AJ 120, 2747; also Fruchter & Hook 2002) depends only on the -drop size (`pixfrac` p) and the output/native scale ratio s. This reduction resamples -9.942 mas -> 10 mas with p = 1.0 — nearly the shift-and-add limit, where R -> 1.5. -""" -scale_ratio = 0.010 / adapter.native_scale # output / native pixel scale. -r_factor = casertano_r( - 1.0, # final_pixfrac — the drizzle drop size used by the combine. - scale_ratio, # s = final_scale / native_scale ~ 1.006 here. -) -print(f"Casertano R at pixfrac 1.0, s = {scale_ratio:.3f}: {r_factor:.3f} (shift-and-add limit is 1.5).") - -""" -The mosaic-level check ties everything in this script together. The shipped noise map is the -decorrelated-equivalent RMS (x R, chi^2-correct); the measurable pixel-to-pixel scatter of -the mosaic is correlation-suppressed by ~1/R. So the honest closure statistic is - - empirical mosaic RMS x R^2 / noise-map background floor ~ 1 - -It lands at ~0.84 on B1938. Every constant above feeds it: a wrong gain, a dropped COADDS -factor or a cps slip would miss by x6-x40, which is exactly why the constants are described -as closure-validated rather than trusted. `start_here.py` computes it on the shipped -products; the recipe and its cross-instrument siblings live in `scripts/guides/noise_maps.py`. - -__Why Not KAI__ - -The community-standard NIRC2/OSIRIS pipeline is KAI (Lu et al. 2022, -https://doi.org/10.5281/zenodo.6677744; https://keck-datareductionpipelines.github.io/KAI/), -which implements the same conceptual chain — darks, flats, sky, distortion, stacking. It is -also Python 2.7 + IRAF/PyRAF, which cannot be a dependency of a modern stack. Every ground -operation above is a simple array operation, so **PyAutoReduce** implements the stages -natively (numpy/astropy, with the standalone `drizzle` package as the resampler) and -validates them against SHARP's published numbers and the internal closures instead of -against a pipeline that can no longer run. - -__Wrap Up__ - -You walked the full ground-based chain by hand: raw KOA level-0 frames, matched -calibrations, master flat/dark and bad pixels, DN -> electrons, the scaled running sky with -its two passes, measured sub-pixel registration, the one-pixmap combine that pays the -resampling price exactly once, and the closure-validated detector noise budget. - -The following locations of the workspace are good places to checkout next: - -- `scripts/keck_nirc2/start_here.py`: the same chain as one `reduce_target` call, end to end. -- `scripts/keck_nirc2/psf.py`: the AO PSF problem — epochs, vetting gates, the provisional contract. -- `scripts/keck_nirc2/simulator.py`: source injection into the prepared frames, using the measured offsets. -- `scripts/guides/noise_maps.py`: noise recipes and closure statistics across all instruments. - -__Env__ (Developer Only) - -Not user documentation: this section configures the automated test harness. This script -reads the cache populated by `start_here.py` and needs the heavy `[keck]` extras, so the -smoke runner skips it. - -ENV: network -""" diff --git a/workspace/scripts/start_here.py b/workspace/scripts/start_here.py deleted file mode 100644 index 1892ab1..0000000 --- a/workspace/scripts/start_here.py +++ /dev/null @@ -1,446 +0,0 @@ -""" -Start Here: Data Reduction for Lens Modeling -============================================ - -Every strong-lens model fit begins long before the first likelihood evaluation: it begins with a -data reduction. **PyAutoReduce** turns raw archival telescope data — HST, JWST, Keck adaptive -optics, ALMA — into the modeling-ready datasets that **PyAutoLens** and **PyAutoGalaxy** load, -and this script is the single best first read of the workspace. In about thirty minutes of -reading (and, if you run it, one real HST reduction) you will understand why reduction quality -decides modeling quality, how a reduction is declared rather than scripted, and where to go for -your own instrument. - -The script performs a complete, real reduction: it downloads the SLACS strong lens -SDSS J0008-0004's HST ACS/WFC exposures from the MAST archive, reduces them with the default -pipeline, walks the provenance record, plots the products, and loads the finished dataset into -**PyAutoLens**. Running it therefore needs network access and the HST reduction stack -(`pip install "autoreduce[hst]"`); reading it needs neither. - -__Contents__ - -- **Why Reduction Quality Matters:** The two hard requirements lens modeling places on a reduction: an accurate PSF and an honest noise map. -- **A Reduction Is Declared, Not Scripted:** The `TargetSpec` philosophy — a reduction is a pure function of a declared spec plus the archive. -- **Imports:** Import **PyAutoReduce** and the plotting libraries. -- **Paths:** Anchor the cache and output directories to the workspace root with absolute paths. -- **Target Spec:** Declare the SLACS J0008-0004 reduction, dial by dial. -- **The Output Contract:** The four FITS products plus `reduction.json` that every reduction emits. -- **Run The Reduction:** One function call: acquire, align, drizzle, noise, PSF, package. -- **Provenance Walk:** Read the diagnostics out of the returned record — weight uniformity, the correlated-noise factor, star counts. -- **Plotting The Products:** Inspect the data, noise map and PSF with matplotlib. -- **The SLACS Quality Bar:** Validate the reduction against the legacy SLACS dataset — including the documented ~6% flux offset and the ~30%-higher-by-design noise. -- **Load Into PyAutoLens:** The seam between reduction and modeling: `al.Imaging.from_fits`. -- **Where To Go Next:** One-paragraph routing to every instrument folder in the workspace. -- **Wrap Up:** Summary and good places to checkout next. - -__Why Reduction Quality Matters__ - -Lens modeling asks more of a reduction than almost any other analysis, and the demands -concentrate in two products most reductions treat as afterthoughts: the PSF and the noise map. - -**The PSF.** A lens model fits the lensed arcs at the pixel level, convolved with the PSF. If the -PSF is wrong, the model leaves structured residuals at the arcs — residuals that can mimic, or -mask, the signal of dark-matter substructure that lens modeling is often trying to detect (see -the discussion of PSF systematics in the HST lensing literature, e.g. Bayer et al., -https://arxiv.org/abs/1803.05952). Empirical effective PSFs built from stars in the science -frames themselves (Anderson & King 2000, PASP 112, 1360, -https://ui.adsabs.harvard.edu/abs/2000PASP..112.1360A) are the modern standard, and a subtle -invariant matters throughout this workspace: the delivered PSF must be processed *identically* to -the science data — drizzled with the same kernel, pixfrac, scale and rotation — or it does not -describe the blurring in the mosaic at all. - -**The noise map.** Every chi-squared and every Bayesian evidence **PyAutoLens** computes assumes -the noise map is the per-pixel RMS of *independent* Gaussian noise. Image combination breaks that -assumption: drizzling (Fruchter & Hook 2002, PASP 114, 144, -https://ui.adsabs.harvard.edu/abs/2002PASP..114..144F) shares each input pixel among neighbouring -output pixels, so the per-pixel RMS underestimates the true uncertainty of any larger-scale -structure. The canonical treatment is the scalar correlation factor of Casertano et al. (2000, -AJ 120, 2747), which **PyAutoReduce** applies to every resampled noise map and records in the -provenance. Get this wrong and every posterior width in your lens model is miscalibrated — the -SL2S survey went as far as avoiding resampling entirely to keep its noise uncorrelated (Gavazzi -et al. 2012, https://arxiv.org/abs/1202.3852), the cleanest precedent for how seriously the -lensing literature takes this. The guide `scripts/guides/noise_maps.py` develops the full story. - -Everything else a reduction does — cosmic-ray rejection, astrometric alignment, sky subtraction, -cutout packaging — exists in service of those two products arriving accurate and honest. - -__A Reduction Is Declared, Not Scripted__ - -The **PyAutoReduce** public API is deliberately tiny — two names: - -```python -from autoreduce import TargetSpec, reduce_target -``` - -You do not write a reduction script that chains pipeline calls. You *declare* the reduction as a -frozen `TargetSpec` — target name, coordinates, instrument, and every literature-contested dial -(pixel scale, pixfrac, kernel, cosmic-ray method, PSF options) — and call `reduce_target` once. -The pipeline is a pure function of the spec plus the archive: re-running the same spec reproduces -the same dataset, modulo upstream reference-file updates, which the provenance record captures. -This is what makes a reduced sample reproducible — commit the spec (one small YAML per target, -see `dataset/README.md`), never the FITS. - -The corollary is the pipeline's failure philosophy: it fails loudly. A NaN in the noise map, a -zero-weight pixel inside the cutout, a star field too poor to build a PSF — these crash the -reduction with an explanatory error rather than shipping a silently-degraded dataset. A product -that reaches `output/` is one the pipeline is prepared to stand behind. - -__Imports__ - -We import the two public **PyAutoReduce** names, plus the standard libraries for reading and -plotting the FITS products afterwards. You'll see these imports in the majority of workspace -examples. -""" - -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the folder -containing `scripts/`). **PyAutoReduce** requires absolute paths: its drizzle step changes the -working directory internally, so relative paths would break. - -Two directories matter: - -- `cache/` — downloaded exposures and CRDS reference files, re-used across runs. Delete it and - the next run simply re-downloads; keep it and re-running a target is fast and can even work - offline (see `sync_references` in `scripts/guides/target_spec.py`). -- `output/` — the reduced datasets, one folder per target. - -Both are gitignored: no FITS is ever committed to this repository. -""" -WORKSPACE = Path(__file__).resolve().parents[1] -CACHE_ROOT = WORKSPACE / "cache" # Downloaded exposures + CRDS references (re-used across runs). -OUTPUT_ROOT = WORKSPACE / "output" # Reduced datasets, one folder per target. - -print(f"Workspace root: {WORKSPACE}") -print(f"Exposure cache: {CACHE_ROOT}") -print(f"Reduced output: {OUTPUT_ROOT}") - -""" -__Target Spec__ - -Our target is SDSS J0008-0004, a strong lens from the SLACS survey — the survey whose HST ACS -reductions (Bolton et al. 2006, ApJ 638, 703; Bolton et al. 2008, ApJ 682, 964, -https://arxiv.org/abs/0805.1931) are the quality bar the **PyAutoReduce** HST pipeline was -validated against. Its F814W exposures were taken under HST proposal 10886, and pinning the -acquisition to that proposal keeps exposures from neighbouring programmes out of the stack. - -Everything else stays at the ACS/WFC defaults, which are themselves the SLACS conventions — -a 0.05"/pixel output grid and drizzle pixfrac 0.8. The spec below is complete: this declaration, -plus the MAST archive, *is* the reduction. -""" -spec = TargetSpec( - name="slacs0008-0004", # Names the output folder: output/slacs0008-0004/. - ra=2.012333, # Target right ascension, degrees (J2000). - dec=-0.068944, # Target declination, degrees (J2000). - proposal_ids=("10886",), # Restrict acquisition to the SLACS HST programme's exposures. - # Defaults doing the work (see scripts/guides/target_spec.py for every dial): - # instrument="acs_wfc" ACS/WFC: CTE-corrected _flc exposures, AstroDrizzle combine. - # filter_name="F814W" The SLACS modeling band. - # cutout_shape=(281, 281) ~14" postage stamp at the output scale. - # final_scale=0.05 Output pixel scale, arcsec/pix — the SLACS convention. - # final_pixfrac=0.8 Drizzle drop size — the SLACS convention. - # cr_method="driz_cr" STScI-default cosmic-ray rejection at the combine. - # psf_shape=(21, 21) Compact PSF for convolution; psf_full_shape=(61, 61) for wings. -) - -print(f"Declared reduction: {spec.name} ({spec.instrument}, {spec.filter_name})") - -""" -__The Output Contract__ - -Every imaging reduction emits the same five products into `output//`, and they are exactly -what `al.Imaging.from_fits` consumes: - -- `data.fits` — the science cutout, drizzled to the modeling pixel scale, WCS and units intact - in the header (electrons/second for HST). -- `noise_map.fits` — the matching per-pixel RMS map, correlated-noise corrected. Pixels the - pipeline masked carry an effectively infinite noise value (1e8) with the data zeroed, so no - separate mask file is needed downstream. -- `psf.fits` — a compact (21x21) unit-normalised PSF for fit convolution. -- `psf_full.fits` — an extended (61x61) PSF carrying the wings, for science that needs them. -- `reduction.json` — the full provenance record: which exposures, which dials, every diagnostic, - and the software versions that produced the dataset. - -The guide `scripts/guides/output_contract.py` walks this contract file by file and -`reduction.json` block by block — read it after this script. - -__Run The Reduction__ - -One call runs the whole pipeline: query MAST for the calibrated exposures, download them into the -cache, sync the CRDS reference files, verify the Gaia-tied archive astrometry, drizzle the -exposures onto the output grid with cosmic-ray rejection, construct the noise map, build the PSF -from field stars, and package the cutouts. - -The return value is the provenance record — the same content written to `reduction.json`. -""" -print( - """ - Starting the reduction. On the first run this downloads the SLACS J0008-0004 F814W - exposures (a few hundred MB) from MAST plus CRDS reference files, then runs the full - AstroDrizzle combine — expect roughly 10-20 minutes depending on your connection and - machine. Re-runs re-use the cache and are much faster. - """ -) - -record = reduce_target( - spec, - cache_root=CACHE_ROOT, # Exposures + CRDS references cached here across runs. - output_root=OUTPUT_ROOT, # Products land in output_root//. -) - -out_dir = OUTPUT_ROOT / spec.name - -print(f"Reduction complete: {out_dir}") - -""" -__Provenance Walk__ - -A reduction you cannot audit is a reduction you cannot trust, so every run reports its -diagnostics in the returned record. Let's read the ones that matter most. - -First, acquisition: how many exposures went into the stack. -""" -print(f"Exposures combined: {record['acquire']['n_exposures']}") - -""" -Next, the drizzle diagnostics. The weight-map uniformity statistic (RMS/median of the drizzle -weight map, an STScI rule of thumb from the DrizzlePac Handbook, -https://hst-docs.stsci.edu/drizzpac) tells you whether the chosen `final_pixfrac` is compatible -with the dither pattern — values above ~0.2 mean the drops are too small for the dithers and the -coverage is speckled. - -The correlated-noise factor R is the Casertano et al. (2000) correction applied to the noise -map — for the SLACS dials (pixfrac 0.8 at native scale) it is ~1.36, meaning drizzle correlation -inflates the effective per-pixel noise by 36%. -""" -print(json.dumps(record["drizzle"]["weight_uniformity"], indent=2)) - -print(f"Correlated-noise factor R: {record['drizzle']['correlated_noise_factor']:.4f}") - -""" -The noise block records the recipe used — for HST, the per-pixel RMS is constructed from the -science and weight mosaics as R * sqrt(max(sci, 0)/exptime + 1/wht), the background-plus-Poisson -recipe of Bayer et al. (https://arxiv.org/abs/1803.05952, section 3.1) — plus the empirical -blank-sky RMS of the mosaic, which closes the loop: the constructed noise floor should match the -sky the mosaic actually shows. -""" -print(f"Noise recipe: {record['noise']['recipe']}") -print(f"Empirical background RMS: {record['noise']['empirical_background_rms']:.3e} e-/s") - -""" -The PSF block records how the PSF was built: the method (the default is a photutils effective PSF -in the Anderson & King 2000 lineage, built from stars in the mosaic itself), how many stars -survived selection and fitting, and which drizzle pass fed the star finding (`star_source_pass` — -see `scripts/hst_acs/psf.py` for why that is a dial at all). -""" -print(json.dumps(record["psf"], indent=2)) - -""" -Finally the package block: the pixel scale and data units the modeling stack needs, read straight -from the provenance rather than remembered by you. -""" -print(f"Pixel scale: {record['package']['pixel_scale']} arcsec/pix") -print(f"Data units: {record['package']['data_units']}") -print(f"Products: {record['package']['products']}") - -""" -__Plotting The Products__ - -Numbers first, but always look at your data. We plot the science cutout with arcsinh scaling -(which shows the faint arcs and the bright deflector core in one stretch), the noise map, and the -compact PSF on a log stretch, saving each as a PNG under the target's output folder. -""" -plots_dir = out_dir / "plots" -plots_dir.mkdir(exist_ok=True) - -data = fits.getdata(out_dir / "data.fits").astype(float) -noise_map = fits.getdata(out_dir / "noise_map.fits").astype(float) -psf = fits.getdata(out_dir / "psf.fits").astype(float) - -plt.figure(figsize=(6, 6)) -plt.imshow(np.arcsinh(data / np.nanstd(data)), origin="lower", cmap="magma") -plt.colorbar(label="arcsinh(data / rms)") -plt.title(f"{spec.name} data.fits ({spec.filter_name})") -data_png = plots_dir / "data.png" -plt.savefig(data_png, dpi=150, bbox_inches="tight") -plt.close() -print(f"Saved {data_png.resolve()}") - -plt.figure(figsize=(6, 6)) -# Masked pixels carry the 1e8 sentinel; clip the display to the physical range. -plt.imshow( - np.clip(noise_map, None, np.nanpercentile(noise_map[noise_map < 1.0e7], 99)), - origin="lower", - cmap="viridis", -) -plt.colorbar(label="RMS noise (e-/s)") -plt.title(f"{spec.name} noise_map.fits") -noise_png = plots_dir / "noise_map.png" -plt.savefig(noise_png, dpi=150, bbox_inches="tight") -plt.close() -print(f"Saved {noise_png.resolve()}") - -plt.figure(figsize=(6, 6)) -plt.imshow(np.log10(np.clip(psf, 1e-8, None)), origin="lower", cmap="cividis") -plt.colorbar(label="log10(PSF)") -plt.title(f"{spec.name} psf.fits (21x21, sums to {psf.sum():.4f})") -psf_png = plots_dir / "psf.png" -plt.savefig(psf_png, dpi=150, bbox_inches="tight") -plt.close() -print(f"Saved {psf_png.resolve()}") - -""" -__The SLACS Quality Bar__ - -How do we know this reduction is *right*? The **PyAutoReduce** HST pipeline was validated against -the legacy SLACS modeling datasets — reductions that have supported fifteen-plus years of -published lens models (Bolton et al. 2008, https://arxiv.org/abs/0805.1931). The validation -principle is worth internalising: *the reduction is correct when the science is invariant under -it* — a lens model fitted to the new reduction should infer the same physics as one fitted to the -legacy data. - -Two honest numbers come out of that comparison, both documented rather than hidden: - -- **Data parity ~0.94-0.96.** The new mosaics carry ~6% less flux than the legacy cutouts — a - known, accepted offset between the modern CTE-corrected `_flc` + AstroDrizzle chain and the - legacy reduction, recorded in the design docs rather than tuned away. - -- **Noise ~30% above legacy, by design.** The legacy SLACS noise maps do not carry the Casertano - correlated-noise factor; ours do (R ~ 1.36 at the SLACS dials). The new noise maps are - *supposed* to be higher — they are the honest ones (see `scripts/guides/noise_maps.py`). - -The comparison machinery is public: `autoreduce.validation.registered_ratios` registers a new -data/noise pair onto a reference pair at sub-pixel precision and reports median bright-pixel -ratios. If you have a trusted previous reduction of your own target, run the same parity check -against it — it is the fastest way to catch a mistake in either. -""" -from autoreduce.validation import registered_ratios # noqa: E402 (used when you have a reference dataset) - -print( - "Parity idiom (needs a reference dataset):\n" - " registered_ratios(new_data, new_noise, ref_data, ref_noise)\n" - " -> data_ratio_median ~0.94-0.96 and noise_ratio_median ~1.3 vs legacy SLACS is expected." -) - -""" -__Load Into PyAutoLens__ - -This is the seam the whole workspace exists to serve: the four FITS files plus the pixel scale -from the provenance are exactly the inputs `al.Imaging.from_fits` takes. The committed example -datasets that `autolens_workspace` models — its `imaging/data_preparation` examples spell out the -standards (electrons/second units, centred postage stamp, RMS noise map, odd unit-normalised -PSF) — are files produced by scripts exactly like this one. - -The import is guarded: **PyAutoReduce** never depends on the modeling stack, and the reduction -above is complete whether or not **PyAutoLens** is installed. -""" -try: - import autolens as al -except ImportError: - al = None - print( - "PyAutoLens is not installed (pip install autolens), so the final loading step is " - "skipped — the reduction itself is complete and the products above are ready for " - "modeling on any machine that has it." - ) - -if al is not None: - dataset = al.Imaging.from_fits( - data_path=out_dir / "data.fits", - noise_map_path=out_dir / "noise_map.fits", - psf_path=out_dir / "psf.fits", - pixel_scales=record["package"]["pixel_scale"], # From provenance, never from memory. - ) - print( - f"Loaded {spec.name} into PyAutoLens: shape {dataset.data.shape_native}, " - f"pixel scale {dataset.pixel_scales}." - ) - print( - "Next: model it with autolens_workspace/scripts/imaging/start_here.py — " - "point its Imaging.from_fits paths at this output folder." - ) - -""" -__Where To Go Next__ - -Each instrument folder follows the same shape — a `start_here.py` running the default pipeline on -that instrument's validation anchor, a `step_by_step.py` teaching what each stage does with the -instrument handbook and literature, and (where the instrument supports them) `psf.py`, -`individual.py` and `simulator.py` deep dives. Route yourself by your data: - -- `scripts/hst_acs/` — HST ACS/WFC, the reference pipeline this script just ran. Its - `step_by_step.py` teaches the AstroDrizzle chain against the ACS Data Handbook - (https://hst-docs.stsci.edu/acsdhb), and `dials.py` is the drizzle trade study — what - scale/pixfrac/kernel/CR-method choices cost and what the literature disagrees on. - -- `scripts/hst_wfc3_uvis/` — HST WFC3/UVIS, anchored to the Bayer et al. - (https://arxiv.org/abs/1803.05952) F390W reduction of SDSS J0252+0039 at the native 0.0396" - scale with pixfrac 1.0. UVIS is where the STARRED super-sampled PSF back-end - (Millon et al. 2024, https://arxiv.org/abs/2402.08725) earns its keep. - -- `scripts/hst_wfc3_ir/` — HST WFC3/IR: no CTE correction exists, cosmic rays are already - rejected by up-the-ramp fitting, and the undersampled 0.128" pixels drive the dither-and- - drizzle strategy — including the honest rule that few-dither IR data at sub-native scales - needs pixfrac 1.0 or a coarser grid. - -- `scripts/jwst_nircam/` — JWST NIRCam through the calwebb_image3 pipeline, anchored to the - COSMOS-Web ring (Mercier et al. 2024, https://arxiv.org/abs/2309.15986). Two deltas from HST - to internalise: the noise map is *read* from the pipeline's propagated ERR array rather than - constructed, and the data stays in surface-brightness units of MJy/sr — not electrons/second. - `multi_band.py` builds the four-band dataset for multi-wavelength modeling. - -- `scripts/keck_nirc2/` — Keck NIRC2 laser-guide-star AO, anchored to B1938+666 — the SHARP - survey's ring (Lagattuta et al. 2012, MNRAS 424, 2800) in which Vegetti et al. (2012, Nature - 481, 341) detected a dark satellite. Ground-based means the workspace's extra stages appear — - darks/flats, running-sky subtraction, epoch-matched distortion — and the AO PSF is delivered - as *provisional candidates*, because final PSF selection belongs to the modeling stage - (Chen et al. 2016, https://arxiv.org/abs/1601.01321). - -- `scripts/alma/` — ALMA visibilities: no images at all. The dataset is the calibrated - visibilities themselves, extracted to the `(N_vis, 2)` triplet `al.Interferometer.from_fits` - loads, because fitting in the uv-plane keeps the noise independent and the likelihood - well-defined (Hezaveh et al. 2016, https://arxiv.org/abs/1601.01388; Dye et al. 2018, - https://arxiv.org/abs/1705.05413). - -- `scripts/surveys/` — Legacy Surveys / SDSS / Pan-STARRS cutouts around a target for colour - context. Deliberately *not* modeling data: no PSF ships, and no noise map beyond what the - service provides. - -- `scripts/guides/` — the cross-instrument deep dives: `output_contract.py` (the products in - depth), `noise_maps.py` (recipes, the Casertano factor, closure checks — mostly offline) and - `target_spec.py` (every dial; fully offline). - -__Wrap Up__ - -This script reduced a real SLACS strong lens from the MAST archive to a modeling-ready dataset -with one declared spec and one function call, audited the reduction through its provenance -record, and handed the products to **PyAutoLens**. - -The two ideas to carry forward: reduction quality *is* modeling quality (the PSF and the noise -map are where it concentrates), and a reduction should be a declaration you can commit, not a -script you can lose. - -The following locations of the workspace are good places to checkout next: - -- `scripts/guides/output_contract.py`: the five output products in depth, on the dataset this script just produced. -- `scripts/guides/noise_maps.py`: the noise story — recipes per instrument, the Casertano factor, closure diagnostics. -- `scripts/guides/target_spec.py`: every `TargetSpec` dial, spec YAML round-trips, validation guard rails (runs offline). -- `scripts/hst_acs/step_by_step.py`: what each pipeline stage actually did to the exposures reduced above. -- `autolens_workspace/scripts/imaging/start_here.py`: model the dataset this script produced. - -__Env__ (Developer Only) - -Not user documentation: this section configures the automated test harness. This script downloads -exposures from MAST and runs the drizzlepac stack, so it is excluded from offline smoke runs. - -ENV: network -""" diff --git a/workspace/scripts/surveys/README.md b/workspace/scripts/surveys/README.md deleted file mode 100644 index 22028fa..0000000 --- a/workspace/scripts/surveys/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Surveys — ground-based colour-context cutouts - -The cutout domain: fetch + package postage stamps from public survey cutout -services, **never reduce**. These are colour/context images for lens fields — -especially ALMA targets with no optical counterpart in their modeling data — -and emphatically not modeling inputs: no PSF, mostly no noise map, and a -`products_optional` provenance block that says so. - -Scripts: - -1. `start_here.py` — the SLACS J0008-0004 field from all three services: - Legacy Surveys DR10 (grz, 0.262"/px, real noise map via inverse variance; - covers the DES footprint), SDSS (gri, 0.396"/px) and Pan-STARRS - (gri, 0.25"/px), plus the multi-survey postage-stamp contact sheet and the - deferred-extensions list (HSC, unWISE/GALEX, survey PSFs). - -Network required; failures are reported per service, never as one crash. diff --git a/workspace/scripts/surveys/__init__.py b/workspace/scripts/surveys/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/workspace/scripts/surveys/start_here.py b/workspace/scripts/surveys/start_here.py deleted file mode 100644 index 2e5c27c..0000000 --- a/workspace/scripts/surveys/start_here.py +++ /dev/null @@ -1,304 +0,0 @@ -""" -Start Here: Survey Cutouts -========================== - -Not every product **PyAutoReduce** delivers is modeling data. This script demonstrates -the third and simplest of its domains: **survey cutouts** — postage stamps fetched from -the public cutout services of the big ground-based imaging surveys, packaged as colour -context for your lens fields. - -Why does a reduction package ship a fetcher? Because knowing what a lens field looks like -in the optical is a routine need — identifying the deflector, spotting neighbouring -galaxies and stars, checking astrometry — and for some lenses it is the *only* optical -view you will ever have. Lensed dusty star-forming galaxies found at submillimetre -wavelengths (the ALMA targets of `scripts/alma/`) often have no optical counterpart at -all: the source is dust-obscured and the deflector may be faint, so a survey cutout of -the field is the entire optical story. In about 10 minutes this script fetches the -SLACS J0008-0004 field from three surveys, walks the honesty machinery that stops these -stamps masquerading as modeling data, and plots the multi-survey contact sheet. - -__Contents__ - -- **The Cutout Domain:** Fetch + package, never reduce — why this is its own branch. -- **Context Not Modeling Data:** What cutouts are for, and emphatically not for. -- **Imports:** Import **PyAutoReduce** and the plotting libraries. -- **Paths:** Anchor every path to the workspace root. -- **The Field SLACS J0008-0004:** The lens field we fetch. -- **The Three Services:** Legacy Surveys DR10, SDSS and Pan-STARRS, and their trade-offs. -- **Fetch The Cutouts:** One `reduce_target` call per service, failures kept per-service. -- **Provenance:** The `products_optional` block — what was NOT produced, and why. -- **Noise From Inverse Variance:** The one service that ships variance, and what we make of it. -- **Postage Stamps:** The multi-survey, multi-band contact sheet. -- **Deferred Extensions:** HSC, unWISE/GALEX, variance elsewhere, approximate PSFs. -- **Wrap Up:** Summary and good places to check out next. - -__The Cutout Domain__ - -DES, SDSS, Pan-STARRS and their peers deliver *pre-reduced* coadds through public cutout -services — mosaics the survey collaborations built with their own calibration, their own -astrometry, their own co-addition, at a quality no external re-reduction of the raw -frames would match. So **PyAutoReduce** treats them as a third adapter domain (`cutout`) -beside imaging and visibility, with a two-stage branch: **fetch + package, never -reduce**. There is no drizzle here, no PSF stage, no noise recipe — a cutout arrives as -the survey made it, gets its provenance stamped, and is written to disk. - -The shared machinery is exactly what you know from the other domains: the same -`TargetSpec` (the `survey_bands` dial plus `cutout_shape` for the stamp size), the same -one-call `reduce_target`, the same `reduction.json` provenance. - -__Context Not Modeling Data__ - -The design's loudest rule. A survey cutout is **for**: - -- colour/context imaging of a lens field — which object is the deflector, what - neighbours and stars sit nearby, does the field match your pointing; -- the optical view of targets whose *modeling* data lives elsewhere — especially ALMA - lens fields with no optical counterpart in the modeling data at all; -- figures: the "here is the field" panel of a paper or proposal. - -It is emphatically **not** a modeling input, for reasons the products themselves encode: - -- **No PSF.** None of these services ships one, and lens modeling without a PSF model is - not lens modeling. No `psf.fits` is ever written by this branch. -- **Mostly no noise map.** Only one of the three services exposes per-pixel variance; - the others deliver data only. Without an RMS map there is no likelihood. -- **Uncharacterised correlations.** Survey coadds resample and stack with kernels these - services do not document per-cutout, so even where variance exists the pixel-to-pixel - correlations are uncharacterised — fine for context, unquantified for chi-squared. - -The provenance block you will inspect below (`products_optional`) states per product -what was not produced and why, so no downstream tool can mistake a stamp for an -`al.Imaging` dataset by accident. - -__Imports__ -""" -import json -import sys -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits - -from autoreduce import TargetSpec, reduce_target - -""" -__Paths__ - -Reductions read and write real data, so we anchor every path to the workspace root (the -folder containing `scripts/`). **PyAutoReduce** requires absolute paths throughout the -workspace, and although this branch never changes directory, we keep the convention. -""" -WORKSPACE = Path(__file__).resolve().parents[2] -CACHE_ROOT = WORKSPACE / "cache" -OUTPUT_ROOT = WORKSPACE / "output" - -""" -__The Field SLACS J0008-0004__ - -We fetch the field of SLACS J0008-0004, one of the strong lenses of the Sloan Lens ACS -Survey (Bolton et al. 2008, https://arxiv.org/abs/0805.1931) — the same target that -anchors the HST/ACS reduction in `scripts/hst_acs/start_here.py`. That makes it the -perfect demonstration of the intended division of labour: the HST reduction produces the -modeling dataset; the survey cutouts below produce the wide-field colour context around -it. (For an ALMA field like G09v1.40 the cutout might be the *only* optical view — the -submillimetre selection of Negrello et al. 2010, Science 330, 800, cares nothing for -optical brightness.) -""" -NAME = "slacs0008-0004" -RA, DEC = 2.012333, -0.068944 - -""" -__The Three Services__ - -Three adapters, three public endpoints, one field: - -- `legacy_surveys` — the DESI Legacy Imaging Surveys DR10 cutout service - (legacysurvey.org), DECam g/r/z at 0.262"/pixel. **The one service that ships - variance**: the same request can append an inverse-variance HDU, so Legacy cutouts get - a real `noise_map.fits`. One more thing makes this the workhorse: **DR10 includes the - full DECam/DES footprint** (and far more southern sky), so Legacy is the DES door — - there is no separate DES adapter because none is needed. -- `sdss` — SDSS frames via astroquery, g/r/i at 0.396"/pixel. The shallowest and - coarsest of the three, but the survey that found the SLACS lenses in the first place, - and coverage where the others may lack it. Data only. -- `panstarrs` — Pan-STARRS PS1 stack cutouts via the STScI fitscut service - (ps1images.stsci.edu), g/r/i at 0.25"/pixel. Northern-sky coverage to dec > -30 deg. - Data only. - -(HSC — deeper than all three — is deliberately absent: its cutout service sits behind a -credential-gated account, deferred until a real need justifies the auth plumbing. See the -deferred list at the end.) - -__Fetch The Cutouts__ - -One `reduce_target` per service. Note the error handling: each service gets its own -try/except, because these are three independent external endpoints — a service outage or -a coverage gap in one must yield a per-service verdict, not one crash that loses the -other two. Each service's products land under its own output folder so the three -`reduction.json` records stay separate. -""" -SERVICES = ("legacy_surveys", "sdss", "panstarrs") - -print(f"Fetching {NAME} cutouts from {len(SERVICES)} services (network, ~seconds each)...") - -records, stamps = {}, {} -for key in SERVICES: - print(f"\n== {key} ==") - spec = TargetSpec( - name=NAME, # products land at output/surveys/// - ra=RA, # field centre (deg) - dec=DEC, - instrument=key, # selects the cutout-domain adapter for this service - cutout_shape=(101, 101), # stamp size in native survey pixels (101 px: ~26" Legacy, ~40" SDSS, ~25" PS1) - ) - try: - record = reduce_target( - spec, - cache_root=CACHE_ROOT, # unused by this branch (cutouts are fetched per run) - output_root=OUTPUT_ROOT / "surveys" / key, # one output tree per service - ) - except Exception as error: - print(f" FAILED: {error}") - records[key] = None - continue - records[key] = record - print(f" bands delivered: {record['acquire']['bands_delivered']}") - print(f" products : {record['package']['products']}") - print(f" pixel scale : {record['package']['pixel_scale']} arcsec/pixel") - - out_dir = OUTPUT_ROOT / "surveys" / key / NAME - for product in record["package"]["products"]: - if Path(product).name != "data.fits": - continue - band = Path(product).parent.name - stamps[(key, band)] = fits.getdata(out_dir / product) - -if not any(records.values()): - print("\nNo survey service could be reached — check your network and re-run.") - sys.exit(0) - -""" -__Provenance__ - -Open any of the three `reduction.json` records and the honesty machinery is right there: -the `package.products_optional` block states, product by product, what was **not** -produced and why. For the data-only services it reads "not produced — service ships no -variance product"; for every service the PSF line records that colour-context products -carry no PSF, by design. This block is the contract that keeps a cutout from ever -pretending to be a modeling-ready dataset — a downstream loader that wants `al.Imaging` -inputs should find `psf.fits` absent *and* the record saying so on purpose. -""" -for key, record in records.items(): - if record is None: - continue - print(f"\n{key} products_optional:") - print(json.dumps(record["package"]["products_optional"], indent=2)) - -""" -__Noise From Inverse Variance__ - -The Legacy service's `&invvar` option returns an inverse-variance map alongside the data -in the same request, and the pipeline converts it to the RMS convention the rest of the -ecosystem speaks: sigma = 1 / sqrt(invvar), with non-positive or non-finite inverse -variances left as NaN rather than invented. So Legacy stamps come with a real -`noise_map.fits` — useful for judging depth and spotting masked regions in the coadd. -The correlation caveat from the top of the script still applies: a coadd's -pixel-to-pixel noise correlations are uncharacterised here, which is (one of the reasons) -why even a Legacy cutout is context, not a modeling input. -""" -legacy_record = records.get("legacy_surveys") -if legacy_record is not None: - for product in legacy_record["package"]["products"]: - if Path(product).name != "noise_map.fits": - continue - band = Path(product).parent.name - noise = fits.getdata(OUTPUT_ROOT / "surveys" / "legacy_surveys" / NAME / product) - finite = noise[np.isfinite(noise)] - print( - f"legacy {band}-band noise map: median sigma = {np.median(finite):.4f} " - f"(nanomaggies), {100.0 * (1.0 - finite.size / noise.size):.1f}% NaN " - f"(no-coverage / masked coadd pixels)" - ) - -""" -__Postage Stamps__ - -The contact sheet: one row per service, one column per band, arcsinh-scaled so both the -lens galaxy's core and the faint outskirts are visible. This is the figure this domain -exists to make — the field at a glance, across surveys and bands, with the survey depth -differences plain to see. -""" -plot_dir = OUTPUT_ROOT / "surveys" / "plots" -plot_dir.mkdir(parents=True, exist_ok=True) - -available = [key for key in SERVICES if records.get(key) is not None] -n_cols = max( - len([band for (k, band) in stamps if k == key]) for key in available -) -fig, axes = plt.subplots( - len(available), n_cols, figsize=(3 * n_cols, 3 * len(available)), squeeze=False -) -for row, key in enumerate(available): - bands = sorted(band for (k, band) in stamps if k == key) - for col in range(n_cols): - ax = axes[row][col] - ax.set_xticks([]) - ax.set_yticks([]) - if col >= len(bands): - ax.set_visible(False) - continue - band = bands[col] - data = stamps[(key, band)] - finite = data[np.isfinite(data)] - scale = np.std(finite) if finite.size else 1.0 - ax.imshow( - np.arcsinh(np.nan_to_num(data) / (scale if scale > 0 else 1.0)), - origin="lower", - cmap="gray_r", - ) - ax.set_title(f"{key} {band}", fontsize=10) -fig.suptitle(f"{NAME}: survey colour context (101 px stamps, native scales)") -stamps_png = plot_dir / f"{NAME}_postage_stamps.png" -fig.savefig(stamps_png, dpi=150, bbox_inches="tight") -plt.close(fig) -print(f"\nSaved postage-stamp contact sheet to {stamps_png.resolve()}") - -""" -__Deferred Extensions__ - -Recorded, assessed, and deliberately not built yet: - -- **HSC** — deeper than everything above, but its cutout service requires a credentialed - account; deferred until a real need justifies the auth plumbing. -- **unWISE and GALEX** — served by the *same* Legacy viewer endpoint via a layer - parameter, making IR and UV context the cheapest future extension on the books. -- **SDSS / Pan-STARRS variance** — both surveys expose the ingredients (frame metadata, - weight files); wired up only if someone actually needs ground-based noise maps. -- **Approximate survey PSFs** — Legacy catalogs record per-brick seeing FWHM values; a - Gaussian-FWHM kernel, clearly flagged approximate in provenance, is the recorded - follow-up if colour context ever needs even a rough PSF. Until then: no PSF, ever, - from this branch. - -__Wrap Up__ - -You fetched one lens field from three survey cutout services through the same -`TargetSpec` + `reduce_target` idiom as every real reduction, saw the `products_optional` -provenance that keeps context data honest, got a real noise map from the one service -that ships variance, and made the multi-survey contact sheet. - -Good places to checkout next: - -- `scripts/hst_acs/start_here.py` — the *modeling* dataset for this same field, reduced - from HST/ACS exposures. -- `scripts/alma/start_here.py` — the ALMA lens fields for which a survey cutout is often - the only optical view. -- `scripts/guides/output_contract.py` — what a real modeling-ready product set contains, - i.e. everything a cutout deliberately is not. -""" - -""" -__Env__ (Developer Only) - -ENV: network -""" diff --git a/workspace/smoke_tests.txt b/workspace/smoke_tests.txt deleted file mode 100644 index cf68826..0000000 --- a/workspace/smoke_tests.txt +++ /dev/null @@ -1,62 +0,0 @@ -# Curated smoke-test list for the autoreduce_workspace. -# -# Every script in this workspace performs a real reduction: it downloads -# exposures from an archive (MAST, KOA, the ALMA archive) and runs the -# instrument's heavy reduction stack (drizzlepac, jwst, casatools). CI smoke -# runners have neither network access to the archives nor the heavy stacks -# installed, so NO script currently runs in smoke tests — every entry below is -# commented out, with the blocking dependency cited per entry. The two offline -# guides (guides/target_spec.py, most of guides/noise_maps.py) are the first -# candidates to enable once an autoreduce install lands in the smoke image. -# -# Keep this list a curated, commented coverage argument: when enabling an -# entry, say why it is now runnable; when adding a script, list it here -# commented with its reason. - -# --- Top level ------------------------------------------------------------- -# scripts/start_here.py # requires MAST network + drizzlepac (full ACS reduction) - -# --- Guides ---------------------------------------------------------------- -# scripts/guides/output_contract.py # offline, but reads output/ produced by scripts/start_here.py (clean-exits without it); needs autoreduce installed -# scripts/guides/noise_maps.py # offline (pure numpy/astropy demos); needs autoreduce installed — enable first -# scripts/guides/target_spec.py # offline (spec construction only); needs autoreduce installed — enable first - -# --- HST ACS/WFC ----------------------------------------------------------- -# scripts/hst_acs/start_here.py # requires MAST network + CRDS sync + drizzlepac -# scripts/hst_acs/step_by_step.py # requires MAST network + CRDS sync + drizzlepac -# scripts/hst_acs/individual.py # requires MAST network + drizzlepac + deepCR ([frames] extra) -# scripts/hst_acs/psf.py # requires MAST network + drizzlepac + photutils star field -# scripts/hst_acs/simulator.py # requires MAST network + drizzlepac (injection re-runs the pipeline) -# scripts/hst_acs/dials.py # requires MAST network + drizzlepac (multiple drizzle passes) - -# --- HST WFC3/UVIS --------------------------------------------------------- -# scripts/hst_wfc3_uvis/start_here.py # requires MAST network + CRDS sync + drizzlepac -# scripts/hst_wfc3_uvis/step_by_step.py # requires MAST network + CRDS sync + drizzlepac -# scripts/hst_wfc3_uvis/psf.py # requires MAST network + drizzlepac (+ [starred] extra for the STARRED leg) -# scripts/hst_wfc3_uvis/simulator.py # requires MAST network + drizzlepac (injection re-runs the pipeline) - -# --- HST WFC3/IR ----------------------------------------------------------- -# scripts/hst_wfc3_ir/start_here.py # requires MAST network + CRDS sync + drizzlepac -# scripts/hst_wfc3_ir/step_by_step.py # requires MAST network + CRDS sync + drizzlepac - -# --- JWST NIRCam ----------------------------------------------------------- -# scripts/jwst_nircam/start_here.py # requires MAST network + jwst==1.14.0 + CRDS_PATH -# scripts/jwst_nircam/step_by_step.py # requires MAST network + jwst==1.14.0 + CRDS_PATH -# scripts/jwst_nircam/multi_band.py # requires MAST network + jwst==1.14.0 (four-band loop, hours) -# scripts/jwst_nircam/psf.py # requires MAST network + jwst==1.14.0 (+ stpsf for the model leg) -# scripts/jwst_nircam/individual.py # requires MAST network + jwst==1.14.0 -# scripts/jwst_nircam/simulator.py # requires MAST network + jwst==1.14.0 (injection re-runs image3) - -# --- Keck NIRC2 ------------------------------------------------------------ -# scripts/keck_nirc2/start_here.py # requires KOA network + pykoa + drizzle ([keck] extra) -# scripts/keck_nirc2/step_by_step.py # requires KOA network + pykoa + drizzle ([keck] extra) -# scripts/keck_nirc2/psf.py # requires KOA network + pykoa (PSF-star epoch frames) -# scripts/keck_nirc2/simulator.py # requires KOA network + pykoa (injection into prepared frames) - -# --- ALMA ------------------------------------------------------------------ -# scripts/alma/start_here.py # requires ALMA archive network + casatools/casatasks -# scripts/alma/step_by_step.py # requires ALMA archive network + casatools/casatasks -# scripts/alma/simulator.py # requires casatools/casatasks (simobserve route) - -# --- Surveys --------------------------------------------------------------- -# scripts/surveys/start_here.py # requires network to the Legacy Surveys / SDSS / Pan-STARRS cutout services