diff --git a/Prompt/100_How_to_use_RSP_tools/101_Alert_archive/101_1_Alert_retrieval_service_Herald.ipynb b/Prompt/100_How_to_use_RSP_tools/101_Alert_archive/101_1_Alert_retrieval_service.ipynb similarity index 96% rename from Prompt/100_How_to_use_RSP_tools/101_Alert_archive/101_1_Alert_retrieval_service_Herald.ipynb rename to Prompt/100_How_to_use_RSP_tools/101_Alert_archive/101_1_Alert_retrieval_service.ipynb index 00f61441..a38c1bfc 100644 --- a/Prompt/100_How_to_use_RSP_tools/101_Alert_archive/101_1_Alert_retrieval_service_Herald.ipynb +++ b/Prompt/100_How_to_use_RSP_tools/101_Alert_archive/101_1_Alert_retrieval_service.ipynb @@ -10,7 +10,7 @@ "id": "b5c1694b-bfcb-48d8-8bab-4e5e7b4ea55f", "metadata": {}, "source": [ - "# 101.1. Alert retrieval service (Herald)\n", + "# 101.1. Alert retrieval service\n", "\n", "
\n", "\n", @@ -22,7 +22,7 @@ "Data Release: [Prompt Products](https://prompt-products.lsst.io/)\\\n", "Container Size: Large\\\n", "LSST Science Pipelines version: r29.2.0\\\n", - "Last verified to run: 2026-04-24\\\n", + "Last verified to run: 2026-05-27\\\n", "Repository: [github.com/lsst/tutorial-notebooks](https://github.com/lsst/tutorial-notebooks)\\\n", "DOI: [10.11578/rubin/dc.20250909.20](https://doi.org/10.11578/rubin/dc.20250909.20)" ] @@ -32,7 +32,7 @@ "id": "dcd70a28-00af-41c5-b81a-ee1da3024db8", "metadata": {}, "source": [ - "**Learning objective:** How to obtain single alert packets from the RSP's alert retrieval service, Herald.\n", + "**Learning objective:** How to obtain single alert packets from the RSP's alert retrieval service.\n", "\n", "**LSST data products:** Alert packets.\n", "\n", @@ -54,15 +54,15 @@ "source": [ "## 1. Introduction\n", "\n", - "The RSP's alert retrieval service, Herald, enables the contents of single alert packets to be retrieved by alert ID number, in one of three formats: Avro, JSON, or FITS. See also the technical note \"Design of an Alert Retrieval Service in the RSP\" ([SQR-114](https://sqr-114.lsst.io/)).\n", + "The RSP's alert retrieval service enables the contents of single alert packets to be retrieved by alert ID number, in one of three formats: Avro, JSON, or FITS. See also the technical note \"Design of an Alert Retrieval Service in the RSP\" ([SQR-114](https://sqr-114.lsst.io/)).\n", "\n", "**Alert packets** are files of measurements for sources detected in difference images that are streamed to brokers within a few minutes of image acquisition.\n", "Alerts are a Prompt Product, and are the result of Rubin's Difference Image Analysis (DIA) and Alert Production (AP) pipelines.\n", "\n", - "**Brokers** are the recommended way to do time-domain science with alerts because their functionality includes, e.g., filtering, cross-match, classification, and users can explore and retrieve scientifically-relevant subsets via their web-based user interfaces or via API clients.\n", + "**Brokers** are the recommended way to do time-domain science with alerts because their functionality includes, e.g., filtering, cross-match, and classification, and users can explore and retrieve scientifically-relevant subsets via their web-based user interfaces or API clients.\n", "\n", - "**Why use the RSP's alert retrieval service, Herald?** \\\n", - "When only the contents of a single alert packet are desired, and the alert ID is known.\n", + "**Why use the RSP's alert retrieval service?** \\\n", + "The alert retrieval service is useful when only the contents of a single alert packet are desired, and the alert ID is known.\n", "Otherwise, brokers are the best way to do real-time science with the alerts, and scientific analyses on longer timescales (days to weeks) can use the other Prompt Products (queryable databases, images).\n", "\n", "**Additional resources:**\n", @@ -70,7 +70,7 @@ "* [Prompt Products documentation](https://prompt-products.lsst.io/).\n", "* [Learn more about the LSST alert brokers](https://rubinobservatory.org/for-scientists/data-products/alerts-and-brokers).\n", "\n", - "**This tutorial** focuses on how Herald works, and options for packet retrieval formats (e.g., FITS, JSON, schema-only, cutouts-only).\n", + "**This tutorial** focuses on how the alert retrieval service works, and options for packet retrieval formats (e.g., FITS, JSON, schema-only, cutouts-only).\n", "\n", "**Related tutorials:** The 200-level tutorial on alert packets focuses on the details of the packets' schema and data contents, and how packets for static-sky and moving objects differ." ] @@ -82,8 +82,8 @@ "source": [ "### 1.1. Import packages\n", "\n", - "Import the `fastavro` package for reading alerts from Herald in the default Avro format. Import the `base64` package for reading image stamps in the JSON format.\n", - "Import the `RSPClient` and `get_service_url` in order to access Herald.\n", + "Import the `fastavro` package for reading alerts in the default Avro format. Import the `base64` package for reading image stamps in the JSON format.\n", + "Import the `RSPClient` and `get_service_url` in order to access the alert retrieval service.\n", "Also import a range of other plotting and analyis packages." ] }, @@ -98,12 +98,14 @@ "import base64\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "import pandas as pd\n", "import io\n", "from astropy.io import fits\n", + "from astropy.nddata import CCDData\n", + "from astropy.visualization import (MinMaxInterval, LinearStretch,\n", + " ImageNormalize)\n", "\n", "from lsst.rsp import RSPClient, get_service_url\n", - "import lsst.afw.display as afwDisplay\n", - "import lsst.afw.image as afwImage\n", "from lsst.utils.plotting import (get_multiband_plot_colors,\n", " get_multiband_plot_symbols)" ] @@ -115,7 +117,7 @@ "source": [ "### 1.2. Define parameters and functions\n", "\n", - "Establish the connection to Herald by getting the `url` for the alerts retrieval service and instantiating the `RSPClient`." + "Establish the connection to the alert retrieval service by getting the `url` for the service and instantiating the `RSPClient`." ] }, { @@ -129,24 +131,6 @@ "client = RSPClient(\"\")" ] }, - { - "cell_type": "markdown", - "id": "09e530e3-1b9e-437c-ae50-94c56513ec8a", - "metadata": {}, - "source": [ - "Set the `afwDisplay` backend to `matplotlib`, to be used when displaying the image stamps." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e321748-27ba-4987-bd22-0cc7c250b0f4", - "metadata": {}, - "outputs": [], - "source": [ - "afwDisplay.setDefaultBackend(\"matplotlib\")" - ] - }, { "cell_type": "markdown", "id": "53d0a4a0-0fd5-4b7d-8b47-4507e713da5c", @@ -172,7 +156,9 @@ "id": "fc57297b-7457-46b4-9329-95f5993e50d5", "metadata": {}, "source": [ - "At the time this tutorial was prepared, the ALeRCE broker had identified the [object associated with this alert](https://lsst.alerce.online/object/170059286935240883?survey=lsst) as a potential Active Galactic Nucleus (AGN)." + "At the time this tutorial was prepared, the ALeRCE broker had identified the [object associated with this alert](https://lsst.alerce.online/object/170059286935240883?survey=lsst) as a potential Active Galactic Nucleus (AGN).\n", + "\n", + "Note that the alert retrieval service will also accept IAU-formatted IDs in the form of: LSST-AP-DS-170059317401616524, where the numerical portion is the `diaSourceId`." ] }, { @@ -192,7 +178,7 @@ "source": [ "## 2. Avro format\n", "\n", - "An Avro Object Container File (OCF) is the default response format from Herald.\n", + "An Avro Object Container File (OCF) is the default response format from the alert retrieval service and has the same format as the originally-transmitted alert packet.\n", "\n", "### 2.1. Full packet\n", "\n", @@ -207,6 +193,7 @@ "outputs": [], "source": [ "response = await client.get(url, params={\"ID\": alert_id})\n", + "response.raise_for_status()\n", "response?" ] }, @@ -302,41 +289,42 @@ "source": [ "#### 2.1.2. Lightcurve\n", "\n", - "Extract the band, MJD, and forced PSF difference-image flux from the alert record as `numpy` arrays, and plot the lightcurve." + "Extract the band, MJD, and PSF difference-image fluxes from the alert record as `numpy` arrays, and plot the lightcurve.\n", + "\n", + "Begin by extracting the `diaSource` and previous diaSource (`prvDiaSources`) measurements to pandas dataframes, then concatenate them into a single dataframe." ] }, { "cell_type": "code", "execution_count": null, - "id": "ba74bfb8-9880-4ef2-842b-b062b054d4a5", + "id": "140c18b7-bbfe-4663-8564-735644c75dbf", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(record['diaSource'], index=[0])\n", + "df_prv = pd.DataFrame(record['prvDiaSources'])\n", + "df_all = pd.concat([df, df_prv], ignore_index=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9013c2b-5612-4c9e-a6ce-351ee0de3501", "metadata": {}, "outputs": [], "source": [ - "band = []\n", - "mjd = []\n", - "flux = []\n", - "for pdfs in record['prvDiaForcedSources']:\n", - " band.append(pdfs['band'])\n", - " mjd.append(pdfs['midpointMjdTai'])\n", - " flux.append(pdfs['psfFlux'])\n", - "lc_band = np.asarray(band, dtype='str')\n", - "lc_mjd = np.asarray(mjd, dtype='float')\n", - "lc_flux = np.asarray(flux, dtype='float')\n", - "del band, mjd, flux\n", - "\n", "fig = plt.figure(figsize=(6, 3))\n", "for f, filt in enumerate(filter_names):\n", - " fx = np.where(lc_band == filt)[0]\n", - " if len(fx) > 0:\n", - " plt.plot(lc_mjd[fx], lc_flux[fx], filter_symbols[filt], ms=5, mew=1,\n", + " df_pick_band = df_all.loc[df_all['band'] == filt]\n", + " if len(df_pick_band) > 0:\n", + " plt.plot(df_pick_band['midpointMjdTai'], df_pick_band['psfFlux'], filter_symbols[filt], ms=5, mew=1,\n", " alpha=0.7, mec=filter_colors[filt], color='None', label=filt)\n", - " del fx\n", + "\n", "plt.xlabel('MJD')\n", - "plt.ylabel('forced difference-image flux [nJy]')\n", + "plt.ylabel('diaSource PSF flux [nJy]')\n", "plt.legend(bbox_to_anchor=(1.05, 1), handletextpad=0, loc='upper left')\n", "plt.tight_layout()\n", - "plt.show()\n", - "del lc_band, lc_mjd, lc_flux" + "plt.show()" ] }, { @@ -344,7 +332,7 @@ "id": "dc1ee5e6-6592-46e2-8147-2f0e69e0442c", "metadata": {}, "source": [ - "> **Figure 1:** The forced PSF difference-image flux lightcurve." + "> **Figure 1:** The PSF difference-image flux lightcurve." ] }, { @@ -360,7 +348,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a2bf9eac-fc48-4ff8-a2da-b91269b3ef90", + "id": "c51bb65a-7214-4f51-92f7-b3df7307b551", "metadata": {}, "outputs": [], "source": [ @@ -373,11 +361,12 @@ "for s, (stamp, name) in enumerate(zip(stamps, stamp_names)):\n", " hdul = fits.HDUList.fromstring(stamp)\n", " data = hdul[0].data\n", - " image = afwImage.ImageF(data.astype(\"float32\"))\n", + " image = CCDData(data.astype(\"float32\"), unit='nJy')\n", + " norm = ImageNormalize(image, interval=MinMaxInterval(),\n", + " stretch=LinearStretch())\n", " plt.sca(ax[s])\n", - " display = afwDisplay.Display(frame=fig)\n", - " display.scale(\"linear\", \"minmax\")\n", - " display.image(image)\n", + " plt.imshow(image, origin='lower', norm=norm, cmap='gray')\n", + " plt.colorbar()\n", " ax[s].set_title(name)\n", "plt.tight_layout()\n", "plt.show()" @@ -456,7 +445,7 @@ } }, "source": [ - "Avro schemas are written in JSON, so convert in order to explore." + "Avro schemas are written in JSON, so use the `response.json()` method to convert to a Python dict in order to explore." ] }, { @@ -474,7 +463,7 @@ "id": "633b43f1-db19-444b-95b4-19e14b84379a", "metadata": {}, "source": [ - "This converts to JSON but use native or something else in the Avro section " + "Print some fields from the schema." ] }, { @@ -539,7 +528,7 @@ "source": [ "### 2.3. Stamps only\n", "\n", - "If only the image stamps (cutouts) from an alert are desired, have Herald only return the images in the response." + "If only the image stamps (cutouts) from an alert are desired, have the alert retrieval service only return the images in the response." ] }, { @@ -583,11 +572,13 @@ "if len(stamps) == 1:\n", " axes = [axes]\n", "for ax, name in zip(axes, stamp_names):\n", - " image = afwImage.ImageF(stamps[name].data.astype(\"float32\"))\n", + " image = CCDData(stamps[name].data.astype(\"float32\"), unit='nJy')\n", + " norm = ImageNormalize(image, interval=MinMaxInterval(),\n", + " stretch=LinearStretch())\n", " plt.sca(ax)\n", - " display = afwDisplay.Display(frame=fig)\n", - " display.scale('linear', 'zscale')\n", - " display.mtv(image, title=name)\n", + " plt.imshow(image, origin='lower', norm=norm, cmap='gray')\n", + " plt.title(name)\n", + " plt.colorbar()\n", "plt.tight_layout()\n", "plt.show()" ] @@ -597,7 +588,7 @@ "id": "5da256b2-fe69-4dae-b3e7-ba1d5472c4fa", "metadata": {}, "source": [ - "> **Figure 3:** The science, template, and difference-image stamps." + "> **Figure 3:** The difference-image, science, and template stamps." ] }, { @@ -625,7 +616,7 @@ "source": [ "## 3. JSON format\n", "\n", - "The JSON response is the full deserialized alert record." + "The JSON response is the full deserialized alert record. Request the JSON formatted response by using the `RESPONSEFORMAT` keyword." ] }, { @@ -683,35 +674,26 @@ { "cell_type": "code", "execution_count": null, - "id": "15235167-1a46-4347-8575-d66532eb3aa9", + "id": "f251faae-1eb2-45c3-900c-59f02d0003fd", "metadata": {}, "outputs": [], "source": [ - "# band = []\n", - "# mjd = []\n", - "# flux = []\n", - "# for pdfs in record['prvDiaForcedSources']:\n", - "# band.append(pdfs['band'])\n", - "# mjd.append(pdfs['midpointMjdTai'])\n", - "# flux.append(pdfs['psfFlux'])\n", - "# lc_band = np.asarray(band, dtype='str')\n", - "# lc_mjd = np.asarray(mjd, dtype='float')\n", - "# lc_flux = np.asarray(flux, dtype='float')\n", - "# del band, mjd, flux\n", - "\n", + "# df = pd.DataFrame(record['diaSource'], index=[0])\n", + "# df_prv = pd.DataFrame(record['prvDiaSources'])\n", + "# df_all = pd.concat([df, df_prv], ignore_index=True)\n", + "# \n", "# fig = plt.figure(figsize=(6, 3))\n", "# for f, filt in enumerate(filter_names):\n", - "# fx = np.where(lc_band == filt)[0]\n", - "# if len(fx) > 0:\n", - "# plt.plot(lc_mjd[fx], lc_flux[fx], filter_symbols[filt], ms=5, mew=1,\n", - "# alpha=0.7, mec=filter_colors[filt], color='None', label=filt)\n", - "# del fx\n", + "# df_pick_band = df_all.loc[df_all['band'] == filt]\n", + "# if len(df_pick_band) > 0:\n", + "# plt.plot(df_pick_band['midpointMjdTai'], df_pick_band['psfFlux'], filter_symbols[filt], ms=5, mew=1,\n", + "# alpha=0.7, mec=filter_colors[filt], color='None', label=filt)\n", + "# \n", "# plt.xlabel('MJD')\n", - "# plt.ylabel('forced difference-image flux [nJy]')\n", + "# plt.ylabel('diaSource PSF flux [nJy]')\n", "# plt.legend(bbox_to_anchor=(1.05, 1), handletextpad=0, loc='upper left')\n", "# plt.tight_layout()\n", - "# plt.show()\n", - "# del lc_band, lc_mjd, lc_flux" + "# plt.show()" ] }, { @@ -719,7 +701,7 @@ "id": "13083cd9-5ee4-40f8-bca1-ef31c1762f72", "metadata": {}, "source": [ - "Option to display the stamps. The code is very similar to that used for the Avro packets, except in the JSON formatted packet the image stamps are base64-encoded bytes. They need to be first decoded with `base64.b64decode` then read as bytes with `io.BytesIO` by `fits.open`." + "Option to display the stamps. The code is very similar to that used for the Avro packets, except in the JSON formatted packet the image stamps are base64-encoded bytes. They need to be first decoded with `base64.b64decode` then read as bytes with `io.BytesIO` by `fits.open`. Note also that the stamps are in a different order than in the Avro packets." ] }, { @@ -732,17 +714,18 @@ "# stamps = [base64.b64decode(record.get(\"cutoutDifference\")),\n", "# base64.b64decode(record.get(\"cutoutScience\")),\n", "# base64.b64decode(record.get(\"cutoutTemplate\"))]\n", - "# stamp_names = ['Science', 'Template', 'Difference']\n", + "# stamp_names = ['Difference', 'Science', 'Template', ]\n", "\n", "# fig, ax = plt.subplots(1, 3, figsize=(9, 3))\n", "# for s, (stamp, name) in enumerate(zip(stamps, stamp_names)):\n", "# hdul = fits.open(io.BytesIO(stamp))\n", "# data = hdul[0].data\n", - "# image = afwImage.ImageF(data.astype(\"float32\"))\n", + "# image = CCDData(data.astype(\"float32\"), unit='nJy')\n", + "# norm = ImageNormalize(image, interval=MinMaxInterval(),\n", + "# stretch=LinearStretch())\n", "# plt.sca(ax[s])\n", - "# display = afwDisplay.Display(frame=fig)\n", - "# display.scale(\"linear\", \"minmax\")\n", - "# display.image(image)\n", + "# plt.imshow(image, origin='lower', norm=norm, cmap='gray')\n", + "# plt.colorbar()\n", "# ax[s].set_title(name)\n", "# plt.tight_layout()\n", "# plt.show()\n", @@ -844,7 +827,24 @@ "outputs": [], "source": [ "hdu = hdul['PRIMARY']\n", - "hdu.header\n", + "hdu.header" + ] + }, + { + "cell_type": "markdown", + "id": "5ba9785f-a260-4bca-ba22-e762cee48762", + "metadata": {}, + "source": [ + "Clean up." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8985dd43-b8c5-4a85-8f99-8c8444f5ff19", + "metadata": {}, + "outputs": [], + "source": [ "del hdu" ] }, @@ -854,7 +854,7 @@ "metadata": {}, "source": [ "**DIFFIM, SCIENCE, TEMPLATE**\\\n", - "Option to display the image stamp \"triplet\" of science, template, and difference image for the `dia_hdul`." + "Option to display the image stamp \"triplet\" of science, template, and difference image for the `hdul`." ] }, { @@ -866,13 +866,13 @@ "source": [ "# fig, ax = plt.subplots(1, 3, figsize=(9, 3))\n", "# for i, ext in enumerate([\"SCIENCE\", \"TEMPLATE\", \"DIFFIM\"]):\n", - "# image = afwImage.ImageF(hdul[ext].data.astype(\"float32\"))\n", + "# image = CCDData(hdul[ext].data.astype(\"float32\"), unit='nJy')\n", + "# norm = ImageNormalize(image, interval=MinMaxInterval(),\n", + "# stretch=LinearStretch())\n", "# plt.sca(ax[i])\n", - "# display = afwDisplay.Display(frame=fig)\n", - "# display.scale(\"asinh\", \"zscale\")\n", - "# display.image(image)\n", + "# plt.imshow(image, origin='lower', norm=norm, cmap='gray')\n", + "# plt.colorbar()\n", "# ax[i].set_title(ext)\n", - "# del image\n", "# plt.tight_layout()\n", "# plt.show()" ] @@ -932,7 +932,7 @@ "# mew=1, alpha=0.7, mec=filter_colors[filt], color='None', label=filt)\n", "# del fx\n", "# plt.xlabel('MJD')\n", - "# plt.ylabel('forced difference-image flux [nJy]')\n", + "# plt.ylabel('difference-image PSF flux [nJy]')\n", "# plt.legend(bbox_to_anchor=(1.05, 1), handletextpad=0, loc='upper left')\n", "# plt.tight_layout()\n", "# plt.show()\n", @@ -945,12 +945,80 @@ "id": "88fe362b-74f3-47e2-9e12-e90329c4cfce", "metadata": {}, "source": [ - "**FORCEDPHOT** \\\n", - "Contains the forced photometry history.\n", + "**FORCEDPHOT**\n", "\n", - "**SSSOURCE** \\\n", - "For alerts associated with moving objects.\n", - "Contains the LSST-computed per-source instantaneous quantities at the time of observation (e.g., heliocentric and topocentric positions and velocities)." + "The `FORCEDPHOT` extension contains forced measurement data on all previous visits at the position of the `diaSource` (the lightcurve). This includes measurements on the difference images and the science images.\n", + "\n", + "Option to extract the `diaSource` data and plot the forced photometry lightcurve." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f14e6bb2-6daf-425f-acfb-08d23044fedd", + "metadata": {}, + "outputs": [], + "source": [ + "# hdu = hdul['FORCEDPHOT']\n", + "# data = hdu.data\n", + "# fig = plt.figure(figsize=(6, 3))\n", + "# for f, filt in enumerate(filter_names):\n", + "# fx = np.where(data['band'] == filt)[0]\n", + "# if len(fx) > 0:\n", + "# plt.plot(data['midpointMjdTai'][fx], data['psfFlux'][fx], filter_symbols[filt], ms=5,\n", + "# mew=1, alpha=0.7, mec=filter_colors[filt], color='None', label=filt)\n", + "# del fx\n", + "# plt.xlabel('MJD')\n", + "# plt.ylabel('forced difference-image flux [nJy]')\n", + "# plt.legend(bbox_to_anchor=(1.05, 1), handletextpad=0, loc='upper left')\n", + "# plt.tight_layout()\n", + "# plt.show()\n", + "# del data\n", + "# del hdu" + ] + }, + { + "cell_type": "markdown", + "id": "d8f5c41d-3ade-48ca-acec-f50c6f6ea080", + "metadata": {}, + "source": [ + "**SSSOURCE**\n", + "\n", + "The `SSSOURCE` extension contains the LSST-computed per-source instantaneous quantities at the time of observation (e.g., heliocentric and topocentric positions and velocities) for alerts associated with moving objects (i.e., Solar System objects).\n", + "\n", + "Using an ID of a known Solar System object (SSO), retrieve the `SSSOURCE` information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5521e26b-372f-4bb3-b7b4-0b8dff89b19c", + "metadata": {}, + "outputs": [], + "source": [ + "sso_alert_id = \"170059294376985743\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2591fd5-d81f-4b86-a63d-bc038e9fedc9", + "metadata": {}, + "outputs": [], + "source": [ + "sso_response = await client.get(url, params={\"ID\": sso_alert_id, \"RESPONSEFORMAT\": \"fits\"})\n", + "sso_response.raise_for_status()\n", + "sso_hdul = fits.open(io.BytesIO(sso_response.content))\n", + "for hdu in sso_hdul:\n", + " print(hdu.name)" + ] + }, + { + "cell_type": "markdown", + "id": "08ccc9f5-c81c-4a30-9370-c31875505774", + "metadata": {}, + "source": [ + "Note that the header cards are the same for all alerts. However, this alert will have data in the `SSSOURCE` extension. Extract that extension, and examine the `ssObjectId` of the alert." ] }, { @@ -959,6 +1027,37 @@ "id": "71595613-25b8-45c3-a730-b24e215a5284", "metadata": {}, "outputs": [], + "source": [ + "hdu = sso_hdul['SSSOURCE']\n", + "sso_data = hdu.data\n", + "print(sso_data['ssObjectId'])" + ] + }, + { + "cell_type": "markdown", + "id": "409d9c8b-b503-4b6c-8691-a57a40d0e915", + "metadata": {}, + "source": [ + "Option to list the names of all the columns in `SSSOURCE`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "804f1a16-59bc-444c-9760-d01ea196d349", + "metadata": {}, + "outputs": [], + "source": [ + "# for col in sso_hdul['SSSOURCE'].columns:\n", + "# print(col)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfda84a2-1323-4625-9abd-de3ddf9ae8d9", + "metadata": {}, + "outputs": [], "source": [] } ], diff --git a/Prompt/200_Data_products/201_Alerts/201_1_Alert_packets.ipynb b/Prompt/200_Data_products/201_Alerts/201_1_Alert_packets.ipynb index 33224bb5..1aad93c7 100644 --- a/Prompt/200_Data_products/201_Alerts/201_1_Alert_packets.ipynb +++ b/Prompt/200_Data_products/201_Alerts/201_1_Alert_packets.ipynb @@ -31,7 +31,7 @@ "Data Release: [Prompt Products](https://prompt-products.lsst.io/)\\\n", "Container Size: Large\\\n", "LSST Science Pipelines version: r29.2.0\\\n", - "Last verified to run: 2026-04-24\\\n", + "Last verified to run: 2026-05-27\\\n", "Repository: [github.com/lsst/tutorial-notebooks](https://github.com/lsst/tutorial-notebooks)\\\n", "DOI: [10.11578/rubin/dc.20250909.20](https://doi.org/10.11578/rubin/dc.20250909.20)" ] @@ -45,7 +45,7 @@ "\n", "**LSST data products:** Alert packets.\n", "\n", - "**Packages:** `lsst.rsp.RSPClient`, `lsst.rsp..get_service_url`, `fastavro`.\n", + "**Packages:** `lsst.rsp.RSPClient`, `lsst.rsp.get_service_url`, `fastavro`.\n", "\n", "**Credit:**\n", "Originally developed by Rubin Data Management team members.\n", @@ -67,21 +67,22 @@ "Alerts are streamed to **brokers** within a few minutes of the acquisition of every new image to enable real-time analysis and spectroscopic follow-up, and brokers are the recommended way to do time-domain science with alerts.\n", "Alerts are world public and have no proprietary period.\n", "\n", - "**Alert packets are generated by the Prompt processing pipeline**, which runs **Difference Image Analysis (DIA)** for each new exposure, provided a template image exists for that sky location.\n", + "**Alert packets are generated by the Alert Production pipeline**, which runs **Difference Image Analysis (DIA)** for each new exposure, provided a template image exists for that sky location.\n", "The template image is subtracted from the new processed visit image (the 'science' image) to produce a difference image.\n", "Source detection is run on the difference image, and every source (positive or negative) detected with signal-to-noise ratio $\\geq 5$ becomes a `diaSource`.\n", - "Every `diaSource` is associated with either a `diaObject` (static sky coordinates) or an `mpc_orbit` (moving object in the solar system with a known orbit).\n", + "Every `diaSource` is associated with either a `diaObject` (static sky coordinates) or an `ssObject` (Solar System object).\n", "One alert is created per `diaSource`.\n", "\n", - "**What is included in an alert packet?**\n", + "**What is included in an alert packet?** \\\n", + "Alert packets include the following information, and much more. See [the Prompt Products documentation](https://prompt-products.lsst.io/products/alerts/index.html#contents) for a more complete list.\n", "1. The triggering `diaSource` record, which includes photometric and astrometric measurements and metadata.\n", "2. The associated `diaObject` or `mpc_orbit` record, which includes derived properties for the object.\n", "3. Previous `diaSource` records associated with the `diaObject` from the last 12 months, maximum.\n", "4. Image cutouts (stamps) from the science, template, and difference images, for the triggering `diaSource`.\n", "\n", "**How do alerts relate to Rubin's other Prompt Products?** \\\n", - "The data in the alert packets will also stored in a queryable database, the Prompt Products Database (PPDB), which will update on a ~24-hour timescale and will available via the Rubin Science Platform (RSP).\n", - "The contents of the PPDB are public but RSP access is not (unlike broker access).\n", + "The data in the alert packets will also be stored in a queryable database, the Prompt Products Database (PPDB), which will update on a ~24-hour timescale and will be available via the Rubin Science Platform (RSP).\n", + "The *contents* of the PPDB are public but RSP access is not (unlike broker access).\n", "The processed visit images and difference images become available via the RSP after an 80-hour embargo, and are subject to the two-year proprietary period. (See the [Rubin Data Policy](https://rubinobservatory.org/for-scientists/data-products/data-policy).)\n", "\n", "**Additional resources:**\n", @@ -91,7 +92,7 @@ "\n", "**This tutorial** focuses on the details of the packets' schema and data contents, and how packets for static-sky and moving objects differ.\n", "\n", - "**Related tutorials:** The 100-level tutorial on the alert retrieval service, Herald, focuses on how the service works, and options for packet retrieval formats (e.g., FITS, JSON, schema-only, cutouts-only)." + "**Related tutorials:** The 100-level tutorial on the alert retrieval service focuses on how the service works, and options for packet retrieval formats (e.g., FITS, JSON, schema-only, cutouts-only)." ] }, { @@ -101,8 +102,8 @@ "source": [ "### 1.1. Import packages\n", "\n", - "Import the `fastavro` package for reading alerts from Herald in the default Avro format.\n", - "Import the `RSPClient` and `get_service_url` in order to access Herald.\n", + "Import the `fastavro` package for reading alerts from the alert retrieval service in the default Avro format.\n", + "Import the `RSPClient` and `get_service_url` in order to access the alert retrieval service.\n", "Also import a range of other plotting and analyis packages." ] }, @@ -116,15 +117,18 @@ "import fastavro\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "import pandas as pd\n", "import io\n", "from astropy.io import fits\n", "from astropy.wcs import WCS\n", "from astropy.coordinates import SkyCoord\n", + "from astropy.nddata import CCDData\n", + "from astropy.visualization import (MinMaxInterval, ZScaleInterval,\n", + " AsinhStretch, LinearStretch,\n", + " ImageNormalize)\n", "from astropy.wcs.utils import skycoord_to_pixel\n", "\n", "from lsst.rsp import RSPClient, get_service_url\n", - "import lsst.afw.display as afwDisplay\n", - "import lsst.afw.image as afwImage\n", "from lsst.utils.plotting import (get_multiband_plot_colors,\n", " get_multiband_plot_symbols)" ] @@ -136,7 +140,7 @@ "source": [ "### 1.2. Define parameters\n", "\n", - "Establish the connection to Herald by getting the `url` for the alerts retrieval service and instantiating the `RSPClient`." + "Establish the connection to the alert retrieval service by getting the `url` for the service and instantiating the `RSPClient`." ] }, { @@ -150,24 +154,6 @@ "client = RSPClient(\"\")" ] }, - { - "cell_type": "markdown", - "id": "7b7a30e7-8b45-4c0e-8994-fddce488724e", - "metadata": {}, - "source": [ - "Set the `afwDisplay` backend to `matplotlib`, to be used when displaying the image stamps." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8aecaa66-9ac5-4592-b8a1-c0f9a625c760", - "metadata": {}, - "outputs": [], - "source": [ - "afwDisplay.setDefaultBackend(\"matplotlib\")" - ] - }, { "cell_type": "markdown", "id": "15bd5483-126a-4652-b45c-10a98d702889", @@ -230,12 +216,12 @@ "source": [ "### 2.1. Retrieve the packets\n", "\n", - "An Avro Object Container File (OCF) is the default response format from Herald.\n", + "An Avro Object Container File (OCF) is the default response format from the alert retrieval service.\n", "Other formats (JSON, FITS) are possible, as demonstrated in the 100-level tutorial on the alert retrieval service.\n", "\n", "Pass the `url` to the `client` along with the `alert_id`.\n", - "Use the `await` command to pause execute of the cell's code until a response is received, then show the `response`.\n", - "Retreive the alert for the `snia_alert_id`." + "Use the `await` command to pause execution of the cell's code until a response is received, then show the `response`.\n", + "Retrieve the alert for the `snia_alert_id`." ] }, { @@ -332,7 +318,7 @@ "source": [ "### 2.2. Packet schema\n", "\n", - "The `fastavro.reader` used above reads the embedded schema automatically\n", + "The `fastavro.reader` used above reads the embedded schema automatically.\n", "\n", "Extract the schema, and show that the keys and fields are the same for each (but for triggering `diaSources` that are associated with a `diaObject` instead of a `ssObject`, the fields `ssSource` and `mpc_orbits` won't be populated)." ] @@ -493,7 +479,7 @@ "id": "aae8cf9b-c0ab-40d3-95f4-dacdc5a9ebe7", "metadata": {}, "source": [ - "Option to print the keys of `diaSource`, which are the same all objects." + "Option to print the keys of `diaSource`, which are the same for all objects." ] }, { @@ -516,7 +502,7 @@ "* `band`: LSST filter of the observation, one of: $ugrizy$.\n", "* `midpointMjdTai`: The MJD at the midpoint of the exposure.\n", "* `psfFlux`, `psfFluxErr`: The PSF flux and its error, measured on the difference image, in nJy.\n", - "* `scienceFlux`, `scienceFluxErr`: This PSF flux and its error, measured on the science image, in nJy.\n", + "* `scienceFlux`, `scienceFluxErr`: The PSF flux and its error, measured on the science image, in nJy.\n", "\n", "> **Warning:** while the science flux is useful for variable stars, for supernovae it is typically contaminated by host galaxy light and is inappropriate for use in a lightcurve.\n", "\n", @@ -767,11 +753,11 @@ "\n", "* `ssObjectid`: Unique ID in the `ssObject` table.\n", "* `phaseAngle`: Phase angle between the Sun, object, and observer, in degrees.\n", - "* `eclBeta`, `eclLambda`: The ecliptic latitude and logintude, in degrees.\n", + "* `eclBeta`, `eclLambda`: The ecliptic latitude and longitude, in degrees.\n", "* `elongation`: Solar elongation, in degrees.\n", "* `ephVmag`: Predicted magnitude in V band.\n", "\n", - "The heliocentric and topocentric positions and velocities (`helio_` and `topo_` `x`, `y`, `z`, `vx`, `vy`, `vz`) and also included, among other parameters.\n", + "The heliocentric and topocentric positions and velocities (`helio_` and `topo_` `x`, `y`, `z`, `vx`, `vy`, `vz`) are also included, among other parameters.\n", "\n", "Print a few of the key column values." ] @@ -865,7 +851,7 @@ } }, "source": [ - "Create `numpy` arrays of the previous detections." + "Create `pandas` dataframes of the previous detections." ] }, { @@ -875,17 +861,11 @@ "metadata": {}, "outputs": [], "source": [ - "band = []\n", - "mjd = []\n", - "flux = []\n", - "for pds in snia_record['prvDiaSources']:\n", - " band.append(pds['band'])\n", - " mjd.append(pds['midpointMjdTai'])\n", - " flux.append(pds['psfFlux'])\n", - "lc_band = np.asarray(band, dtype='str')\n", - "lc_mjd = np.asarray(mjd, dtype='float')\n", - "lc_flux = np.asarray(flux, dtype='float')\n", - "del band, mjd, flux" + "df = pd.DataFrame(snia_record['diaSource'], index=[0])\n", + "df_prv = pd.DataFrame(snia_record['prvDiaSources'])\n", + "df_lc = pd.concat([df, df_prv], ignore_index=True)\n", + "\n", + "del df, df_prv" ] }, { @@ -901,7 +881,7 @@ } }, "source": [ - "Create `numpy` arrays of the forced PSF photometry on the past images, which includes the image with the triggering detection." + "Create `pandas` dataframes of the forced PSF photometry on the past images, which includes the image with the triggering detection." ] }, { @@ -911,17 +891,11 @@ "metadata": {}, "outputs": [], "source": [ - "band = []\n", - "mjd = []\n", - "flux = []\n", - "for pds in snia_record['prvDiaForcedSources']:\n", - " band.append(pds['band'])\n", - " mjd.append(pds['midpointMjdTai'])\n", - " flux.append(pds['psfFlux'])\n", - "flc_band = np.asarray(band, dtype='str')\n", - "flc_mjd = np.asarray(mjd, dtype='float')\n", - "flc_flux = np.asarray(flux, dtype='float')\n", - "del band, mjd, flux" + "df = pd.DataFrame(snia_record['diaSource'], index=[0])\n", + "df_prv = pd.DataFrame(snia_record['prvDiaForcedSources'])\n", + "df_flc = pd.concat([df, df_prv], ignore_index=True)\n", + "\n", + "del df, df_prv" ] }, { @@ -929,7 +903,7 @@ "id": "ea1a9163-5706-4a55-ae75-da94df78f3a1", "metadata": {}, "source": [ - "Plot separately the lightcurves using the detections (plus the triggering detection), and using the force photometry.\n", + "Plot separately the lightcurves using the detections (plus the triggering detection), and using the forced photometry.\n", "\n", "The fluxes in the g, r, and i bands overlap, so impose a flux offset by filter when plotting." ] @@ -948,15 +922,15 @@ " if alert_band == filt:\n", " ax[0].plot(alert_mjd-61000, alert_flux+offsets[f], filter_symbols[filt], ms=7,\n", " mew=2, alpha=1, color=filter_colors[filt])\n", - " fx = np.where(lc_band == filt)[0]\n", - " if len(fx) > 0:\n", - " ax[0].plot(lc_mjd[fx]-61000, lc_flux[fx]+offsets[f], filter_symbols[filt], ms=7,\n", - " mew=2, alpha=0.5, mec=filter_colors[filt], color='None')\n", - " fx = np.where(flc_band == filt)[0]\n", - " if len(fx) > 0:\n", - " ax[1].plot(flc_mjd[fx]-61000, flc_flux[fx]+offsets[f], filter_symbols[filt], ms=7,\n", - " mew=2, alpha=0.5, mec=filter_colors[filt], color='None',\n", - " label=filt + \" +\" + str(offsets[f]))\n", + " df_pick_band = df_lc.loc[df_lc['band'] == filt]\n", + " if len(df_pick_band) > 0:\n", + " ax[0].plot(df_pick_band['midpointMjdTai']-61000, df_pick_band['psfFlux']+offsets[f], filter_symbols[filt],\n", + " ms=5, mew=1, alpha=0.7, mec=filter_colors[filt], color='None')\n", + " df_pick_band = df_flc.loc[df_flc['band'] == filt]\n", + " if len(df_pick_band) > 0:\n", + " ax[1].plot(df_pick_band['midpointMjdTai']-61000, df_pick_band['psfFlux']+offsets[f], filter_symbols[filt],\n", + " ms=5, mew=1, alpha=0.7, mec=filter_colors[filt], color='None',\n", + " label=filt + \" +\" + str(offsets[f]))\n", "ax[0].set_xlabel('MJD-61000')\n", "ax[1].set_xlabel('MJD-61000')\n", "ax[0].set_ylabel('difference flux [nJy]')\n", @@ -1093,14 +1067,18 @@ "for s, (stamp, name) in enumerate(zip(stamps, stamp_names)):\n", " hdul = fits.HDUList.fromstring(stamp)\n", " data = hdul[0].data\n", - " image = afwImage.ImageF(data.astype(\"float32\"))\n", + " image = CCDData(data.astype(\"float32\"), unit='nJy')\n", + "# norm = ImageNormalize(image, interval=ZScaleInterval(),\n", + "# stretch=AsinhStretch())\n", + "# norm = ImageNormalize(image, interval=ZScaleInterval(),\n", + "# stretch=LinearStretch())\n", + " norm = ImageNormalize(image, interval=MinMaxInterval(),\n", + " stretch=LinearStretch())\n", " plt.sca(ax[s])\n", - " display = afwDisplay.Display(frame=fig)\n", - " # display.scale(\"asinh\", \"zscale\")\n", - " # display.scale(\"linear\", \"zscale\")\n", - " display.scale(\"linear\", \"minmax\")\n", - " display.image(image)\n", - " plt.plot(pixels[0], pixels[0], 'o', ms=20, mew=1, color='None', mec='yellow')\n", + " plt.imshow(image, origin='lower', norm=norm, cmap='gray')\n", + " plt.colorbar()\n", + " ax[s].set_title(name)\n", + " plt.plot(pixels[0], pixels[1], 'o', ms=20, mew=1, color='None', mec='yellow')\n", " plt.plot([pixels[0], temp_pixels[0]], [pixels[1], temp_pixels[1]], color='cyan')\n", " plt.text(temp_pixels[0], temp_pixels[1], 'N', color='cyan', fontsize=12)\n", " ax[s].set_title(name)\n", @@ -1114,7 +1092,7 @@ "id": "aa4d7992-b4f8-4725-bd8a-e01a90b2badb", "metadata": {}, "source": [ - "> **Figure 2:** Image stamps from the SNIa alert packet. The literature coordinats of the SNIa are marked with a yellow circle, and the north direction indicated with a cyan line." + "> **Figure 2:** Image stamps from the SNIa alert packet. The literature coordinates of the SNIa are marked with a yellow circle, and the north direction indicated with a cyan line." ] }, { @@ -1140,11 +1118,12 @@ "for s, (stamp, name) in enumerate(zip(stamps, stamp_names)):\n", " hdul = fits.HDUList.fromstring(stamp)\n", " data = hdul[0].data\n", - " image = afwImage.ImageF(data.astype(\"float32\"))\n", + " image = CCDData(data.astype(\"float32\"), unit='nJy')\n", + " norm = ImageNormalize(image, interval=ZScaleInterval(),\n", + " stretch=LinearStretch())\n", " plt.sca(ax[s])\n", - " display = afwDisplay.Display(frame=fig)\n", - " display.scale(\"linear\", \"zscale\")\n", - " display.image(image)\n", + " plt.imshow(image, origin='lower', norm=norm, cmap='gray')\n", + " plt.colorbar()\n", " ax[s].set_title(name)\n", "plt.suptitle('Image stamps from the MBA alert')\n", "plt.tight_layout()\n",