From 0b1b44f7e724459428e039ff807d82307a76ff11 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Tue, 8 Sep 2026 18:19:41 +0000 Subject: [PATCH] Get DAAC credentials from MAAP when there are no Earthdata credentials A MAAP DPS worker has no Earthdata credentials of any kind: it runs as root with no ~/.netrc, and earthaccess's netrc and environment strategies both come up empty. What it does have is MAAP's own auth -- $MAAP_PGT is set in the container -- and maap.aws.earthdata_s3_credentials() exchanges that for the DAAC's short-lived read credentials. See docs.maap-project.org, science/NISAR/NISAR_access.html cell [2]. get_s3fs(daac=...) now tries that broker first and falls back to earthaccess.get_s3fs_session() otherwise, so nothing changes off MAAP: without $MAAP_PGT the new path costs one dict lookup and returns None. A DAAC with no endpoint listed falls through the same way. Every other failure -- maap-py missing, the broker refusing, a malformed response -- warns before falling back, rather than quietly yielding a session from somewhere other than where the caller thinks, which is the kind of thing that only surfaces much later as an unexplained permission error. find_ATL11_granules no longer hard-codes earthaccess.login(strategy='netrc'). That was the single strategy a DPS worker cannot satisfy, and it raised LoginStrategyUnavailable before the search ever reached CMR -- for a CMR metadata search, which needs no authentication at all. Only the granule READS need credentials, and those go through get_s3fs(). Bare login() still picks up whatever a local user has (environment, then netrc, then interactive), and failure now warns instead of raising. Verified end to end from the MAAP ADE before this was committed: - CMR granule search with no auth: OK - _s3fs_from_maap('NSIDC') returns a working S3FileSystem - fs.info() and a real read of s3://nsidc-cumulus-prod-protected/ATLAS/ATL11/007/2019/03/29/ ATL11_000103_0331_007_04.h5 -> 3979910 bytes, first 8 bytes \x89HDF... - an unlisted DAAC returns None, so the earthaccess path is unchanged Note that fs.ls() on the bucket ROOT is denied by NSIDC policy even with valid credentials; object reads are what work, and what we do. One behaviour change worth knowing: in the ADE, where $MAAP_PGT is also set, DAAC sessions now come from MAAP rather than earthaccess. That is the path just verified above, and it falls back with a warning if the broker declines. The credentials are short-lived and get_s3fs() caches the session for the life of the process. Fine for a per-tile job of minutes; a process outliving the token's `expiration` would need to re-derive, and nothing does that yet. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013MdMEXw2s6eeGtaGcaDxpF --- pointCollection/io_utils.py | 93 ++++++++++++++++++-- pointCollection/scripts/query_ATL11_cloud.py | 29 +++++- 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/pointCollection/io_utils.py b/pointCollection/io_utils.py index ce3d5e4..ccfaa3c 100644 --- a/pointCollection/io_utils.py +++ b/pointCollection/io_utils.py @@ -10,6 +10,16 @@ _S3FS_CACHE = {} +# DAAC -> the DAAC's own S3-credentials endpoint, for the MAAP path in +# get_s3fs(). MAAP brokers these: maap.aws.earthdata_s3_credentials() +# returns short-lived accessKeyId/secretAccessKey/sessionToken using MAAP's own +# auth, so a MAAP DPS worker needs no Earthdata credentials of its own -- no +# .netrc, nothing at rest. A DAAC that is not listed simply falls through to +# earthaccess, which is still the right answer off MAAP. +MAAP_S3_CREDENTIALS_ENDPOINTS = { + 'NSIDC': 'https://data.nsidc.earthdatacloud.nasa.gov/s3credentials', +} + # Block size for remote reads that pull windows out of a large file. fsspec's # 5 MiB default is sized for reading a file end to end; a windowed read of a # chunked HDF5 file touches scattered chunks, and the read-ahead is then mostly @@ -72,9 +82,11 @@ def get_s3fs(daac='NSIDC', **kwargs): Parameters ---------- daac : str or None, default 'NSIDC' - If a DAAC name, the session is created with - earthaccess.get_s3fs_session(), which supplies the short-lived - in-region credentials that DAAC's cloud buckets require. + If a DAAC name, the session carries the short-lived in-region + credentials that DAAC's cloud buckets require. On MAAP those come + from MAAP's own credential broker (see _s3fs_from_maap), which needs + no Earthdata credentials at all; everywhere else, and whenever the + broker declines, they come from earthaccess.get_s3fs_session(). If None, an ordinary s3fs.S3FileSystem() is returned, which picks up whatever the default AWS credential chain provides (environment, ~/.aws, or an instance/task role). That is the right choice for @@ -91,10 +103,81 @@ def get_s3fs(daac='NSIDC', **kwargs): import s3fs _S3FS_CACHE[key] = s3fs.S3FileSystem(**kwargs) else: - import earthaccess - _S3FS_CACHE[key] = earthaccess.get_s3fs_session(daac=daac, **kwargs) + fs = _s3fs_from_maap(daac, **kwargs) + if fs is None: + import earthaccess + fs = earthaccess.get_s3fs_session(daac=daac, **kwargs) + _S3FS_CACHE[key] = fs return _S3FS_CACHE[key] + +def _s3fs_from_maap(daac, **kwargs): + """ + Build an s3fs session from MAAP-brokered DAAC credentials. + + Returns None -- with a warning saying why -- whenever this is not a MAAP + environment or the broker will not answer, so the caller falls back to + earthaccess. Off MAAP this costs one dict lookup and returns None. + + This exists because a MAAP DPS worker has NO Earthdata credentials: it runs + as root with no ~/.netrc, and earthaccess's netrc and environment + strategies both come up empty there. What it does have is MAAP's own auth + ($MAAP_PGT, and a maap_token from config), which + maap.aws.earthdata_s3_credentials() exchanges for the DAAC's temporary + read credentials. See docs.maap-project.org, science/NISAR/NISAR_access.html. + + The credentials are short-lived and get_s3fs() caches the session for the + life of the process. That is fine for a per-tile job of minutes; a process + that runs longer than the token's `expiration` would need to re-derive, and + nothing here does that yet. + + Every failure warns rather than passing silently: a session that quietly + came from somewhere other than where you think is exactly the kind of + failure that only shows up later, as a permission error with no obvious + cause. + """ + import os + import warnings + + endpoint = MAAP_S3_CREDENTIALS_ENDPOINTS.get(str(daac).upper()) + if endpoint is None: + return None + if not os.environ.get('MAAP_PGT'): + # The ADE and DPS workers both set it; its absence means this is not a + # MAAP environment, which is not worth warning about. + return None + + try: + from maap.maap import MAAP + except ImportError as exc: + warnings.warn(f'MAAP_PGT is set but maap-py is not importable ({exc}); ' + f'falling back to earthaccess for {daac}.') + return None + + try: + creds = MAAP( + maap_host=os.environ.get('MAAP_API_HOST', 'api.maap-project.org') + ).aws.earthdata_s3_credentials(endpoint) + return _s3fs_with_credentials(creds, **kwargs) + except Exception as exc: + warnings.warn(f'MAAP could not broker {daac} credentials from {endpoint} ' + f'({type(exc).__name__}: {exc}); falling back to earthaccess.') + return None + + +def _s3fs_with_credentials(creds, **kwargs): + """s3fs session from an earthdata_s3_credentials() response.""" + import s3fs + missing = [k for k in ('accessKeyId', 'secretAccessKey', 'sessionToken') + if k not in creds] + if missing: + raise KeyError(f'credential response is missing {missing}') + return s3fs.S3FileSystem(anon=False, + key=creds['accessKeyId'], + secret=creds['secretAccessKey'], + token=creds['sessionToken'], + **kwargs) + def open_remote(filename, mode='rb', fs=None, block_size=None, daac='NSIDC'): """ Open a remote (e.g. s3://) file as a file object. diff --git a/pointCollection/scripts/query_ATL11_cloud.py b/pointCollection/scripts/query_ATL11_cloud.py index bb906b7..98f04f3 100644 --- a/pointCollection/scripts/query_ATL11_cloud.py +++ b/pointCollection/scripts/query_ATL11_cloud.py @@ -49,10 +49,37 @@ def find_ATL11_granules(bounding_box, short_name='ATL11', **search_kwargs): list of earthaccess.DataGranule """ import earthaccess - earthaccess.login(strategy='netrc') + _try_earthaccess_login() return earthaccess.search_data(short_name=short_name, bounding_box=bounding_box, **search_kwargs) +def _try_earthaccess_login(): + """ + Log in to Earthdata if we can, and carry on if we cannot. + + A CMR metadata search needs no authentication -- only the granule READS do, + and those go through pc.io_utils.get_s3fs(), which on MAAP gets its + credentials from MAAP's broker rather than from earthaccess. This used to + be earthaccess.login(strategy='netrc'), which hard-codes the ONE strategy a + MAAP DPS worker cannot satisfy: it runs as root with no ~/.netrc, so every + search raised LoginStrategyUnavailable before reaching CMR at all, even + though the search itself needed no credentials. + + Bare login() tries environment, then netrc, then interactive, so it still + picks up whatever a local user has. Failure warns rather than raising: + the caller may well not need it. + """ + import warnings + import earthaccess + try: + earthaccess.login() + except Exception as exc: + warnings.warn(f'earthaccess.login() failed ({type(exc).__name__}: {exc}); ' + 'continuing, since a CMR search needs no credentials. ' + 'Granule reads get their credentials separately, via ' + 'pointCollection.io_utils.get_s3fs().') + + def index_path_for_granule(granule_basename, index_root): """ Naming convention for a granule's pre-built geoIndex file, matching the