bioepic-data/ecosim-agent / .agents/skills/ameriflux-site-info
ameriflux-site-info
VerifiedExtract AmeriFlux site metadata and map it to EcoSIM JSON variables. Use when a task needs site latitude, longitude, elevation, mean annual temperature, Koppen climate code, or vegetation type for an AmeriFlux site ID or flux-site name.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 8.7 KB |
| extract_ameriflux_site_data.py | script | 30.4 KB |
| test_extract_ameriflux_site_data.py | script | 9.9 KB |
extract_ameriflux_site_data.py
View on GitHub#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import os
import re
import sys
from html import unescape
from html.parser import HTMLParser
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
from urllib.request import Request, urlopen
import numpy as np
try:
import requests
except ImportError: # pragma: no cover - cached metadata should still work.
requests = None
try:
from playwright.sync_api import sync_playwright
except ImportError: # pragma: no cover - requests-only extraction should still work.
sync_playwright = None
AMERIFLUX_SITEINFO_URL = "https://ameriflux.lbl.gov/sites/siteinfo/{site_id}"
KOPPEN_MAP = {
"Af": 11, "Am": 12, "As": 13, "Aw": 14, "BWk": 21, "BWh": 22, "BSk": 26, "BSh": 27,
"Cfa": 31, "Cfb": 32, "Cfc": 33, "Csa": 34, "Csb": 35, "Csc": 36, "Cwa": 37, "Cwb": 38, "Cwc": 39,
"Dfa": 41, "Dfb": 42, "Dfc": 43, "Dfd": 44, "Dsa": 45, "Dsb": 46, "Dsc": 47, "Dsd": 48,
"Dwa": 49, "Dwb": 50, "Dwc": 51, "Dwd": 52, "ET": 61, "EF": 62,
}
KOPPEN_BY_UPPER = {code.upper(): code for code in KOPPEN_MAP}
IGBP_CODES = {
"ENF", "EBF", "DNF", "DBF", "MF", "CSH", "OSH", "WSA", "SAV", "GRA",
"WET", "CRO", "URB", "CNV", "SNO", "BSV", "WAT",
}
IGBP_NAME_TO_CODE = {
"evergreen needleleaf": "ENF",
"evergreen broadleaf": "EBF",
"deciduous needleleaf": "DNF",
"deciduous broadleaf": "DBF",
"mixed forest": "MF",
"closed shrub": "CSH",
"open shrub": "OSH",
"woody savanna": "WSA",
"savanna": "SAV",
"grassland": "GRA",
"grasslands": "GRA",
"wetland": "WET",
"wetlands": "WET",
"cropland": "CRO",
"croplands": "CRO",
"urban": "URB",
"snow": "SNO",
"barren": "BSV",
"water": "WAT",
}
REQUIRED_RAW_FIELDS = ("latitude", "longitude", "elevation", "MAT", "climate_code", "igbp_type")
FIELD_PATTERNS = {
"latitude": (r"\bsite\s+latitude\b", r"\blatitude\b", r"\blat\b"),
"longitude": (r"\bsite\s+longitude\b", r"\blongitude\b", r"\blong\b", r"\blon\b"),
"elevation": (r"\belevation\b", r"\baltitude\b", r"\belev\b"),
"MAP": (r"\bmean\s+annual\s+precip(?:itation)?\b\.?",),
"MAT": (
r"\bmean\s+annual\s+(?:air\s+)?temperature\b",
r"\bmean\s+annual\s+temp\b",
r"\bmean\s+air\s+temperature\b",
r"\bMAT\b",
),
"climate_code": (
r"\bkoppen[-\s]*(?:geiger)?(?:\s+climate)?(?:\s+class(?:ification)?)?\b",
r"\bkoeppen[-\s]*(?:geiger)?(?:\s+climate)?(?:\s+class(?:ification)?)?\b",
r"\bclimate\s+(?:code|class|classification|zone)\b",
),
"igbp_type": (
r"\bIGBP\b",
r"\bvegetation\s+(?:type|class|classification)\b",
r"\bland\s+cover\s+(?:type|class|classification)\b",
),
}
# Approximate (lat_min, lat_max, lon_min, lon_max) envelopes of countries that host
# AmeriFlux sites; the site ID prefix is the ISO 3166 country code.
COUNTRY_BOUNDS = {
"US": (18.5, 71.6, -179.9, -66.5),
"CA": (41.6, 83.2, -141.1, -52.5),
"MX": (14.5, 32.8, -118.5, -86.7),
"PR": (17.8, 18.6, -67.3, -65.2),
"CR": (8.0, 11.3, -86.0, -82.5),
"PA": (7.1, 9.7, -83.1, -77.1),
"BR": (-33.8, 5.3, -74.0, -28.8),
"AR": (-55.1, -21.7, -73.6, -53.6),
"CL": (-56.0, -17.5, -75.8, -66.4),
"PE": (-18.4, -0.0, -81.4, -68.6),
"BO": (-22.9, -9.6, -69.7, -57.4),
"CO": (-4.3, 13.4, -79.1, -66.8),
"EC": (-5.1, 1.5, -92.1, -75.1),
"GF": (2.1, 5.8, -54.6, -51.6),
"UY": (-35.0, -30.0, -58.5, -53.0),
"PY": (-27.6, -19.2, -62.7, -54.2),
}
COUNTRY_BOUNDS_MARGIN_DEG = 0.5
# Region words in the site Description and the latitude range they imply.
DESCRIPTION_LATITUDE_HINTS = (
(r"\bsouthern hemisphere\b", -90.0, 0.0),
(r"\bnorthern hemisphere\b", 0.0, 90.0),
(r"\bsouth america\b", -56.0, 13.0),
(r"\bcentral america\b", 7.0, 19.0),
(r"\bcaribbean\b", 10.0, 27.0),
(r"\bamazon(?:ia|ian)?\b", -20.0, 6.0),
(r"\bpantanal\b", -22.0, -15.0),
(r"\bpatagonia\b", -56.0, -37.0),
(r"\balaska\b", 51.0, 72.0),
(r"\barctic\b", 60.0, 90.0),
)
# Plausible mean annual temperature (degC) for each Koppen main group.
KOPPEN_MAT_RANGES = {"A": (15.0, 35.0), "C": (-3.0, 30.0), "D": (-25.0, 16.0), "E": (-60.0, 8.0)}
TROPICAL_MAX_ABS_LATITUDE = 35.0
# Gridded Koppen-Geiger map (Beck et al. 2018, Sci. Data 5:180214) with integer
# `ids(lat, lon)` indexing a `labels` list, e.g. the ILAMB regions/Koppen.nc
# 0.5-degree product. Searched in this order when --koppen-raster is not given.
KOPPEN_RASTER_ENV = "ECOSIM_KOPPEN_RASTER"
KOPPEN_RASTER_CANDIDATES = ("data/Koppen.nc", "data/koppen/Koppen.nc", "data/regions/Koppen.nc")
class HtmlTextExtractor(HTMLParser):
def __init__(self) -> None:
super().__init__()
self._parts: List[str] = []
self._skip_depth = 0
def handle_starttag(self, tag: str, attrs: Sequence[Tuple[str, Optional[str]]]) -> None:
if tag in {"script", "style", "noscript"}:
self._skip_depth += 1
if tag in {"br", "p", "div", "li", "tr", "td", "th", "section", "article", "h1", "h2", "h3"}:
self._parts.append("\n")
def handle_endtag(self, tag: str) -> None:
if tag in {"script", "style", "noscript"} and self._skip_depth:
self._skip_depth -= 1
if tag in {"p", "div", "li", "tr", "td", "th", "section", "article", "h1", "h2", "h3"}:
self._parts.append("\n")
def handle_data(self, data: str) -> None:
if not self._skip_depth:
self._parts.append(data)
@property
def text(self) -> str:
return "".join(self._parts)
def html_to_text(value: str) -> str:
parser = HtmlTextExtractor()
parser.feed(value)
return parser.text
def normalize_text(value: str) -> str:
value = unescape(value)
value = value.replace("\xa0", " ")
value = value.replace("K\u00f6ppen", "Koppen")
value = value.replace("k\u00f6ppen", "koppen")
value = value.replace("Koeppen", "Koppen")
value = value.replace("koeppen", "koppen")
return value
def split_lines(text: str) -> List[str]:
text = normalize_text(text)
rough_lines = re.split(r"[\r\n]+", text)
lines = []
for line in rough_lines:
cleaned = clean_line(line)
if cleaned:
lines.append(cleaned)
return lines
def clean_line(line: str) -> str:
"""Collapse whitespace and drop label punctuation without touching numeric signs.
Values such as "-16.4980, -56.4120" or "-12.6" sit on their own line on
AmeriFlux pages, so a leading "-" is only removed when it is a bullet
(dash followed by whitespace), never when it is the sign of a number.
"""
cleaned = re.sub(r"\s+", " ", line).strip(" \t:|")
cleaned = re.sub(r"^-+\s+", "", cleaned)
return cleaned.rstrip(" -:|")
def candidate_segments(lines: Sequence[str], patterns: Sequence[str], lookahead: int = 2) -> Iterable[str]:
for index, line in enumerate(lines):
for pattern in patterns:
match = re.search(pattern, line, flags=re.IGNORECASE)
if not match:
continue
yield line[match.end():]
yield line
for offset in range(1, lookahead + 1):
if index + offset < len(lines):
yield lines[index + offset]
def bounded_float(text: str, lower: float, upper: float) -> Optional[float]:
for match in re.finditer(r"[-+]?\d+(?:\.\d+)?", text):
value = float(match.group(0))
if lower <= value <= upper:
return value
return None
def bounded_floats(text: str, lower: float, upper: float) -> List[float]:
values = []
for match in re.finditer(r"[-+]?\d+(?:\.\d+)?", text):
value = float(match.group(0))
if lower <= value <= upper:
values.append(value)
return values
def extract_float(lines: Sequence[str], field: str, lower: float, upper: float) -> Optional[float]:
for segment in candidate_segments(lines, FIELD_PATTERNS[field]):
value = bounded_float(segment, lower, upper)
if value is not None:
return value
return None
def extract_lat_long_pair(lines: Sequence[str]) -> Optional[Tuple[float, float]]:
pair_pattern = r"\blat(?:itude)?\s*,\s*lon(?:g|gitude)?\b"
for index, line in enumerate(lines):
if not re.search(pair_pattern, line, flags=re.IGNORECASE):
continue
for offset in range(0, 4):
if index + offset >= len(lines):
break
values = bounded_floats(lines[index + offset], -180.0, 180.0)
if len(values) >= 2 and -90.0 <= values[0] <= 90.0:
return values[0], values[1]
return None
def normalize_koppen_code(value: Any) -> Optional[str]:
if value is None:
return None
code = str(value).strip()
return KOPPEN_BY_UPPER.get(code.upper())
def extract_koppen_code(lines: Sequence[str]) -> Optional[str]:
code_pattern = "|".join(re.escape(code) for code in sorted(KOPPEN_MAP, key=len, reverse=True))
for segment in candidate_segments(lines, FIELD_PATTERNS["climate_code"], lookahead=3):
match = re.search(rf"\b({code_pattern})\b", segment, flags=re.IGNORECASE)
if match:
return normalize_koppen_code(match.group(1))
full_text = " ".join(lines)
matches = {
normalize_koppen_code(match.group(1))
for match in re.finditer(rf"\b({code_pattern})\b", full_text, flags=re.IGNORECASE)
}
matches.discard(None)
if len(matches) == 1:
return next(iter(matches))
return None
def normalize_igbp_code(value: Any) -> Optional[str]:
if value is None:
return None
candidate = str(value).strip().upper()
if candidate in IGBP_CODES:
return candidate
lowered = str(value).strip().lower()
for name, code in IGBP_NAME_TO_CODE.items():
if name in lowered:
return code
return None
def extract_igbp_type(lines: Sequence[str]) -> Optional[str]:
code_pattern = "|".join(sorted(IGBP_CODES, key=len, reverse=True))
for segment in candidate_segments(lines, FIELD_PATTERNS["igbp_type"], lookahead=3):
match = re.search(rf"\b({code_pattern})\b", segment, flags=re.IGNORECASE)
if match:
return normalize_igbp_code(match.group(1))
code = normalize_igbp_code(segment)
if code:
return code
return None
def extract_site_name(lines: Sequence[str], site_id: str) -> str:
title_pattern = rf"^{re.escape(site_id)}\s*:\s*(.+)$"
for line in lines:
match = re.search(title_pattern, line)
if match:
return match.group(1).strip()
for segment in candidate_segments(lines, (r"\bsite\s+name\b",), lookahead=2):
value = segment.strip(" :|-")
if value and site_id not in value:
return value
return site_id
def extract_precip_climatology(lines: Sequence[str]) -> Optional[float]:
"""Optional MAP without borrowing a number from the following metadata field."""
pattern = FIELD_PATTERNS['MAP'][0]
for index, line in enumerate(lines):
match = re.search(pattern, line, re.IGNORECASE)
if not match:
continue
tail = re.sub(r"\(mm\)", "", line[match.end():], flags=re.IGNORECASE).strip(' :|')
candidates = [tail] if tail else list(lines[index+1:index+2])
for value in candidates:
number = re.fullmatch(r"([+-]?\d+(?:\.\d+)?)\s*(?:mm)?", value.strip(), re.IGNORECASE)
if number and 0 < float(number[1]) <= 15000:
return float(number[1])
return None
return None
def parse_site_metadata_from_text(text: str, site_id: str, source: str) -> Optional[Dict[str, Any]]:
lines = split_lines(text)
if not lines:
return None
lat_long = extract_lat_long_pair(lines)
description = extract_description(lines)
raw = {
"site_name": extract_site_name(lines, site_id),
"latitude": lat_long[0] if lat_long else extract_float(lines, "latitude", -90.0, 90.0),
"longitude": lat_long[1] if lat_long else extract_float(lines, "longitude", -180.0, 180.0),
"elevation": extract_float(lines, "elevation", -500.0, 9000.0),
"MAT": extract_float(lines, "MAT", -90.0, 70.0),
"MAP": extract_precip_climatology(lines),
"climate_code": extract_koppen_code(lines),
"igbp_type": extract_igbp_type(lines),
"description": description,
"source": source,
}
if all(raw.get(field) is not None for field in REQUIRED_RAW_FIELDS):
raw["sign_checks"] = check_signs_against_page(text, raw)
return raw
return None
def extract_description(lines: Sequence[str]) -> Optional[str]:
for index, line in enumerate(lines):
if re.fullmatch(r"(?:site\s+)?description", line, flags=re.IGNORECASE):
for candidate in lines[index + 1:index + 3]:
if len(candidate) > 20:
return candidate[:600]
return None
SIGNED_NUMBER = r"[-+\u2212]?\d+(?:\.\d+)?"
def signed_numbers_after_label(text: str, label_pattern: str, lookahead: int = 2) -> List[float]:
"""Numbers, with their signs, on the label line and the next lines of the raw page text."""
raw_lines = [line.strip() for line in re.split(r"[\r\n]+", normalize_text(text)) if line.strip()]
for index, line in enumerate(raw_lines):
if re.search(label_pattern, line, flags=re.IGNORECASE):
window = " ".join(raw_lines[index:index + 1 + lookahead])
window = re.sub(label_pattern, " ", window, count=1, flags=re.IGNORECASE)
return [float(value.replace("\u2212", "-")) for value in re.findall(SIGNED_NUMBER, window)]
return []
def check_signs_against_page(text: str, raw: Dict[str, Any]) -> Dict[str, str]:
"""Confirm each parsed number appears with the same sign in the raw page text."""
labels = {
"latitude": r"\blat(?:itude)?\b",
"longitude": r"\blon(?:g|gitude)?\b",
"elevation": r"\belevation\b",
"MAT": r"\bmean\s+annual\s+temp(?:erature)?\b",
}
pair = signed_numbers_after_label(text, r"\blat(?:itude)?\s*,\s*lon(?:g|gitude)?\b")
results = {}
for field, label in labels.items():
value = float(raw[field])
numbers = pair if field in ("latitude", "longitude") and pair else signed_numbers_after_label(text, label)
if any(abs(number - value) < 1.0e-9 for number in numbers):
results[field] = "matched"
elif value != 0.0 and any(abs(number + value) < 1.0e-9 for number in numbers):
results[field] = "sign_mismatch"
else:
results[field] = "not_found"
return results
def map_vegetation(igbp: str) -> int:
igbp = str(igbp).upper()
if "ENF" in igbp:
return 11
if "DBF" in igbp:
return 10
return 10
def raw_to_ecosim(site_id: str, raw: Dict[str, Any]) -> Dict[str, Any]:
climate_code = normalize_koppen_code(raw.get("climate_code"))
igbp_type = normalize_igbp_code(raw.get("igbp_type"))
if climate_code is None:
raise ValueError(f"Unknown Koppen climate code: {raw.get('climate_code')}")
if igbp_type is None:
raise ValueError(f"Unknown IGBP vegetation type: {raw.get('igbp_type')}")
normalized_raw = dict(raw)
normalized_raw["climate_code"] = climate_code
normalized_raw["igbp_type"] = igbp_type
return {
"site_name": site_id,
"ALATG": float(raw["latitude"]),
"ALONG": float(raw["longitude"]),
"ALTIG": float(raw["elevation"]),
"ATCAG": float(raw["MAT"]),
"IETYPG": KOPPEN_MAP[climate_code],
"IXTYP1": map_vegetation(igbp_type),
**({'mean_annual_precip_mm': float(raw['MAP']), 'precipitation_reference': raw.get('source', '')} if raw.get('MAP') is not None else {}),
"_raw": normalized_raw,
}
def find_koppen_raster(path: Optional[str] = None) -> Optional[str]:
candidates = [path, os.environ.get(KOPPEN_RASTER_ENV), *KOPPEN_RASTER_CANDIDATES]
for candidate in candidates:
if candidate and os.path.exists(candidate):
return candidate
return None
def koppen_from_coordinates(raster_path: str, lat: float, lon: float) -> Tuple[Optional[str], List[str]]:
"""Koppen class of the nearest raster cell and the classes in its 3x3 neighbourhood.
The neighbourhood absorbs climate boundaries that a coarse grid blurs;
ocean cells are missing and contribute nothing.
"""
from netCDF4 import Dataset
with Dataset(raster_path) as ds:
lats = np.asarray(ds["lat"][:], dtype=float)
lons = np.asarray(ds["lon"][:], dtype=float)
labels = [str(label) for label in ds["labels"][:]]
i = int(np.abs(lats - lat).argmin())
j = int(np.abs(((lons - lon) + 180.0) % 360.0 - 180.0).argmin())
rows = sorted({min(max(i + di, 0), len(lats) - 1) for di in (-1, 0, 1)})
cols = sorted({(j + dj) % len(lons) for dj in (-1, 0, 1)})
block = ds["ids"][rows, cols]
center = ds["ids"][i, j]
def label(value: Any) -> Optional[str]:
if np.ma.is_masked(value) or int(value) < 0 or int(value) >= len(labels):
return None
return normalize_koppen_code(labels[int(value)])
neighbourhood = sorted({code for code in (label(v) for v in np.ma.ravel(block)) if code})
return label(center), neighbourhood
def check_koppen_from_coordinates(
raster_path: Optional[str], lat: float, lon: float, page_code: Optional[str]
) -> Tuple[str, Optional[str], Optional[str], Dict[str, Any]]:
"""Compare the page Koppen class with the raster at the parsed coordinates.
Returns (status, error, warning, detail). Mismatches are warnings: maps and
periods differ near class boundaries, and an opposite-sign location can match
by coincidence. The page-sign and country checks are the hard sign checks;
this one adds a "suspect a lost minus sign" hint when the flipped location fits.
"""
if not raster_path:
return "not_available", None, None, {"reason": f"No Koppen raster (--koppen-raster or ${KOPPEN_RASTER_ENV})."}
if not page_code:
return "not_applicable", None, None, {}
center, neighbourhood = koppen_from_coordinates(raster_path, lat, lon)
detail = {"raster": os.path.abspath(raster_path), "center": center, "neighbourhood": neighbourhood}
if page_code in neighbourhood:
return "passed", None, None, detail
for name, (flat, flon) in (("latitude", (-lat, lon)), ("longitude", (lat, -lon))):
if (flat, flon) == (lat, lon):
continue
_, flipped = koppen_from_coordinates(raster_path, flat, flon)
if page_code in flipped:
detail[f"opposite_{name}_neighbourhood"] = flipped
return "failed", None, (
f"Page Koppen {page_code} is absent near ({lat}, {lon}) (raster {neighbourhood or 'ocean'}) "
f"but present at the opposite-sign {name}; suspect a lost minus sign."
), detail
return "failed", None, (
f"Page Koppen {page_code} differs from the Beck et al. map near ({lat}, {lon}): "
f"{center or 'ocean'} (neighbourhood {neighbourhood or 'ocean'}). Check the coordinates "
"or which Koppen source to trust."
), detail
def validate_site_location(
site_id: str, data: Dict[str, Any], koppen_raster: Optional[str] = None
) -> Dict[str, Any]:
"""Cross-check coordinates and MAT against the site ID, page text, and Koppen class.
Errors (sign lost while parsing, coordinates outside the site-ID country)
mean the metadata must not be used; warnings flag implausible combinations
for review.
"""
raw = data.get("_raw", {})
lat, lon, mat = float(data["ALATG"]), float(data["ALONG"]), float(data["ATCAG"])
errors: List[str] = []
warnings: List[str] = []
checks: Dict[str, str] = {}
sign_checks = raw.get("sign_checks") or {}
mismatched = [field for field, status in sign_checks.items() if status == "sign_mismatch"]
checks["page_signs"] = "failed" if mismatched else ("passed" if sign_checks else "not_available")
if mismatched:
errors.append(f"Parsed {', '.join(mismatched)} lost the sign shown on the AmeriFlux page.")
country = site_id[:2].upper()
bounds = COUNTRY_BOUNDS.get(country)
if bounds is None:
checks["country_bounds"] = f"skipped (no envelope for {country})"
else:
def inside(la: float, lo: float) -> bool:
m = COUNTRY_BOUNDS_MARGIN_DEG
return bounds[0] - m <= la <= bounds[1] + m and bounds[2] - m <= lo <= bounds[3] + m
if inside(lat, lon):
checks["country_bounds"] = "passed"
else:
checks["country_bounds"] = "failed"
hint = ""
if inside(-lat, lon):
hint = f" Latitude {-lat} (opposite sign) would fit; suspect a lost minus sign."
elif inside(lat, -lon):
hint = f" Longitude {-lon} (opposite sign) would fit; suspect a lost minus sign."
errors.append(f"({lat}, {lon}) is outside the {country} envelope implied by {site_id}.{hint}")
description = (raw.get("description") or "") + " " + (raw.get("site_name") or "")
hint_results = []
for pattern, lower, upper in DESCRIPTION_LATITUDE_HINTS:
match = re.search(pattern, description, flags=re.IGNORECASE)
if match:
ok = lower <= lat <= upper
hint_results.append(ok)
if not ok:
warnings.append(
f"Description mentions '{match.group(0)}' (latitude {lower} to {upper}) but latitude is {lat}."
)
checks["description_region"] = (
"not_available" if not hint_results else ("passed" if all(hint_results) else "failed")
)
koppen = normalize_koppen_code(raw.get("climate_code")) or next(
(code for code, number in KOPPEN_MAP.items() if number == data.get("IETYPG")), None
)
group = koppen[0] if koppen else None
if group in KOPPEN_MAT_RANGES:
lower, upper = KOPPEN_MAT_RANGES[group]
koppen_ok = lower <= mat <= upper
if not koppen_ok:
warnings.append(f"MAT {mat} degC is implausible for Koppen {koppen} ({lower} to {upper} degC).")
if group == "A" and abs(lat) > TROPICAL_MAX_ABS_LATITUDE:
koppen_ok = False
warnings.append(f"Tropical Koppen {koppen} at latitude {lat}.")
checks["koppen_consistency"] = "passed" if koppen_ok else "failed"
else:
checks["koppen_consistency"] = "not_applicable"
koppen_status, koppen_error, koppen_warning, koppen_detail = check_koppen_from_coordinates(
koppen_raster, lat, lon, koppen
)
checks["koppen_from_coordinates"] = koppen_status
if koppen_error:
errors.append(koppen_error)
if koppen_warning:
warnings.append(koppen_warning)
status = "failed" if errors else ("warnings" if warnings else "passed")
return {
"status": status,
"checks": checks,
"errors": errors,
"warnings": warnings,
"koppen_from_coordinates": koppen_detail,
}
def output_directory(site_id: str, output_dir: Optional[str]) -> str:
if output_dir:
return output_dir
return os.path.join("result", site_id)
def candidate_cache_files(site_id: str, output_dir: Optional[str]) -> List[str]:
outdir = output_directory(site_id, output_dir)
return [
os.path.join(outdir, f"{site_id}_ecosim_site.json"),
os.path.join("result", site_id, f"{site_id}_ecosim_site.json"),
os.path.join("result", site_id, "input", f"{site_id}_ecosim_site.json"),
os.path.join("result", f"{site_id}_ecosim_site.json"),
]
def load_cached_metadata(site_id: str, output_dir: Optional[str]) -> Optional[Dict[str, Any]]:
for path in dict.fromkeys(candidate_cache_files(site_id, output_dir)):
if not os.path.exists(path):
continue
with open(path, "r") as handle:
data = json.load(handle)
if all(key in data for key in ("ALATG", "ALONG", "ALTIG", "ATCAG", "IETYPG", "IXTYP1")):
data.setdefault("site_name", site_id)
return data
return None
def urllib_page_text(site_id: str) -> Iterable[Tuple[str, str]]:
url = AMERIFLUX_SITEINFO_URL.format(site_id=site_id)
request = Request(
url,
headers={"User-Agent": "EcoSIM-python-tools/1.0 (+https://github.com/bioepic-data/ecosim-agent)"},
)
with urlopen(request, timeout=60) as response:
charset = response.headers.get_content_charset() or "utf-8"
text = response.read().decode(charset, errors="replace")
yield f"AmeriFlux siteinfo/{site_id} text", html_to_text(text)
yield f"AmeriFlux siteinfo/{site_id} html", text
def requests_page_text(site_id: str) -> Iterable[Tuple[str, str]]:
if requests is None:
return
url = AMERIFLUX_SITEINFO_URL.format(site_id=site_id)
response = requests.get(
url,
headers={"User-Agent": "EcoSIM-python-tools/1.0 (+https://github.com/bioepic-data/ecosim-agent)"},
timeout=60,
)
response.raise_for_status()
yield f"AmeriFlux siteinfo/{site_id} text", html_to_text(response.text)
yield f"AmeriFlux siteinfo/{site_id} html", response.text
def playwright_page_text(site_id: str) -> Iterable[Tuple[str, str]]:
if sync_playwright is None:
return
url = AMERIFLUX_SITEINFO_URL.format(site_id=site_id)
with sync_playwright() as playwright:
browser = playwright.chromium.launch(headless=True)
try:
page = browser.new_page()
page.set_viewport_size({"width": 1280, "height": 1600})
page.goto(url, wait_until="networkidle", timeout=90000)
yield f"AmeriFlux siteinfo/{site_id} rendered_text", page.locator("body").inner_text(timeout=30000)
yield f"AmeriFlux siteinfo/{site_id} rendered_html", page.content()
finally:
browser.close()
def fetch_and_parse(site_id: str) -> Dict[str, Any]:
errors: List[str] = []
sources_seen = 0
for provider in (urllib_page_text, requests_page_text, playwright_page_text):
try:
for source, text in provider(site_id) or ():
sources_seen += 1
raw = parse_site_metadata_from_text(text, site_id, source)
if raw:
return raw
except Exception as exc:
errors.append(f"{provider.__name__}: {exc}")
if sources_seen:
detail = f"fetched {sources_seen} page text sources but required fields were incomplete"
if errors:
detail = f"{detail}; " + "; ".join(errors)
else:
detail = "; ".join(errors) if errors else "no page text providers were available"
raise RuntimeError(f"Could not derive complete AmeriFlux metadata for {site_id}: {detail}")
def extract_site_info(
site_id: str,
output_dir: Optional[str] = None,
force_refresh: bool = False,
check_location: bool = True,
koppen_raster: Optional[str] = None,
) -> Dict[str, Any]:
koppen_raster = find_koppen_raster(koppen_raster)
if not force_refresh:
cached = load_cached_metadata(site_id, output_dir)
if cached:
validation = validate_site_location(site_id, cached, koppen_raster)
if not check_location or validation["status"] != "failed":
cached["_location_validation"] = validation
for message in validation["warnings"]:
print(f"WARNING: {site_id}: {message}", file=sys.stderr)
return cached
print(
f"Cached metadata for {site_id} failed location checks "
f"({'; '.join(validation['errors'])}); refetching.",
file=sys.stderr,
)
raw = fetch_and_parse(site_id)
final_json = raw_to_ecosim(site_id, raw)
validation = validate_site_location(site_id, final_json, koppen_raster)
final_json["_location_validation"] = validation
for message in validation["warnings"]:
print(f"WARNING: {site_id}: {message}", file=sys.stderr)
if check_location and validation["status"] == "failed":
raise ValueError(
f"Location checks failed for {site_id}: {'; '.join(validation['errors'])} "
"Use --skip-location-check only after confirming the coordinates."
)
outdir = output_directory(site_id, output_dir)
os.makedirs(outdir, exist_ok=True)
output_file = os.path.join(outdir, f"{site_id}_ecosim_site.json")
with open(output_file, "w") as handle:
json.dump(final_json, handle, indent=4)
return final_json
def run_site_metadata_flow(
site_id: str,
output_dir: Optional[str] = None,
force_refresh: bool = False,
check_location: bool = True,
koppen_raster: Optional[str] = None,
) -> Dict[str, Any]:
final_json = extract_site_info(
site_id,
output_dir=output_dir,
force_refresh=force_refresh,
check_location=check_location,
koppen_raster=koppen_raster,
)
outdir = output_directory(site_id, output_dir)
output_file = os.path.join(outdir, f"{site_id}_ecosim_site.json")
print(f"Successfully resolved EcoSIM site metadata for {site_id}")
print(f"Output file: {output_file}")
print(json.dumps(final_json, indent=4))
return final_json
def run_vision_rag_flow(site_id: str, output_dir: Optional[str] = None) -> Dict[str, Any]:
"""Backward-compatible wrapper for callers that used the old function name."""
return run_site_metadata_flow(site_id, output_dir=output_dir)
def main() -> int:
parser = argparse.ArgumentParser(description="Extract AmeriFlux site metadata for EcoSIM inputs.")
parser.add_argument("site_id", help="AmeriFlux site identifier, for example US-Ha1.")
parser.add_argument("output_dir", nargs="?", help="Optional output directory. Defaults to result/<SITE_ID>.")
parser.add_argument("--force-refresh", action="store_true", help="Ignore cached metadata and refetch AmeriFlux.")
parser.add_argument(
"--skip-location-check",
action="store_true",
help="Write metadata even if coordinates fail the page-sign, country, or Koppen-sign checks.",
)
parser.add_argument(
"--koppen-raster",
help=f"Gridded Koppen-Geiger NetCDF (Beck et al. 2018 ids/labels) for the coordinate check; "
f"defaults to ${KOPPEN_RASTER_ENV} or data/Koppen.nc.",
)
args = parser.parse_args()
try:
run_site_metadata_flow(
args.site_id,
output_dir=args.output_dir,
force_refresh=args.force_refresh,
check_location=not args.skip_location_check,
koppen_raster=args.koppen_raster,
)
except Exception as exc:
print(str(exc), file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| receiving-code-reviewobra/superpowersUse when receiving code review feedback, before implementing suggestions, especially if feedback seems unclear or technically questionable - requires technical rigor and verification, not performative agreement or blind implementation | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| requesting-code-reviewobra/superpowersUse when completing tasks, implementing major features, or before merging to verify work meets requirements | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| using-git-worktreesobra/superpowersUse when starting feature work that needs isolation from current workspace or before executing implementation plans - ensures an isolated workspace exists via native tools or git worktree fallback | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| verification-before-completionobra/superpowersUse when about to claim work is complete, fixed, or passing, before committing or creating PRs - requires running verification commands and confirming output before making any success claims; evidence before assertions always | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |