bioepic-data/ecosim-agent / .agents/skills/ameriflux-era5-to-ecosim
ameriflux-era5-to-ecosim
VerifiedConvert AmeriFlux ERA5 half-hourly meteorological CSV files into EcoSIM hourly climate forcing NetCDF files, audit annual and seasonal precipitation budgets against sourced site climatology, compare generated forcing against overlapping measured tower meteorology, and report biases or unverified targets. Use when preparing or validating EcoSIM climate inputs for AmeriFlux sites, checking ERA5 against in situ records, diagnosing forcing discrepancies, or writing climate-quality reports. For non-AmeriFlux paper sites that need a fresh Copernicus CDS ERA5 point download by longitude/latitude, use era5-cds-point-download first.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 15.6 KB |
| era5_to_ecosim_converter.py | script | 46.4 KB |
| test_converter.py | script | 16.1 KB |
era5_to_ecosim_converter.py
View on GitHub#!/usr/bin/env python3
"""
Convert Ameriflux ERA5 half-hourly climate forcing data to ECOSIM hourly format.
This script reads half-hourly ERA5 climate data from Ameriflux format and
converts it to the ECOSIM hourly climate forcing format as described in
the Blodget.clim.2012-2022.template file.
"""
import pandas as pd
import numpy as np
from netCDF4 import Dataset
import glob
import os
import sys
import json
import subprocess
import argparse
import math
from datetime import datetime
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from Tools.climate_naming import finalize_climate_file_name
from Tools.climate_quality import sanitize_era5_dataframe
from Tools.precipitation_quality import precipitation_budget, rainfall_hash
FILL_VALUE_THRESHOLD = 1.0e29
DEFAULT_COMPARISON_MIN_PAIRS = 720
# Warn when ZNOONG and the radiation-weighted diurnal centroid disagree by more
# than this many hours; EcoSIM caps hourly SRADH by the sun angle from ZNOONG.
SOLAR_NOON_MISMATCH_WARNING_HOURS = 0.75
# Warn when the forcing and tower SW_IN diurnal centroids differ by more than this.
CLOCK_LAG_WARNING_HOURS = 0.5
# ERA5-to-tower time registration: lags searched (hours), minimum measured
# pairs, and the correlation gain a non-zero lag needs over zero lag.
TIME_REGISTRATION_MAX_LAG_HOURS = 3.0
TIME_REGISTRATION_MIN_PAIRS = 2000
TIME_REGISTRATION_MIN_CORRELATION_GAIN = 0.01
TIME_REGISTRATION_MODES = ("auto", "check", "off")
def saturation_vapor_pressure_kpa(temp_c):
"""Saturation vapor pressure (kPa), matching EcoSIM MiniMathMod vapsat0."""
temp_k = np.asarray(temp_c, dtype=float) + 273.15
return 0.61 * np.exp(5360.0 * (3.661e-3 - 1.0 / temp_k))
def vapor_pressure_from_vpd_kpa(temp_c, vpd_hpa):
"""Actual vapor pressure (kPa) from air temperature (degC) and VPD (hPa).
EcoSIM reads DWPTH as atmospheric vapor pressure (WthrMod: VPK=min(DWPTH,VPS)),
so AmeriFlux VPD must be converted rather than written directly.
"""
es = saturation_vapor_pressure_kpa(temp_c)
return np.clip(es - np.asarray(vpd_hpa, dtype=float) * 0.1, 0.0, es)
# These are broad operational screens for detecting forcing failures, not
# universal performance targets for ERA5 at flux-tower scale.
IN_SITU_COMPARISON_SPECS = {
"TMPH": {
"candidates": (("TA_F", "TA_F_QC"), ("TA", None)),
"aggregation": "mean",
"scale": 1.0,
"units": "degC",
"valid_range": (-90.0, 60.0),
"max_abs_mean_bias": 2.0,
"max_rmse": 5.0,
},
"WINDH": {
"candidates": (("WS_F", "WS_F_QC"), ("WS", None)),
"aggregation": "mean",
"scale": 1.0,
"units": "m s-1",
"valid_range": (0.0, 75.0),
"max_abs_mean_bias": 1.5,
"max_rmse": 3.0,
"max_abs_relative_mean_bias": 0.50,
"relative_bias_min_abs_bias": 0.5,
},
"RAINH": {
"candidates": (("P_F", "P_F_QC"), ("P", None)),
"aggregation": "sum",
"scale": 1.0,
"units": "mm h-1",
"valid_range": (0.0, 500.0),
"max_abs_relative_total_bias": 0.30,
"relative_total_min_observed": 10.0,
},
"DWPTH": {
# Observed VPD (hPa) is converted to vapor pressure with measured TA.
"candidates": (("VPD_F", "VPD_F_QC"), ("VPD", None)),
"temperature_candidates": (("TA_F", "TA_F_QC"), ("TA", None)),
"aggregation": "mean",
"scale": 1.0,
"units": "kPa",
"valid_range": (0.0, 100.0),
"max_abs_mean_bias": 0.30,
"max_rmse": 0.75,
"max_abs_relative_mean_bias": 0.40,
"relative_bias_min_abs_bias": 0.15,
},
"SRADH": {
"candidates": (("SW_IN_F", "SW_IN_F_QC"), ("SW_IN", None)),
"aggregation": "mean",
"scale": 1.0,
"units": "W m-2",
"valid_range": (0.0, 1400.0),
"max_abs_mean_bias": 40.0,
"max_rmse": 150.0,
"max_abs_relative_mean_bias": 0.30,
"relative_bias_min_abs_bias": 20.0,
},
"PATM": {
"candidates": (("PA_F", "PA_F_QC"), ("PA", None)),
"aggregation": "mean",
"scale": 1.0,
"units": "kPa",
"valid_range": (45.0, 110.0),
"max_abs_mean_bias": 2.0,
"max_rmse": 4.0,
},
}
def get_site_metadata(site_id):
"""Get site metadata using ameriflux_site_info skill."""
result_root = "result"
result_dir = os.path.join(result_root, site_id)
os.makedirs(result_dir, exist_ok=True)
candidate_files = [
os.path.join(result_dir, f"{site_id}_ecosim_site.json"),
os.path.join(result_root, f"{site_id}_ecosim_site.json"),
]
for site_file in candidate_files:
if os.path.exists(site_file):
with open(site_file, 'r') as f:
return json.load(f)
skill_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
script_candidates = [
os.path.join(skill_root, "ameriflux-site-info", "extract_ameriflux_site_data.py"),
os.path.join(skill_root, "ameriflux_site_info", "extract_ameriflux_site_data.py"),
]
script_path = next((path for path in script_candidates if os.path.exists(path)), script_candidates[0])
cmd = [sys.executable, script_path, site_id, result_dir]
result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.getcwd())
if result.returncode != 0:
print(f"Error running site info: {result.stderr}")
return None
for site_file in candidate_files:
if os.path.exists(site_file):
with open(site_file, 'r') as f:
return json.load(f)
return None
def get_site_longitude(site_id):
"""Get longitude for the site using ameriflux_site_info skill."""
site_data = get_site_metadata(site_id)
if not site_data:
return None
return site_data.get('ALONG')
def parse_timestamps(timestamp_str):
"""Parse timestamp string from Ameriflux data."""
# Format: YYYYMMDDHHMM (e.g., 198101010000)
# Convert to string first to ensure it's a string
timestamp_str = str(timestamp_str)
year = int(timestamp_str[:4])
month = int(timestamp_str[4:6])
day = int(timestamp_str[6:8])
hour = int(timestamp_str[8:10])
minute = int(timestamp_str[10:12])
return datetime(year, month, day, hour, minute)
def infer_source_frequency(timestamps):
"""Infer the AmeriFlux source timestep for QC reindexing."""
deltas = timestamps.sort_values().diff().dropna().dt.total_seconds()
if deltas.empty:
return "30min"
median_seconds = float(deltas.median())
if 3300 <= median_seconds <= 3900:
return "1h"
if 1500 <= median_seconds <= 2100:
return "30min"
inferred = pd.infer_freq(timestamps.sort_values())
if inferred:
return inferred
return "30min"
def discover_in_situ_file(era5_file, site_id=None):
"""Find a matching AmeriFlux FULLSET HH/HR file beside an ERA5 CSV."""
source_dir = os.path.dirname(os.path.abspath(era5_file))
source_name = os.path.basename(era5_file)
resolved_site_id = site_id
if not resolved_site_id and "_FLUXNET_ERA5_" in source_name:
prefix = source_name.split("_FLUXNET_ERA5_", 1)[0]
if prefix.startswith("AMF_"):
resolved_site_id = prefix[4:]
cadence_order = []
for cadence in ("HH", "HR"):
if f"_ERA5_{cadence}_" in source_name:
cadence_order.append(cadence)
cadence_order.extend(cadence for cadence in ("HH", "HR") if cadence not in cadence_order)
patterns = []
for cadence in cadence_order:
if resolved_site_id:
patterns.append(f"AMF_{resolved_site_id}_FLUXNET_FULLSET_{cadence}_*.csv")
else:
patterns.append(f"*_FLUXNET_FULLSET_{cadence}_*.csv")
for pattern in patterns:
candidates = sorted(glob.glob(os.path.join(source_dir, pattern)))
if candidates:
return os.path.abspath(candidates[-1])
return None
def load_tower_shortwave(in_situ_file):
"""Measured-only tower SW_IN (W m-2) indexed by TIMESTAMP_START, or None."""
header = set(pd.read_csv(in_situ_file, nrows=0).columns)
value_column, qc_column = _choose_observation_column(
header, IN_SITU_COMPARISON_SPECS["SRADH"]
)
if "TIMESTAMP_START" not in header or value_column is None:
return None, None
columns = ["TIMESTAMP_START", value_column] + ([qc_column] if qc_column else [])
tower = pd.read_csv(in_situ_file, usecols=columns, dtype={"TIMESTAMP_START": str})
timestamps = pd.to_datetime(tower["TIMESTAMP_START"], format="%Y%m%d%H%M", errors="coerce")
values = pd.to_numeric(tower[value_column], errors="coerce")
valid = timestamps.notna() & np.isfinite(values) & values.between(0.0, 1400.0)
if qc_column:
valid &= pd.to_numeric(tower[qc_column], errors="coerce").eq(0)
series = pd.Series(values[valid].to_numpy(), index=timestamps[valid]).sort_index()
return series[~series.index.duplicated()], value_column
def estimate_time_registration(era5_shortwave, tower_shortwave, frequency):
"""Lag (hours) of the ERA5 clock relative to the tower clock.
A positive lag means an ERA5 value stamped t + lag belongs to tower time t,
i.e. the ERA5 file is late. Lags are searched in source-timestep steps by
maximising the Pearson correlation of shortwave radiation, and checked
against the difference of the radiation-weighted diurnal centroids.
"""
step = pd.Timedelta(frequency)
step_hours = step.total_seconds() / 3600.0
max_steps = int(round(TIME_REGISTRATION_MAX_LAG_HOURS / step_hours))
era5_shortwave = era5_shortwave[~era5_shortwave.index.duplicated()]
correlations = {}
pairs = {}
for k in range(-max_steps, max_steps + 1):
shifted = pd.Series(era5_shortwave.to_numpy(), index=era5_shortwave.index - k * step)
joined = pd.concat([shifted.rename("era5"), tower_shortwave.rename("tower")], axis=1, join="inner").dropna()
pairs[k] = int(len(joined))
if len(joined) >= 2 and joined["era5"].std() > 0 and joined["tower"].std() > 0:
correlations[k] = float(np.corrcoef(joined["era5"], joined["tower"])[0, 1])
result = {
"step_hours": step_hours,
"pairs_at_zero_lag": pairs.get(0, 0),
"correlation_by_lag_hours": {f"{k * step_hours:+g}": round(r, 4) for k, r in correlations.items()},
}
if 0 not in correlations or pairs[0] < TIME_REGISTRATION_MIN_PAIRS:
result["status"] = "insufficient_overlap"
return result
best = max(correlations, key=correlations.get)
result.update(
{
"status": "estimated",
"lag_hours": best * step_hours,
"correlation_at_zero_lag": round(correlations[0], 4),
"correlation_at_lag": round(correlations[best], 4),
}
)
overlap = tower_shortwave.index.intersection(era5_shortwave.index)
era5_centroid = radiation_centroid_hour(
pd.DataFrame({"hour": overlap.hour, "SRADH": era5_shortwave.reindex(overlap).to_numpy()})
)
tower_centroid = radiation_centroid_hour(
pd.DataFrame({"hour": overlap.hour, "SRADH": tower_shortwave.reindex(overlap).to_numpy()})
)
if era5_centroid is not None and tower_centroid is not None:
result["centroid_lag_hours"] = round(wrap_hours(era5_centroid - tower_centroid), 4)
return result
def register_era5_to_tower(df, frequency, in_situ_file=None, mode="auto", shift_hours=None):
"""Shift ERA5 timestamps onto the tower clock.
Returns the (possibly shifted) frame and a report. A manual ``shift_hours``
(ERA5 lag relative to the tower) is applied as given. Otherwise, in "auto"
mode a detected lag is applied only when it improves the correlation by
TIME_REGISTRATION_MIN_CORRELATION_GAIN and agrees with the centroid lag to
within half an hour; "check" only reports; "off" skips the check.
"""
if mode not in TIME_REGISTRATION_MODES:
raise ValueError(f"time registration mode must be one of {TIME_REGISTRATION_MODES}")
report = {"mode": mode, "in_situ_file": os.path.abspath(in_situ_file) if in_situ_file else None,
"applied_shift_hours": 0.0, "warning": None}
lag = None
if shift_hours is not None:
lag = float(shift_hours)
report["status"] = "manual"
elif mode == "off":
report["status"] = "skipped"
elif not in_situ_file:
report["status"] = "unverified"
report["reason"] = "No AmeriFlux FULLSET file to register ERA5 against."
else:
tower, column = load_tower_shortwave(in_situ_file)
if tower is None or tower.empty:
report["status"] = "unverified"
report["reason"] = "Tower file has no measured shortwave radiation."
else:
era5 = pd.Series(
pd.to_numeric(df["SW_IN_ERA"], errors="coerce").to_numpy(),
index=pd.DatetimeIndex(df["timestamp_start"]),
)
era5 = era5[np.isfinite(era5) & era5.between(0.0, 1400.0)]
estimate = estimate_time_registration(era5, tower, frequency)
estimate["tower_column"] = column
report.update(estimate)
if estimate["status"] == "estimated" and estimate["lag_hours"] != 0:
gain = estimate["correlation_at_lag"] - estimate["correlation_at_zero_lag"]
centroid_lag = estimate.get("centroid_lag_hours")
consistent = centroid_lag is not None and abs(centroid_lag - estimate["lag_hours"]) <= 0.5
if gain >= TIME_REGISTRATION_MIN_CORRELATION_GAIN and consistent:
report["status"] = "misregistered"
if mode == "auto":
lag = estimate["lag_hours"]
else:
report["warning"] = (
f"ERA5 clock is {estimate['lag_hours']:+g} h relative to the tower "
"(not corrected: time registration mode 'check')."
)
else:
report["status"] = "ambiguous"
report["warning"] = (
f"Best ERA5-tower lag {estimate['lag_hours']:+g} h was not applied: "
f"correlation gain {gain:.4f}, centroid lag {centroid_lag}."
)
elif estimate["status"] == "estimated":
report["status"] = "registered"
if lag:
start, end = df["timestamp_start"].min(), df["timestamp_start"].max()
df = df.copy()
offset = pd.Timedelta(hours=lag)
df["timestamp_start"] = df["timestamp_start"] - offset
if "timestamp_end" in df:
df["timestamp_end"] = df["timestamp_end"] - offset
report["applied_shift_hours"] = -lag
report["record_span"] = [start.isoformat(), end.isoformat()]
report["note"] = (
f"ERA5 timestamps shifted by {-lag:+g} h onto the tower clock; the record "
"span is kept and edge steps are filled from the nearest valid value."
)
if report["status"] == "misregistered":
report["status"] = "corrected"
return df, report
def read_ecosim_hourly_forcing(netcdf_file):
"""Read the generated EcoSIM NetCDF back into an hourly table."""
frames = []
with Dataset(netcdf_file, "r") as nc_file:
years = np.asarray(nc_file.variables["year"][:], dtype=int)
for year_index, year in enumerate(years):
timestamps = pd.date_range(
start=f"{int(year):04d}-01-01 00:00",
periods=366 * 24,
freq="h",
)
data = {"timestamp_start": timestamps}
for variable in IN_SITU_COMPARISON_SPECS:
raw = nc_file.variables[variable][year_index, :, :, 0]
values = np.asarray(np.ma.filled(raw, np.nan), dtype=float).reshape(-1)
values[np.abs(values) >= FILL_VALUE_THRESHOLD] = np.nan
data[variable] = values
frame = pd.DataFrame(data)
frame = frame.dropna(subset=tuple(IN_SITU_COMPARISON_SPECS), how="all")
frames.append(frame)
if not frames:
return pd.DataFrame(columns=("timestamp_start",) + tuple(IN_SITU_COMPARISON_SPECS))
return pd.concat(frames, ignore_index=True)
def _choose_observation_column(columns, spec):
for value_column, qc_column in spec["candidates"]:
if value_column in columns and (qc_column is None or qc_column in columns):
return value_column, qc_column
return None, None
def load_in_situ_hourly(in_situ_file):
"""Load measured-only AmeriFlux records and aggregate complete hours."""
header = set(pd.read_csv(in_situ_file, nrows=0).columns)
if "TIMESTAMP_START" not in header:
raise ValueError(f"In situ file lacks TIMESTAMP_START: {in_situ_file}")
selected = {"TIMESTAMP_START"}
choices = {}
temperature_choices = {}
for forcing_variable, spec in IN_SITU_COMPARISON_SPECS.items():
value_column, qc_column = _choose_observation_column(header, spec)
if value_column and "temperature_candidates" in spec:
temp_column, temp_qc_column = _choose_observation_column(
header, {"candidates": spec["temperature_candidates"]}
)
if temp_column is None:
value_column, qc_column = None, None
else:
temperature_choices[forcing_variable] = (temp_column, temp_qc_column)
selected.add(temp_column)
if temp_qc_column:
selected.add(temp_qc_column)
choices[forcing_variable] = (value_column, qc_column)
if value_column:
selected.add(value_column)
if qc_column:
selected.add(qc_column)
observations = pd.read_csv(
in_situ_file,
usecols=sorted(selected),
dtype={"TIMESTAMP_START": str},
)
observations["timestamp_start"] = pd.to_datetime(
observations["TIMESTAMP_START"],
format="%Y%m%d%H%M",
errors="coerce",
)
observations = observations.dropna(subset=["timestamp_start"])
observations = observations.sort_values("timestamp_start")
observations = observations.drop_duplicates("timestamp_start", keep="first")
deltas = observations["timestamp_start"].diff().dropna().dt.total_seconds()
median_seconds = float(deltas.median()) if not deltas.empty else 3600.0
expected_records_per_hour = max(1, int(round(3600.0 / median_seconds)))
observations["hour"] = observations["timestamp_start"].dt.floor("h")
hourly = {}
for forcing_variable, spec in IN_SITU_COMPARISON_SPECS.items():
value_column, qc_column = choices[forcing_variable]
if not value_column:
hourly[forcing_variable] = {
"status": "variable_not_available",
"observed_column": None,
"qc_column": None,
}
continue
values = pd.to_numeric(observations[value_column], errors="coerce")
lower, upper = spec["valid_range"]
valid = np.isfinite(values) & values.between(lower, upper, inclusive="both")
if qc_column:
qc = pd.to_numeric(observations[qc_column], errors="coerce")
valid &= qc.eq(0)
if forcing_variable in temperature_choices:
temp_column, temp_qc_column = temperature_choices[forcing_variable]
temperature = pd.to_numeric(observations[temp_column], errors="coerce")
valid &= np.isfinite(temperature) & temperature.between(-90.0, 60.0)
if temp_qc_column:
temp_qc = pd.to_numeric(observations[temp_qc_column], errors="coerce")
valid &= temp_qc.eq(0)
values = pd.Series(
vapor_pressure_from_vpd_kpa(temperature, values),
index=values.index,
)
measured = pd.DataFrame(
{
"timestamp_start": observations.loc[valid, "hour"],
"value": values.loc[valid] * spec["scale"],
}
)
grouped = measured.groupby("timestamp_start")["value"]
if spec["aggregation"] == "sum":
values_hourly = grouped.sum(min_count=1)
else:
values_hourly = grouped.mean()
counts = grouped.count()
values_hourly = values_hourly[counts >= expected_records_per_hour]
hourly[forcing_variable] = {
"status": "available",
"observed_column": value_column,
"qc_column": qc_column,
"expected_records_per_hour": expected_records_per_hour,
"data": values_hourly.rename("observed").reset_index(),
}
return hourly
def _finite_float(value):
value = float(value)
return value if math.isfinite(value) else None
def _large_difference_reasons(metrics, spec):
reasons = []
abs_bias = abs(metrics["mean_bias"])
rmse = metrics["root_mean_square_error"]
threshold = spec.get("max_abs_mean_bias")
if threshold is not None and abs_bias > threshold:
reasons.append(f"absolute mean bias {abs_bias:.3g} > {threshold:g} {spec['units']}")
threshold = spec.get("max_rmse")
if threshold is not None and rmse > threshold:
reasons.append(f"RMSE {rmse:.3g} > {threshold:g} {spec['units']}")
threshold = spec.get("max_abs_relative_mean_bias")
minimum_bias = spec.get("relative_bias_min_abs_bias", 0.0)
relative_bias = metrics.get("relative_mean_bias_fraction")
if (
threshold is not None
and relative_bias is not None
and abs(relative_bias) > threshold
and abs_bias >= minimum_bias
):
reasons.append(f"relative mean bias {100.0 * relative_bias:.1f}% exceeds +/-{100.0 * threshold:g}%")
threshold = spec.get("max_abs_relative_total_bias")
minimum_total = spec.get("relative_total_min_observed", 0.0)
relative_total_bias = metrics.get("relative_total_bias_fraction")
if (
threshold is not None
and relative_total_bias is not None
and metrics.get("observed_total", 0.0) >= minimum_total
and abs(relative_total_bias) > threshold
):
reasons.append(f"relative total bias {100.0 * relative_total_bias:.1f}% exceeds +/-{100.0 * threshold:g}%")
return reasons
def compare_forcing_with_in_situ(netcdf_file, in_situ_file, min_pairs=DEFAULT_COMPARISON_MIN_PAIRS):
"""Compare generated forcing against measured-only AmeriFlux observations."""
min_pairs = max(1, int(min_pairs))
report = {
"status": "not_available",
"in_situ_file": os.path.abspath(in_situ_file) if in_situ_file else None,
"minimum_paired_hours_for_warning": min_pairs,
"screening_note": (
"Thresholds are broad operational screens for forcing failures, not universal "
"acceptance limits for ERA5 at tower scale."
),
"variables": {},
"warnings": [],
}
if not in_situ_file:
report["reason"] = "No matching AmeriFlux FULLSET HH/HR file was found."
return report
if not os.path.exists(in_situ_file):
raise FileNotFoundError(f"In situ comparison file not found: {in_situ_file}")
forcing = read_ecosim_hourly_forcing(netcdf_file)
observations = load_in_situ_hourly(in_situ_file)
compared_variables = 0
sufficient_variables = 0
for variable, spec in IN_SITU_COMPARISON_SPECS.items():
observation = observations[variable]
if observation["status"] != "available":
report["variables"][variable] = {
"status": observation["status"],
"units": spec["units"],
}
continue
paired = forcing[["timestamp_start", variable]].merge(
observation["data"],
on="timestamp_start",
how="inner",
)
paired = paired.dropna(subset=[variable, "observed"])
paired = paired.sort_values("timestamp_start")
paired_hours = int(len(paired))
compared_variables += int(paired_hours > 0)
variable_report = {
"status": "insufficient_data" if paired_hours < min_pairs else "compared",
"units": spec["units"],
"observed_column": observation["observed_column"],
"qc_column": observation["qc_column"],
"measured_qc_value": 0 if observation["qc_column"] else None,
"paired_hours": paired_hours,
}
if paired_hours == 0:
report["variables"][variable] = variable_report
continue
forcing_values = paired[variable].to_numpy(dtype=float)
observed_values = paired["observed"].to_numpy(dtype=float)
differences = forcing_values - observed_values
forcing_mean = float(np.mean(forcing_values))
observed_mean = float(np.mean(observed_values))
mean_bias = float(np.mean(differences))
metrics = {
"forcing_mean": forcing_mean,
"observed_mean": observed_mean,
"mean_bias": mean_bias,
"mean_absolute_error": float(np.mean(np.abs(differences))),
"root_mean_square_error": float(np.sqrt(np.mean(differences ** 2))),
"relative_mean_bias_fraction": (
mean_bias / abs(observed_mean) if abs(observed_mean) > 1.0e-12 else None
),
"forcing_total": float(np.sum(forcing_values)),
"observed_total": float(np.sum(observed_values)),
"first_paired_timestamp": paired["timestamp_start"].iloc[0].isoformat(),
"last_paired_timestamp": paired["timestamp_start"].iloc[-1].isoformat(),
}
metrics["relative_total_bias_fraction"] = (
(metrics["forcing_total"] - metrics["observed_total"])
/ abs(metrics["observed_total"])
if abs(metrics["observed_total"]) > 1.0e-12
else None
)
if np.std(forcing_values) > 0 and np.std(observed_values) > 0:
metrics["pearson_correlation"] = _finite_float(
np.corrcoef(forcing_values, observed_values)[0, 1]
)
else:
metrics["pearson_correlation"] = None
if variable == "SRADH":
# A whole-hour clock shift leaves mean bias near zero, so compare the
# diurnal radiation centroids over the same paired hours.
hours = paired["timestamp_start"].dt.hour
forcing_centroid = radiation_centroid_hour(
pd.DataFrame({"hour": hours, "SRADH": forcing_values})
)
observed_centroid = radiation_centroid_hour(
pd.DataFrame({"hour": hours, "SRADH": observed_values})
)
if forcing_centroid is not None and observed_centroid is not None:
metrics["diurnal_centroid_lag_hours"] = wrap_hours(
forcing_centroid - observed_centroid
)
variable_report.update(metrics)
if paired_hours >= min_pairs:
sufficient_variables += 1
reasons = _large_difference_reasons(metrics, spec)
lag = metrics.get("diurnal_centroid_lag_hours")
if lag is not None and abs(lag) > CLOCK_LAG_WARNING_HOURS:
reasons.append(
f"diurnal radiation centroid lags the tower by {lag:+.2f} h "
"(forcing clock likely shifted)"
)
if reasons:
warning = {
"variable": variable,
"message": (
f"{variable} forcing differs substantially from measured "
f"{observation['observed_column']}: " + "; ".join(reasons)
),
}
variable_report["warning"] = warning["message"]
report["warnings"].append(warning)
report["variables"][variable] = variable_report
report['unverified_variables'] = [name for name, item in report['variables'].items() if item['status'] != 'compared']
if sufficient_variables:
report["status"] = "partially_compared" if report['unverified_variables'] else "compared"
elif compared_variables:
report["status"] = "insufficient_data"
report["reason"] = "Overlapping measured records did not meet the paired-hour minimum."
else:
report["reason"] = "No overlapping measured records were available."
return report
def convert_era5_to_ecosim(
era5_file,
output_file,
longitude,
elevation=None,
quality_report_file=None,
site_id=None,
in_situ_file=None,
compare_in_situ=True,
comparison_min_pairs=DEFAULT_COMPARISON_MIN_PAIRS,
utc_offset_hours=None,
time_registration="auto",
time_shift_hours=None,
mean_annual_precip_mm=None,
precip_reference=None,
):
"""
Convert ERA5 half-hourly data to ECOSIM hourly format.
Parameters:
era5_file (str): Path to the Ameriflux ERA5 CSV file
output_file (str): Path to output netCDF file
longitude (float): Longitude for solar noon calculation
in_situ_file (str): Optional AmeriFlux FULLSET HH/HR file for validation
utc_offset_hours (float): UTC offset of the CSV timestamps; inferred when None
time_registration (str): "auto" shifts ERA5 onto the tower clock when a lag
is detected, "check" only reports it, "off" skips the check
time_shift_hours (float): Known ERA5 lag relative to the tower (hours);
applied directly, overriding detection
"""
if mean_annual_precip_mm is not None and (not math.isfinite(float(mean_annual_precip_mm)) or float(mean_annual_precip_mm) <= 0 or not str(precip_reference or '').strip()):
raise ValueError('A positive site MAP and precipitation reference citation/path are required together')
# Read the CSV data with dtype specification to avoid automatic conversion
df = pd.read_csv(era5_file, dtype={'TIMESTAMP_START': str, 'TIMESTAMP_END': str})
# Parse timestamps
df['timestamp_start'] = df['TIMESTAMP_START'].apply(parse_timestamps)
df['timestamp_end'] = df['TIMESTAMP_END'].apply(parse_timestamps)
source_frequency = infer_source_frequency(df['timestamp_start'])
record_start = df['timestamp_start'].min()
record_end = df['timestamp_start'].max()
# Coregister ERA5 with the tower clock before QC and hourly aggregation.
registration_in_situ = in_situ_file
if registration_in_situ is None and time_registration != "off":
registration_in_situ = discover_in_situ_file(era5_file, site_id)
df, registration = register_era5_to_tower(
df,
source_frequency,
in_situ_file=registration_in_situ,
mode=time_registration,
shift_hours=time_shift_hours,
)
if registration["applied_shift_hours"]:
print(f"Time registration: {registration['note']}")
if registration["warning"]:
print(f"WARNING: {registration['warning']}", file=sys.stderr)
df, quality_report = sanitize_era5_dataframe(
df,
elevation_m=elevation,
frequency=source_frequency,
index_start=record_start,
index_end=record_end,
)
quality_report["time_registration"] = registration
# Convert to hourly data by averaging consecutive half-hourly values.
df['hour'] = df['timestamp_start'].dt.hour
df['day_of_year'] = df['timestamp_start'].dt.dayofyear
df['year'] = df['timestamp_start'].dt.year
# Convert VPD to vapor pressure before averaging so each half-hour uses its
# own saturation vapor pressure.
df['EA_KPA'] = vapor_pressure_from_vpd_kpa(df['TA_ERA'], df['VPD_ERA'])
hourly_df = (
df.groupby(['year', 'day_of_year', 'hour'], as_index=False)
.agg(
TMPH=('TA_ERA', 'mean'),
WINDH=('WS_ERA', 'mean'),
RAINH=('P_ERA', 'sum'),
DWPTH=('EA_KPA', 'mean'),
SRADH=('SW_IN_ERA', 'mean'),
PATM=('PA_ERA', 'mean'),
)
.rename(columns={'day_of_year': 'day'})
)
# Create the netCDF file
solar_noon = create_ecosim_climate_file(
hourly_df, output_file, longitude, utc_offset_hours=utc_offset_hours
)
quality_report["solar_noon"] = solar_noon
if solar_noon["warning"]:
print(f"WARNING: {solar_noon['warning']}", file=sys.stderr)
if compare_in_situ:
selected_in_situ_file = in_situ_file or discover_in_situ_file(era5_file, site_id)
comparison = compare_forcing_with_in_situ(
output_file,
selected_in_situ_file,
min_pairs=comparison_min_pairs,
)
quality_report["in_situ_comparison"] = comparison
if comparison["status"] == "not_available":
print(f"In situ comparison skipped: {comparison.get('reason', 'observations unavailable')}")
elif comparison["status"] == "insufficient_data":
print(f"In situ comparison incomplete: {comparison.get('reason', 'insufficient overlap')}")
else:
print(
f"Tower validation {comparison['status']}: {comparison['in_situ_file']}; "
f"{len(comparison['warnings'])} large-difference warning(s)."
)
for warning in comparison["warnings"]:
print(f"WARNING: {warning['message']}", file=sys.stderr)
else:
quality_report["in_situ_comparison"] = {
"status": "skipped",
"reason": "Disabled by caller.",
"variables": {},
"warnings": [],
}
# Persist scientific validation metadata for later namelist-driven preflight.
rain_status = quality_report['in_situ_comparison'].get('variables', {}).get('RAINH', {}).get('status', 'unverified')
with Dataset(output_file, 'a') as dataset:
dataset.precip_tower_validation_status = rain_status
dataset.precip_tower_validation_rain_sha256 = rainfall_hash(dataset)
if mean_annual_precip_mm is not None:
dataset.precip_climatology_mm = float(mean_annual_precip_mm)
dataset.precip_climatology_source = str(precip_reference)
quality_report['precipitation_budget'] = precipitation_budget(output_file)
for finding in quality_report['precipitation_budget']['findings']:
print('WARNING: '+finding['message'], file=sys.stderr)
unverified = quality_report['in_situ_comparison'].get('unverified_variables', [])
if unverified:
print('WARNING: Tower comparison incomplete for: '+', '.join(unverified), file=sys.stderr)
quality_report['validation_status'] = 'review_required' if quality_report['precipitation_budget']['findings'] or unverified or quality_report['in_situ_comparison'].get('warnings') else 'screened'
# Name by forcing years (e.g. US-UMB_ecosim_climate_1981-2021.nc). Intermediate files that do not end in
# .nc (such as the climate tool's *.nc.temp) are left for the caller to name.
if str(output_file).endswith(".nc"):
output_file = str(finalize_climate_file_name(output_file))
print(f"Climate forcing written to {output_file}")
quality_report["output_file"] = os.path.abspath(output_file)
# Record the output in a YAML file
try:
import yaml
if site_id:
result_dir = f"result/{site_id}"
os.makedirs(result_dir, exist_ok=True)
yaml_path = os.path.join(result_dir, f"{site_id}_forcing.yaml")
forcing_data = {"clm_hour_file_in": os.path.abspath(output_file)}
# If YAML already exists, preserve other keys (like grid_file_in)
if os.path.exists(yaml_path):
with open(yaml_path, 'r') as f:
existing_data = yaml.safe_load(f) or {}
forcing_data.update(existing_data)
forcing_data["clm_hour_file_in"] = os.path.abspath(output_file)
with open(yaml_path, 'w') as f:
yaml.dump(forcing_data, f, default_flow_style=False)
print(f"Climate forcing path recorded in {yaml_path}")
except ImportError:
print("PyYAML not installed; could not record forcing path to YAML.", file=sys.stderr)
except Exception as e:
print(f"Error recording forcing path to YAML: {e}", file=sys.stderr)
if quality_report_file:
quality_report_dir = os.path.dirname(quality_report_file)
if quality_report_dir:
os.makedirs(quality_report_dir, exist_ok=True)
with open(quality_report_file, "w") as f:
json.dump(quality_report, f, indent=2)
return quality_report
def radiation_centroid_hour(df):
"""Radiation-weighted mean hour of the diurnal SRADH cycle on the forcing clock.
Hour index h holds the average over [h, h+1), so its midpoint is h + 0.5.
A circular mean keeps the result valid when the daylight period wraps midnight.
"""
profile = df.groupby('hour')['SRADH'].mean().clip(lower=0.0)
if profile.sum() <= 0.0:
return None
angles = 2.0 * np.pi * (profile.index.to_numpy(dtype=float) + 0.5) / 24.0
weights = profile.to_numpy(dtype=float)
angle = math.atan2((weights * np.sin(angles)).sum(), (weights * np.cos(angles)).sum())
return (angle * 24.0 / (2.0 * np.pi)) % 24.0
def wrap_hours(value):
"""Wrap an hour difference into [-12, 12)."""
return (value + 12.0) % 24.0 - 12.0
def infer_utc_offset_hours(df, longitude):
"""Infer the forcing clock's whole-hour UTC offset from the diurnal SRADH cycle.
AmeriFlux timestamps are local standard time, but the offset is not stored in
the CSV, and rounding longitude/15 fails for sites west of their zone meridian
(e.g. US-UMB at -84.7 deg uses UTC-5, not UTC-6).
"""
centroid = radiation_centroid_hour(df)
if centroid is None:
raise ValueError("Cannot infer UTC offset: SRADH has no positive values.")
return int(round(wrap_hours(centroid - (12.0 - longitude / 15.0))))
def mean_solar_noon_hour(longitude, utc_offset_hours):
"""Mean solar noon (hours) on the forcing clock.
EcoSIM DayMod adds the daily equation of time to ZNOONG, so it is not
included here.
"""
return (12.0 - longitude / 15.0 + utc_offset_hours) % 24.0
def create_ecosim_climate_file(df, output_file, longitude, utc_offset_hours=None):
"""
Create ECOSIM climate forcing netCDF file from hourly data.
Parameters:
df (DataFrame): Hourly climate data on the AmeriFlux (local standard time) clock
output_file (str): Path to output netCDF file
longitude (float): Longitude of the site for solar noon calculation
utc_offset_hours (float): UTC offset of the forcing clock; inferred from the
diurnal SRADH cycle when None
Returns:
dict: Solar-noon derivation and consistency check for the quality report
"""
centroid = radiation_centroid_hour(df)
if utc_offset_hours is None:
utc_offset_hours = infer_utc_offset_hours(df, longitude)
offset_source = "inferred_from_SRADH_diurnal_cycle"
else:
offset_source = "user_supplied"
znoong = mean_solar_noon_hour(longitude, utc_offset_hours)
mismatch = None if centroid is None else wrap_hours(centroid - znoong)
solar_noon = {
"ZNOONG_hours": round(znoong, 4),
"utc_offset_hours": utc_offset_hours,
"utc_offset_source": offset_source,
"SRADH_centroid_hour": None if centroid is None else round(centroid, 4),
"centroid_minus_ZNOONG_hours": None if mismatch is None else round(mismatch, 4),
"warning": None,
}
if mismatch is not None and abs(mismatch) > SOLAR_NOON_MISMATCH_WARNING_HOURS:
solar_noon["warning"] = (
f"ZNOONG={znoong:.2f} h differs from the SRADH diurnal centroid "
f"{centroid:.2f} h by {mismatch:+.2f} h; EcoSIM will clip radiation "
"with a misplaced sun angle. Check the UTC offset."
)
# Create a new netCDF file
nc_file = Dataset(output_file, 'w', format='NETCDF4')
# Define dimensions
nyears = len(df['year'].unique())
ndays = 366
nhours = 24
ngrid = 1
nc_file.createDimension('year', nyears)
nc_file.createDimension('day', ndays)
nc_file.createDimension('hour', nhours)
nc_file.createDimension('ngrid', ngrid)
# Create variables
# Temperature (oC)
tmp_var = nc_file.createVariable('TMPH', 'f4', ('year', 'day', 'hour', 'ngrid'), fill_value=1e30)
tmp_var.long_name = "hourly air temperature"
tmp_var.units = "oC"
# Wind speed (m/s)
wind_var = nc_file.createVariable('WINDH', 'f4', ('year', 'day', 'hour', 'ngrid'), fill_value=1e30)
wind_var.long_name = "horizontal wind speed"
wind_var.units = "m s^-1"
# Precipitation (mm m^-2 hr^-1) - Need to convert from mm/h to mm m^-2 hr^-1
rain_var = nc_file.createVariable('RAINH', 'f4', ('year', 'day', 'hour', 'ngrid'), fill_value=1e30)
rain_var.long_name = "Total precipitation"
rain_var.units = "mm m^-2 hr^-1"
# Vapor pressure (kPa)
dwpt_var = nc_file.createVariable('DWPTH', 'f4', ('year', 'day', 'hour', 'ngrid'), fill_value=1e30)
dwpt_var.long_name = "atmospheric vapor pressure"
dwpt_var.units = "kPa"
# Solar radiation (W m^-2)
srad_var = nc_file.createVariable('SRADH', 'f4', ('year', 'day', 'hour', 'ngrid'), fill_value=1e30)
srad_var.long_name = "Incident solar radiation"
srad_var.units = "W m^-2"
# Atmospheric pressure (kPa)
patm_var = nc_file.createVariable('PATM', 'f4', ('year', 'day', 'hour', 'ngrid'), fill_value=1e30)
patm_var.long_name = "Surface atmospheric pressure"
patm_var.units = "kPa"
# Year variable
year_var = nc_file.createVariable('year', 'i4', ('year',))
year_var.long_name = "year AD"
# Other variables with fixed values for this site
z0g_var = nc_file.createVariable('Z0G', 'f4', ('year', 'ngrid'), fill_value=1e30)
z0g_var.long_name = "windspeed measurement height"
z0g_var.units = "m"
iflgw_var = nc_file.createVariable('IFLGW', 'i4', ('year', 'ngrid')) # No fill_value for integer variables
iflgw_var.long_name = "flag for raising Z0G with vegeation"
znoong_var = nc_file.createVariable('ZNOONG', 'f4', ('year', 'ngrid'), fill_value=1e30)
znoong_var.long_name = "time of solar noon"
znoong_var.units = "hour"
znoong_var.note = (
f"Mean solar noon on the forcing clock (UTC{utc_offset_hours:+g}, {offset_source}); "
"EcoSIM adds the daily equation of time."
)
# Create a simple mapping of years to indices
unique_years = sorted(df['year'].unique())
year_var[:] = unique_years
year_lookup = {year: idx for idx, year in enumerate(unique_years)}
year_idx = df['year'].map(year_lookup).to_numpy(dtype=np.int64)
day_idx = df['day'].to_numpy(dtype=np.int64) - 1
hour_idx = df['hour'].to_numpy(dtype=np.int64)
for variable, column in (
(tmp_var, 'TMPH'),
(wind_var, 'WINDH'),
(rain_var, 'RAINH'),
(dwpt_var, 'DWPTH'),
(srad_var, 'SRADH'),
(patm_var, 'PATM'),
):
data = np.full((nyears, ndays, nhours, ngrid), 1e30, dtype='f4')
data[year_idx, day_idx, hour_idx, 0] = df[column].to_numpy(dtype='f4')
variable[:] = data
# Set other fixed variables
for i, year in enumerate(unique_years):
z0g_var[i, 0] = 1.0 # Fixed value
iflgw_var[i, 0] = 0 # Fixed value
znoong_var[i, 0] = znoong
# Close the file
nc_file.close()
print(f"ECOSIM climate file created successfully: {output_file}")
return solar_noon
def main():
parser = argparse.ArgumentParser(description='Convert Ameriflux ERA5 half-hourly climate data to ECOSIM hourly format')
parser.add_argument('--input', '-i', required=True, help='Input CSV file path')
parser.add_argument('--output', '-o', required=True, help='Output netCDF file path')
parser.add_argument('--site-id', '-s', required=True, help='AmeriFlux site ID (e.g., US-Ha1) to get longitude from')
parser.add_argument(
'--quality-report',
help='Optional JSON report for physical checks, interpolation repairs, and in situ comparison',
)
parser.add_argument(
'--in-situ',
help='Optional AmeriFlux FULLSET HH/HR CSV; otherwise auto-discover beside the ERA5 input',
)
parser.add_argument(
'--comparison-min-pairs',
type=int,
default=DEFAULT_COMPARISON_MIN_PAIRS,
help='Minimum paired measured hours required before large-difference warnings (default: 720)',
)
parser.add_argument(
'--skip-in-situ-comparison',
action='store_true',
help='Disable comparison against an available AmeriFlux FULLSET file',
)
parser.add_argument(
'--utc-offset',
type=float,
default=None,
help='UTC offset (hours) of the CSV timestamps, e.g. -5 for EST; '
'inferred from the diurnal SW_IN cycle when omitted',
)
parser.add_argument(
'--time-registration',
choices=TIME_REGISTRATION_MODES,
default='auto',
help='Coregister ERA5 with the tower clock using SW_IN cross-correlation: '
'auto (detect and shift, default), check (report only), off',
)
parser.add_argument(
'--time-shift-hours',
type=float,
default=None,
help='Known ERA5 lag relative to the tower in hours (positive = ERA5 late); '
'applied directly instead of detection',
)
parser.add_argument('--mean-annual-precip-mm', type=float, help='Sourced site climatological precipitation, mm/year; not an automatic correction target')
parser.add_argument('--precip-reference', help='Citation, URL or evidence path for site precipitation climatology')
args = parser.parse_args()
if not os.path.exists(args.input):
print(f"Error: Input file {args.input} does not exist", file=sys.stderr)
sys.exit(1)
# Get longitude and elevation from site info
site_data = get_site_metadata(args.site_id)
longitude = site_data.get('ALONG') if site_data else None
if longitude is None:
print(f"Error: Could not get longitude for site {args.site_id}", file=sys.stderr)
sys.exit(1)
elevation = site_data.get('ALTIG') if site_data else None
print(f"Using longitude {longitude} for site {args.site_id}")
convert_era5_to_ecosim(
args.input,
args.output,
longitude,
elevation=elevation,
quality_report_file=args.quality_report,
site_id=args.site_id,
in_situ_file=args.in_situ,
compare_in_situ=not args.skip_in_situ_comparison,
comparison_min_pairs=args.comparison_min_pairs,
utc_offset_hours=args.utc_offset,
time_registration=args.time_registration,
time_shift_hours=args.time_shift_hours,
mean_annual_precip_mm=args.mean_annual_precip_mm if args.mean_annual_precip_mm is not None else site_data.get('mean_annual_precip_mm', site_data.get('MAP')),
precip_reference=args.precip_reference or site_data.get('precipitation_reference'),
)
print("Conversion completed successfully!")
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |