K-Dense-AI/scientific-agent-skills / skills/exploratory-data-analysis
exploratory-data-analysis
VerifiedPerforms bounded, local exploratory analysis of explicitly supported scientific files. Supports redacted CSV/TSV/JSON profiles; optional NumPy, HDF5, FASTA/FASTQ, and basic image metadata inspection; missingness/leakage audits; outlier and transformation sensitivity; and rigorous EDA report scaffolds. Other domain formats are reference-only and unknown formats fail closed.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 14.4 KB |
| assets/report_template.md | markdown | 7.7 KB |
| references/bioinformatics_genomics_formats.md | markdown | 8.8 KB |
| references/chemistry_molecular_formats.md | markdown | 8.5 KB |
| references/general_scientific_formats.md | markdown | 12.0 KB |
| references/microscopy_imaging_formats.md | markdown | 9.1 KB |
| references/proteomics_metabolomics_formats.md | markdown | 9.6 KB |
| references/spectroscopy_analytical_formats.md | markdown | 9.3 KB |
| scripts/__init__.py | script | 69 B |
| scripts/_capabilities.py | script | 20.2 KB |
| scripts/_common.py | script | 14.0 KB |
| scripts/_structured.py | script | 14.3 KB |
| scripts/_tabular.py | script | 32.0 KB |
| scripts/capability_manifest.py | script | 5.3 KB |
| scripts/distribution_sensitivity.py | script | 3.2 KB |
| scripts/eda_analyzer.py | script | 10.3 KB |
| scripts/image_inspector.py | script | 7.7 KB |
| scripts/missingness_leakage_audit.py | script | 3.8 KB |
| scripts/report_scaffold.py | script | 4.1 KB |
| scripts/sequence_inspector.py | script | 8.4 KB |
| scripts/tabular_profile.py | script | 3.0 KB |
scripts/_structured.py
View on GitHub#!/usr/bin/env python3
"""Bounded inspectors for JSON, NumPy containers, and HDF5 metadata."""
from __future__ import annotations
import itertools
import math
from pathlib import Path
from typing import Any
from _capabilities import preflight_npz
from _common import (
MAX_JSON_BYTES,
CliError,
display_identifier,
load_strict_json,
)
from _tabular import profile_json_structure
MAX_ARRAY_SAMPLE = 4096
MAX_HDF5_OBJECTS = 1000
MAX_HDF5_DEPTH = 16
def inspect_json(
path: Path,
*,
reveal_identifiers: bool = False,
) -> dict[str, Any]:
"""Strictly parse bounded JSON and emit structure, never scalar values."""
document = load_strict_json(path, max_bytes=MAX_JSON_BYTES)
return profile_json_structure(
document,
reveal_identifiers=reveal_identifiers,
)
def _dtype_report(
dtype: Any,
*,
reveal_identifiers: bool,
) -> dict[str, Any]:
fields = getattr(dtype, "fields", None)
report: dict[str, Any] = {
"kind": str(dtype.kind),
"itemsize": int(dtype.itemsize),
"byteorder": str(dtype.byteorder),
"contains_python_objects": bool(dtype.hasobject),
"structured": fields is not None,
}
if fields is not None:
names = list(dtype.names or ())
report["field_count"] = len(names)
report["field_ids"] = [
display_identifier(
name,
kind="field",
reveal_identifiers=reveal_identifiers,
)
for name in names[:128]
]
report["field_ids_truncated"] = len(names) > 128
else:
report["dtype"] = str(dtype)
return report
def _sample_array(array: Any, np: Any) -> Any:
size = int(array.size)
if size <= MAX_ARRAY_SAMPLE:
return np.asarray(array).reshape(-1)
positions = np.linspace(
0,
size - 1,
num=MAX_ARRAY_SAMPLE,
dtype=np.int64,
)
# ndarray.flat indexes in C order without materializing a full C-order
# copy of a Fortran-contiguous memory map.
return np.asarray(array.flat[positions])
def _array_report(
array: Any,
np: Any,
*,
array_id: str | None,
reveal_identifiers: bool,
) -> dict[str, Any]:
if bool(array.dtype.hasobject):
raise CliError("NumPy object arrays are not accepted because they require pickle")
report: dict[str, Any] = {
"shape": [int(value) for value in array.shape],
"dimension_count": int(array.ndim),
"element_count": int(array.size),
"dtype": _dtype_report(
array.dtype,
reveal_identifiers=reveal_identifiers,
),
}
if array_id is not None:
report["array_id"] = array_id
if int(array.size) == 0:
report["numeric_sample"] = {"sample_count": 0}
return report
if np.issubdtype(array.dtype, np.bool_):
sample = _sample_array(array, np)
report["boolean_sample"] = {
"sample_count": int(sample.size),
"true_count": int(np.count_nonzero(sample)),
"sample_is_bounded": int(sample.size) < int(array.size),
}
return report
if np.issubdtype(array.dtype, np.number):
sample = _sample_array(array, np)
if np.iscomplexobj(sample):
finite = np.isfinite(sample.real) & np.isfinite(sample.imag)
magnitudes = np.abs(sample[finite])
report["numeric_sample"] = {
"sample_count": int(sample.size),
"finite_count": int(np.count_nonzero(finite)),
"complex_values_summarized_by_magnitude": True,
"magnitude_mean": (
float(np.mean(magnitudes)) if magnitudes.size else None
),
"magnitude_minimum": (
float(np.min(magnitudes)) if magnitudes.size else None
),
"magnitude_maximum": (
float(np.max(magnitudes)) if magnitudes.size else None
),
"sample_is_bounded": int(sample.size) < int(array.size),
}
else:
finite = np.isfinite(sample)
finite_values = sample[finite]
report["numeric_sample"] = {
"sample_count": int(sample.size),
"finite_count": int(np.count_nonzero(finite)),
"nan_count": int(np.count_nonzero(np.isnan(sample)))
if np.issubdtype(sample.dtype, np.inexact)
else 0,
"infinite_count": int(np.count_nonzero(np.isinf(sample)))
if np.issubdtype(sample.dtype, np.inexact)
else 0,
"mean": (
float(np.mean(finite_values)) if finite_values.size else None
),
"minimum": (
float(np.min(finite_values)) if finite_values.size else None
),
"maximum": (
float(np.max(finite_values)) if finite_values.size else None
),
"sample_is_bounded": int(sample.size) < int(array.size),
}
return report
def inspect_numpy(
path: Path,
*,
suffix: str,
reveal_identifiers: bool = False,
) -> dict[str, Any]:
"""Inspect NPY/NPZ with pickle disabled and bounded decompression."""
try:
import numpy as np
except ImportError as exc:
raise CliError(
'optional dependency missing; install with: uv pip install "numpy==2.5.3"'
) from exc
if suffix == ".npy":
try:
array = np.load(
path,
mmap_mode="r",
allow_pickle=False,
max_header_size=10_000,
)
except (OSError, ValueError, TypeError, MemoryError) as exc:
raise CliError(
"the NPY file could not be inspected safely; object arrays are rejected"
) from exc
return {
"profile_type": "numpy_npy_bounded_profile",
"pickle_allowed": False,
"array": _array_report(
array,
np,
array_id=None,
reveal_identifiers=reveal_identifiers,
),
"raw_values_emitted": False,
}
if suffix != ".npz":
raise CliError("the NumPy inspector received an unsupported suffix")
preflight = preflight_npz(path)
arrays: list[dict[str, Any]] = []
try:
with np.load(
path,
allow_pickle=False,
max_header_size=10_000,
) as archive:
for name in archive.files:
array = archive[name]
arrays.append(
_array_report(
array,
np,
array_id=display_identifier(
name,
kind="array",
reveal_identifiers=reveal_identifiers,
),
reveal_identifiers=reveal_identifiers,
)
)
del array
except (OSError, ValueError, TypeError, MemoryError) as exc:
raise CliError(
"the NPZ arrays could not be inspected safely; object arrays are rejected"
) from exc
return {
"profile_type": "numpy_npz_bounded_profile",
"pickle_allowed": False,
"archive_preflight": preflight,
"arrays": arrays,
"raw_values_emitted": False,
}
def _shape_elements(shape: Any) -> int | None:
if shape is None:
return None
total = 1
for value in shape:
total *= int(value)
if total > 2**63 - 1:
return None
return total
def inspect_hdf5(
path: Path,
*,
reveal_identifiers: bool = False,
) -> dict[str, Any]:
"""Inspect HDF5 links and metadata without reading dataset values."""
try:
import h5py
except ImportError as exc:
raise CliError(
'optional dependency missing; install with: uv pip install "h5py==3.16.0"'
) from exc
objects: list[dict[str, Any]] = []
link_counts = {"hard": 0, "soft_not_followed": 0, "external_not_followed": 0}
object_limit_reached = False
depth_limit_reached = False
seen_addresses: set[int] = set()
try:
with h5py.File(path, "r") as handle:
stack: list[tuple[Any, str, int]] = [(handle, "/", 0)]
while stack:
group, logical_path, depth = stack.pop()
if len(objects) >= MAX_HDF5_OBJECTS:
object_limit_reached = True
break
for name in itertools.islice(
group.keys(),
MAX_HDF5_OBJECTS - len(objects) + 1,
):
if len(objects) >= MAX_HDF5_OBJECTS:
object_limit_reached = True
break
link = group.get(name, getlink=True)
child_path = (
f"/{name}" if logical_path == "/" else f"{logical_path}/{name}"
)
object_id = display_identifier(
child_path,
kind="hdf_object",
reveal_identifiers=reveal_identifiers,
)
if isinstance(link, h5py.ExternalLink):
link_counts["external_not_followed"] += 1
objects.append(
{
"object_id": object_id,
"type": "external_link",
"followed": False,
}
)
continue
if isinstance(link, h5py.SoftLink):
link_counts["soft_not_followed"] += 1
objects.append(
{
"object_id": object_id,
"type": "soft_link",
"followed": False,
}
)
continue
if not isinstance(link, h5py.HardLink):
raise CliError("the HDF5 file contains an unsupported link type")
link_counts["hard"] += 1
child = group.get(name)
if child is None:
raise CliError("the HDF5 file contains an unresolved hard link")
address = int(h5py.h5o.get_info(child.id).addr)
already_seen = address in seen_addresses
seen_addresses.add(address)
if isinstance(child, h5py.Dataset):
compression = child.compression
external_count = len(child.external or ())
objects.append(
{
"object_id": object_id,
"type": "dataset",
"hard_link_alias": already_seen,
"shape": [int(value) for value in child.shape]
if child.shape is not None
else None,
"element_count": _shape_elements(child.shape),
"dtype": _dtype_report(
child.dtype,
reveal_identifiers=reveal_identifiers,
),
"attribute_count": len(child.attrs),
"chunked": child.chunks is not None,
"chunk_shape": [int(value) for value in child.chunks]
if child.chunks is not None
else None,
"compressed": compression is not None,
"compression_id": display_identifier(
str(compression),
kind="compression",
reveal_identifiers=reveal_identifiers,
)
if compression is not None
else None,
"external_storage_file_count": external_count,
"virtual_dataset": bool(child.is_virtual),
"values_read": False,
}
)
elif isinstance(child, h5py.Group):
objects.append(
{
"object_id": object_id,
"type": "group",
"hard_link_alias": already_seen,
"attribute_count": len(child.attrs),
}
)
if already_seen:
continue
if depth >= MAX_HDF5_DEPTH:
depth_limit_reached = True
else:
stack.append((child, child_path, depth + 1))
else:
raise CliError("the HDF5 file contains an unsupported object type")
except CliError:
raise
except (OSError, RuntimeError, ValueError, TypeError, MemoryError) as exc:
raise CliError("the HDF5 metadata could not be inspected safely") from exc
return {
"profile_type": "hdf5_metadata_only",
"object_count_reported": len(objects),
"object_limit_reached": object_limit_reached,
"depth_limit_reached": depth_limit_reached,
"links": link_counts,
"objects": objects,
"dataset_values_read": False,
"attributes_values_read": False,
"soft_links_followed": False,
"external_links_followed": False,
"external_dataset_storage_read": False,
"filter_plugins_invoked_for_data": False,
"raw_values_emitted": False,
"limitations": [
"Generic hierarchy inspection is not semantic validation of H5AD, Loom, or other HDF5 conventions.",
"Dataset payloads are intentionally not read, so value-level statistics are unavailable.",
],
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 463k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 463k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 463k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 192k | Updated | Claude CodeCodex | Verified | Install |