Datasets

Spilling large values to disk as references.

datasets

Dataset references: moving large data payloads between agents via disk pointers.

When agents return large arrays or DataFrames, storing them inline in trace.json would produce unwieldy traces. Large payloads are spilled to disk into data/ and replaced with structured pointers ($dataset), which are transparently hydrated when read by downstream agents.

is_reference

is_reference(value)

Returns True if value is a dataset reference pointer dictionary.

Parameters:

Name	Type	Description	Default
value	Any	The object to inspect.	required

Returns:

Type	Description
bool	True if value is a dataset reference dictionary; False otherwise.

Source code in python/src/science_adk/datasets.py

def is_reference(value: Any) -> bool:
    """Returns True if value is a dataset reference pointer dictionary.

    Args:
      value: The object to inspect.

    Returns:
      True if value is a dataset reference dictionary; False otherwise.
    """
    return isinstance(value, dict) and REF_KEY in value

should_spill

should_spill(value)

Determines whether a value should be saved to an external file.

Parameters:

Name	Type	Description	Default
value	Any	The value to inspect.	required

Returns:

Type	Description
bool	True if the value exceeds the spill threshold or is tabular/array data.

Source code in python/src/science_adk/datasets.py

def should_spill(value: Any) -> bool:
    """Determines whether a value should be saved to an external file.

    Args:
      value: The value to inspect.

    Returns:
      True if the value exceeds the spill threshold or is tabular/array data.
    """
    if is_reference(value) or value is None:
        return False
    if _is_dataframe(value) or _is_ndarray(value):
        return True
    return _size_of(value) > SPILL_THRESHOLD

save

save(value, data_dir, name)

Persists a value to data_dir and returns its reference pointer dictionary.

Parameters:

Name	Type	Description	Default
value	Any	The data object to persist.	required
data_dir	Path	Directory where the file should be saved.	required
name	str	Base name for the saved file.	required

Returns:

Type	Description
dict[str, Any]	A reference pointer dictionary.

Source code in python/src/science_adk/datasets.py

def save(value: Any, data_dir: Path, name: str) -> dict[str, Any]:
    """Persists a value to data_dir and returns its reference pointer dictionary.

    Args:
      value: The data object to persist.
      data_dir: Directory where the file should be saved.
      name: Base name for the saved file.

    Returns:
      A reference pointer dictionary.
    """
    data_dir.mkdir(parents=True, exist_ok=True)

    if _is_dataframe(value):
        path = data_dir / f"{name}.parquet"
        fmt = "parquet"
        try:
            value.to_parquet(path, index=False)
        except Exception:
            path = data_dir / f"{name}.csv"
            fmt = "csv"
            value.to_csv(path, index=False)
        return _pointer(
            path,
            fmt,
            shape=[int(value.shape[0]), int(value.shape[1])],
            columns=[str(c) for c in value.columns],
        )

    if _is_ndarray(value):
        numpy = _try_import("numpy")
        path = data_dir / f"{name}.npy"
        numpy.save(path, value)
        return _pointer(
            path,
            "npy",
            shape=[int(d) for d in value.shape],
            dtype=str(value.dtype),
        )

    path = data_dir / f"{name}.json"
    path.write_text(
        json.dumps(_jsonable(value), indent=2, default=str), encoding="utf-8"
    )
    pointer = _pointer(path, "json")
    if isinstance(value, (list, tuple)):
        pointer["length"] = len(value)
    elif isinstance(value, dict):
        pointer["keys"] = sorted(str(k) for k in value)[:50]
    return pointer

load

load(pointer, run_dir)

Rehydrates a dataset reference pointer into an in-memory object.

Parameters:

Name	Type	Description	Default
pointer	dict[str, Any]	Dataset pointer dictionary containing $dataset path.	required
run_dir	Path	Path to the run directory.	required

Returns:

Type	Description
Any	Rehydrated data object.

Raises:

Type	Description
FileNotFoundError	If the target file is missing.
ImportError	If required libraries (pandas, numpy) are missing.

Source code in python/src/science_adk/datasets.py

def load(pointer: dict[str, Any], run_dir: Path) -> Any:
    """Rehydrates a dataset reference pointer into an in-memory object.

    Args:
      pointer: Dataset pointer dictionary containing $dataset path.
      run_dir: Path to the run directory.

    Returns:
      Rehydrated data object.

    Raises:
      FileNotFoundError: If the target file is missing.
      ImportError: If required libraries (pandas, numpy) are missing.
    """
    if not is_reference(pointer):
        return pointer

    rel = str(pointer[REF_KEY])
    path = (run_dir / rel).resolve()
    if not path.exists():
        raise FileNotFoundError(
            f"Dataset {rel} is referenced but missing from {run_dir}."
        )

    fmt = pointer.get("format", path.suffix.lstrip("."))

    if fmt == "parquet":
        pandas = _try_import("pandas")
        if pandas is None:
            raise ImportError(f"Reading {rel} requires pandas: pip install pandas")
        return pandas.read_parquet(path)

    if fmt == "csv":
        pandas = _try_import("pandas")
        if pandas is None:
            raise ImportError(f"Reading {rel} requires pandas: pip install pandas")
        return pandas.read_csv(path)

    if fmt == "npy":
        numpy = _try_import("numpy")
        if numpy is None:
            raise ImportError(f"Reading {rel} requires numpy: pip install numpy")
        return numpy.load(path, allow_pickle=False)

    return json.loads(path.read_text(encoding="utf-8"))

hydrate

hydrate(value, run_dir)

Recursively hydrates dataset pointers in nested data structures.

Parameters:

Name	Type	Description	Default
value	Any	Object or nested structure that may contain dataset pointers.	required
run_dir	Path	Path to the active run directory.	required

Returns:

Type	Description
Any	Data structure with all pointers rehydrated into real objects.

Source code in python/src/science_adk/datasets.py

def hydrate(value: Any, run_dir: Path) -> Any:
    """Recursively hydrates dataset pointers in nested data structures.

    Args:
      value: Object or nested structure that may contain dataset pointers.
      run_dir: Path to the active run directory.

    Returns:
      Data structure with all pointers rehydrated into real objects.
    """
    if is_reference(value):
        return load(value, run_dir)
    if isinstance(value, dict):
        return {k: hydrate(v, run_dir) for k, v in value.items()}
    if isinstance(value, list):
        return [hydrate(v, run_dir) for v in value]
    return value

externalize

externalize(value, data_dir, name)

Spills large objects to disk or converts small objects to JSON-ready form.

Parameters:

Name	Type	Description	Default
value	Any	The value to process.	required
data_dir	Path	Destination data directory.	required
name	str	Base file name for externalized data.	required

Returns:

Type	Description
Any	Either a dataset pointer dictionary or a JSON-ready Python primitive.

Source code in python/src/science_adk/datasets.py

def externalize(value: Any, data_dir: Path, name: str) -> Any:
    """Spills large objects to disk or converts small objects to JSON-ready form.

    Args:
      value: The value to process.
      data_dir: Destination data directory.
      name: Base file name for externalized data.

    Returns:
      Either a dataset pointer dictionary or a JSON-ready Python primitive.
    """
    if should_spill(value):
        return save(value, data_dir, name)
    return _jsonable(value)

describe

describe(value)

Generates a concise single-line description of a value or dataset pointer.

Parameters:

Name	Type	Description	Default
value	Any	Value to describe.	required

Returns:

Type	Description
str	A short human-readable string summary.

Source code in python/src/science_adk/datasets.py

def describe(value: Any) -> str:
    """Generates a concise single-line description of a value or dataset pointer.

    Args:
      value: Value to describe.

    Returns:
      A short human-readable string summary.
    """
    if is_reference(value):
        bits = [str(value.get("format", "data"))]
        if "shape" in value:
            bits.append("x".join(str(d) for d in value["shape"]))
        elif "length" in value:
            bits.append(f"{value['length']} items")
        bits.append(f"{value.get('bytes', 0) / 1024:.1f} KiB")
        return f"{value[REF_KEY]} ({', '.join(bits)})"
    if isinstance(value, float):
        return f"{value:.6g}"
    text = str(value)
    return text if len(text) <= 80 else text[:77] + "..."