Skip to content

dataset

dataset

DatasetProvider adapter for personal benchmarks.

Classes

PersonalBenchmarkDataset 

PersonalBenchmarkDataset(benchmark: PersonalBenchmark)

Bases: DatasetProvider

Wraps a PersonalBenchmark as a DatasetProvider for EvalRunner.

Source code in src/diapason/learning/optimize/personal/dataset.py 
18
19
20
def __init__(self, benchmark: PersonalBenchmark) -> None:
    self._benchmark = benchmark
    self._records: List[EvalRecord] = []
Methods:
load 
load(
    *,
    max_samples: Optional[int] = None,
    split: Optional[str] = None,
    seed: Optional[int] = None,
) -> None

Convert :class:PersonalBenchmarkSample instances to :class:EvalRecord.

Source code in src/diapason/learning/optimize/personal/dataset.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
def load(
    self,
    *,
    max_samples: Optional[int] = None,
    split: Optional[str] = None,
    seed: Optional[int] = None,
) -> None:
    """Convert :class:`PersonalBenchmarkSample` instances to :class:`EvalRecord`."""
    samples = self._benchmark.samples
    if max_samples is not None:
        samples = samples[:max_samples]
    self._records = [
        EvalRecord(
            record_id=s.trace_id,
            problem=s.query,
            reference=s.reference_answer,
            category=s.category,
            subject=s.agent or "general",
            metadata=s.metadata,
        )
        for s in samples
    ]
iter_records 
iter_records() -> Iterable[EvalRecord]

Iterate over loaded records.

Source code in src/diapason/learning/optimize/personal/dataset.py 
45
46
47
def iter_records(self) -> Iterable[EvalRecord]:
    """Iterate over loaded records."""
    return iter(self._records)
size 
size() -> int

Return the number of loaded records.

Source code in src/diapason/learning/optimize/personal/dataset.py 
49
50
51
def size(self) -> int:
    """Return the number of loaded records."""
    return len(self._records)