Module: llm_completion_trial

Expand source code

# Copyright (C) 2023-present The Project Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from dataclasses import dataclass

from cl.runtime.experiments.binary_trial import BinaryTrial
from cl.runtime.records.dataclasses_extensions import missing
from cl.runtime.experiments.trial import Trial
from cl.runtime.experiments.trial_key import TrialKey


@dataclass(slots=True, kw_only=True)
class LlmCompletionTrial(BinaryTrial):
    """Run and analyze the results of a single binary (pass/fail) trial."""

    binary_result: bool = missing()
    """True for pass and False for fail."""

Classes

class LlmCompletionTrial (*, trial_id: str = None, experiment: ExperimentKey = None, trial_label: str = None, binary_result: bool = None)

Run and analyze the results of a single binary (pass/fail) trial.

Expand source code

@dataclass(slots=True, kw_only=True)
class LlmCompletionTrial(BinaryTrial):
    """Run and analyze the results of a single binary (pass/fail) trial."""

    binary_result: bool = missing()
    """True for pass and False for fail."""

Ancestors

BinaryTrial
Trial
TrialKey
KeyMixin
abc.ABC
RecordMixin
typing.Generic

Static methods

def check_trial_id(trial_id: str) -> None: Inherited from: BinaryTrial.check_trial_id

Check that trial_id has the expected ‘ExperimentId: TrialLabel’ format.
def get_key_type() -> Type: Inherited from: BinaryTrial.get_key_type

Return key type even when called from a record.
def get_trial_id(experiment: ExperimentKey, trial_label: str) -> str: Inherited from: BinaryTrial.get_trial_id

Generate trial_id in ‘ExperimentId: TrialLabel’ format.

Fields

var binary_result -> bool: Inherited from: BinaryTrial.binary_result

True for pass and False for fail.
var experiment -> ExperimentKey: Inherited from: BinaryTrial.experiment

Experiment for which the trial is performed.
var trial_id -> str: Inherited from: BinaryTrial.trial_id

Unique identifier of the trial using ‘ExperimentId: TrialLabel’ format.
var trial_label -> str: Inherited from: BinaryTrial.trial_label

Identifier of the trial is unique within the experiment.

Methods

def get_key(self) -> TrialKey: Inherited from: BinaryTrial.get_key

Return a new key object whose fields populated from self, do not return self.
def init(self) -> None: Inherited from: BinaryTrial.init

Generate trial_id in ‘ExperimentId: TrialLabel’ format.
def init_all(self) -> None: Inherited from: BinaryTrial.init_all

Invoke ‘init’ for each class in the order from base to derived, then validate against schema.