Module: llm_completion_experiment

Expand source code

# Copyright (C) 2023-present The Project Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from abc import ABC
from dataclasses import dataclass
from cl.convince.llms.llm_key import LlmKey
from cl.runtime.experiments.binary_experiment import BinaryExperiment
from cl.runtime.records.dataclasses_extensions import missing


@dataclass(slots=True, kw_only=True)
class LlmCompletionExperiment(BinaryExperiment, ABC):
    """A binary LLM experiment where pass means returning the expected completion."""

    llm: LlmKey = missing()
    """LLM used to perform the experiment."""

    query: str = missing()
    """Query passed to the LLM."""

    expected_completion: str = missing()
    """Expected completion (the comparison ignores case and whitespace)."""

Classes

class LlmCompletionExperiment (*, experiment_id: str = None, llm: LlmKey = None, query: str = None, expected_completion: str = None)

A binary LLM experiment where pass means returning the expected completion.

Expand source code

@dataclass(slots=True, kw_only=True)
class LlmCompletionExperiment(BinaryExperiment, ABC):
    """A binary LLM experiment where pass means returning the expected completion."""

    llm: LlmKey = missing()
    """LLM used to perform the experiment."""

    query: str = missing()
    """Query passed to the LLM."""

    expected_completion: str = missing()
    """Expected completion (the comparison ignores case and whitespace)."""

Ancestors

Static methods

def check_experiment_id(experiment_id: str) -> None: Inherited from: BinaryExperiment.check_experiment_id

Check that experiment_id does not have semicolon-and-space delimiter.
def get_key_type() -> Type: Inherited from: BinaryExperiment.get_key_type

Return key type even when called from a record.

Fields

var expected_completion -> str: Expected completion (the comparison ignores case and whitespace).
var experiment_id -> str: Inherited from: BinaryExperiment.experiment_id

Unique experiment identifier.
var llm -> LlmKey: LLM used to perform the experiment.
var query -> str: Query passed to the LLM.

Methods

def get_key(self) -> ExperimentKey: Inherited from: BinaryExperiment.get_key

Return a new key object whose fields populated from self, do not return self.
def init_all(self) -> None: Inherited from: BinaryExperiment.init_all

Invoke ‘init’ for each class in the order from base to derived, then validate against schema.