ClassyGitshadowzzh

model.py

model.py

import abc
import enum
from typing import Any, TypeVar

from pydantic import BaseModel

from tau_bench.model_utils.api.datapoint import (
    BinaryClassifyDatapoint,
    ClassifyDatapoint,
    Datapoint,
    GenerateDatapoint,
    ParseDatapoint,
    ParseForceDatapoint,
    ScoreDatapoint,
)
from tau_bench.model_utils.api.types import PartialObj

T = TypeVar("T", bound=BaseModel)


class Platform(enum.Enum):
    OPENAI = "openai"
    MISTRAL = "mistral"
    ANTHROPIC = "anthropic"
    ANYSCALE = "anyscale"
    OUTLINES = "outlines"
    VLLM_CHAT = "vllm-chat"
    VLLM_COMPLETION = "vllm-completion"


# @runtime_checkable
# class Model(Protocol):
class Model(abc.ABC):
    @abc.abstractmethod
    def get_capability(self) -> float:
        """Return the capability of the model, a float between 0.0 and 1.0."""
        raise NotImplementedError

    @abc.abstractmethod
    def get_approx_cost(self, dp: Datapoint) -> float:
        raise NotImplementedError

    @abc.abstractmethod
    def get_latency(self, dp: Datapoint) -> float:
        raise NotImplementedError

    @abc.abstractmethod
    def supports_dp(self, dp: Datapoint) -> bool:
        raise NotImplementedError


class ClassifyModel(Model):
    @abc.abstractmethod
    def classify(
        self,
        instruction: str,
        text: str,
        options: list[str],
        examples: list[ClassifyDatapoint] | None = None,
        temperature: float | None = None,
    ) -> int:
        raise NotImplementedError


class BinaryClassifyModel(Model):
    @abc.abstractmethod
    def binary_classify(
        self,
        instruction: str,
        text: str,
        examples: list[BinaryClassifyDatapoint] | None = None,
        temperature: float | None = None,
    ) -> bool:
        raise NotImplementedError


class ParseModel(Model):
    @abc.abstractmethod
    def parse(
        self,
        text: str,
        typ: type[T] | dict[str, Any],
        examples: list[ParseDatapoint] | None = None,
        temperature: float | None = None,
    ) -> T | PartialObj | dict[str, Any]:
        raise NotImplementedError


class GenerateModel(Model):
    @abc.abstractmethod
    def generate(
        self,
        instruction: str,
        text: str,
        examples: list[GenerateDatapoint] | None = None,
        temperature: float | None = None,
    ) -> str:
        raise NotImplementedError


class ParseForceModel(Model):
    @abc.abstractmethod
    def parse_force(
        self,
        instruction: str,
        typ: type[T] | dict[str, Any],
        text: str | None = None,
        examples: list[ParseForceDatapoint] | None = None,
        temperature: float | None = None,
    ) -> T | dict[str, Any]:
        raise NotImplementedError


class ScoreModel(Model):
    @abc.abstractmethod
    def score(
        self,
        instruction: str,
        text: str,
        min: int,
        max: int,
        examples: list[ScoreDatapoint] | None = None,
        temperature: float | None = None,
    ) -> int:
        raise NotImplementedError


AnyModel = (
    BinaryClassifyModel | ClassifyModel | ParseForceModel | GenerateModel | ParseModel | ScoreModel
)