Evaluation_binder

This module contains functionality related to the the evaluation_binder module for common.bootstrap.configuration.pipeline.evaluation.

Evaluation_binder

EvaluationBinder

Bases: BaseBinder

Binder for the evaluation components.

Source code in src/common/bootstrap/configuration/pipeline/evaluation/evaluation_binder.py
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
class EvaluationBinder(BaseBinder):
    """Binder for the evaluation components."""

    def bind(self) -> None:
        """Bind the evaluation components."""
        self._bind_judge_llm()
        self._bind_judge_embedding_model()
        self._bind_ragas_evaluator()
        self._bind_langfuse_evaluator()

    def _bind_judge_llm(self) -> None:
        """Bind the judge LLM.

        If the judge LLM configuration is the same as the LLM configuration or auto retriever LLM bind the the same LLM to judge LLM.
        """
        judge_llm_configuration = (
            self.configuration.pipeline.evaluation.judge_llm
        )
        llm_configuration = (
            self.configuration.pipeline.augmentation.query_engine.synthesizer.llm
        )
        retriever_configuration = (
            self.configuration.pipeline.augmentation.query_engine.retriever
        )

        if self._pydantic_config_is_equal(
            judge_llm_configuration, llm_configuration
        ):
            self.binder.bind(
                BoundJudgeLLM,
                to=self._get_bind(BoundLLM),
                scope=singleton,
            )
        elif (
            isinstance(retriever_configuration, AutoRetrieverConfiguration)
            and retriever_configuration.llm == judge_llm_configuration
        ):
            self.binder.bind(
                BoundJudgeLLM,
                to=self._get_bind(BoundAutoRetrieverLLM),
                scope=singleton,
            )
        else:
            self.binder.bind(
                BoundJudgeLLM,
                to=lambda: judge_llm_configuration.builder(
                    judge_llm_configuration
                ),
                scope=singleton,
            )

    def _bind_judge_embedding_model(self) -> None:
        """Bind the judge embedding model.

        If the judge embedding model configuration is the same as the embedding model configuration bind the same embedding model to judge embedding model.
        """
        judge_embedding_model_configuration = (
            self.configuration.pipeline.evaluation.judge_embedding_model
        )
        embedding_model_configuration = (
            self.configuration.pipeline.embedding.embedding_model
        )

        if self._pydantic_config_is_equal(
            judge_embedding_model_configuration, embedding_model_configuration
        ):
            self.binder.bind(
                BoundJudgeEmbeddingModel,
                to=self._get_bind(BoundEmbeddingModel),
                scope=singleton,
            )
        else:
            self.binder.bind(
                BoundJudgeEmbeddingModel,
                to=lambda: judge_embedding_model_configuration.builder(
                    judge_embedding_model_configuration
                ),
                scope=singleton,
            )

    def _bind_ragas_evaluator(self) -> None:
        """Bind the Ragas evaluator."""
        self.binder.bind(
            RagasEvaluator,
            to=RagasEvaluatorBuilder.build,
        )

    def _bind_langfuse_evaluator(self) -> None:
        """Bind the Langfuse evaluator."""
        self.binder.bind(
            LangfuseEvaluator,
            to=LangfuseEvaluatorBuilder.build,
        )

_bind_judge_embedding_model()

Bind the judge embedding model.

If the judge embedding model configuration is the same as the embedding model configuration bind the same embedding model to judge embedding model.

Source code in src/common/bootstrap/configuration/pipeline/evaluation/evaluation_binder.py
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def _bind_judge_embedding_model(self) -> None:
    """Bind the judge embedding model.

    If the judge embedding model configuration is the same as the embedding model configuration bind the same embedding model to judge embedding model.
    """
    judge_embedding_model_configuration = (
        self.configuration.pipeline.evaluation.judge_embedding_model
    )
    embedding_model_configuration = (
        self.configuration.pipeline.embedding.embedding_model
    )

    if self._pydantic_config_is_equal(
        judge_embedding_model_configuration, embedding_model_configuration
    ):
        self.binder.bind(
            BoundJudgeEmbeddingModel,
            to=self._get_bind(BoundEmbeddingModel),
            scope=singleton,
        )
    else:
        self.binder.bind(
            BoundJudgeEmbeddingModel,
            to=lambda: judge_embedding_model_configuration.builder(
                judge_embedding_model_configuration
            ),
            scope=singleton,
        )

_bind_judge_llm()

Bind the judge LLM.

If the judge LLM configuration is the same as the LLM configuration or auto retriever LLM bind the the same LLM to judge LLM.

Source code in src/common/bootstrap/configuration/pipeline/evaluation/evaluation_binder.py
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def _bind_judge_llm(self) -> None:
    """Bind the judge LLM.

    If the judge LLM configuration is the same as the LLM configuration or auto retriever LLM bind the the same LLM to judge LLM.
    """
    judge_llm_configuration = (
        self.configuration.pipeline.evaluation.judge_llm
    )
    llm_configuration = (
        self.configuration.pipeline.augmentation.query_engine.synthesizer.llm
    )
    retriever_configuration = (
        self.configuration.pipeline.augmentation.query_engine.retriever
    )

    if self._pydantic_config_is_equal(
        judge_llm_configuration, llm_configuration
    ):
        self.binder.bind(
            BoundJudgeLLM,
            to=self._get_bind(BoundLLM),
            scope=singleton,
        )
    elif (
        isinstance(retriever_configuration, AutoRetrieverConfiguration)
        and retriever_configuration.llm == judge_llm_configuration
    ):
        self.binder.bind(
            BoundJudgeLLM,
            to=self._get_bind(BoundAutoRetrieverLLM),
            scope=singleton,
        )
    else:
        self.binder.bind(
            BoundJudgeLLM,
            to=lambda: judge_llm_configuration.builder(
                judge_llm_configuration
            ),
            scope=singleton,
        )

_bind_langfuse_evaluator()

Bind the Langfuse evaluator.

Source code in src/common/bootstrap/configuration/pipeline/evaluation/evaluation_binder.py
111
112
113
114
115
116
def _bind_langfuse_evaluator(self) -> None:
    """Bind the Langfuse evaluator."""
    self.binder.bind(
        LangfuseEvaluator,
        to=LangfuseEvaluatorBuilder.build,
    )

_bind_ragas_evaluator()

Bind the Ragas evaluator.

Source code in src/common/bootstrap/configuration/pipeline/evaluation/evaluation_binder.py
104
105
106
107
108
109
def _bind_ragas_evaluator(self) -> None:
    """Bind the Ragas evaluator."""
    self.binder.bind(
        RagasEvaluator,
        to=RagasEvaluatorBuilder.build,
    )

bind()

Bind the evaluation components.

Source code in src/common/bootstrap/configuration/pipeline/evaluation/evaluation_binder.py
27
28
29
30
31
32
def bind(self) -> None:
    """Bind the evaluation components."""
    self._bind_judge_llm()
    self._bind_judge_embedding_model()
    self._bind_ragas_evaluator()
    self._bind_langfuse_evaluator()