Builders

This module contains functionality related to the the builders module for evaluation.

Builders

LangfuseEvaluatorBuilder

Builder for creating Langfuse evaluator instances.

Provides factory method to create configured LangfuseEvaluator with required components and metadata.

Source code in src/evaluation/builders.py
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
class LangfuseEvaluatorBuilder:
    """Builder for creating Langfuse evaluator instances.

    Provides factory method to create configured LangfuseEvaluator
    with required components and metadata.
    """

    @staticmethod
    @inject
    def build(
        query_engine: BaseQueryEngine,
        langfuse_dataset_service: LangfuseDatasetService,
        ragas_evaluator: RagasEvaluator,
        configuration: Configuration,
    ) -> LangfuseEvaluator:
        """Create configured Langfuse evaluator instance.

        Args:
            query_engine: Engine for generating responses
            langfuse_dataset_service: Service for dataset operations
            ragas_evaluator: Evaluator for quality metrics
            configuration: Evaluation settings

        Returns:
            LangfuseEvaluator: Configured evaluator instance
        """
        return LangfuseEvaluator(
            query_engine=query_engine,
            langfuse_dataset_service=langfuse_dataset_service,
            ragas_evaluator=ragas_evaluator,
            run_metadata={
                "build_name": configuration.metadata.build_name,
                "llm_configuration": configuration.pipeline.augmentation.query_engine.synthesizer.llm.name,
                "judge_llm_configuration": configuration.pipeline.evaluation.judge_llm.name,
            },
        )

build(query_engine, langfuse_dataset_service, ragas_evaluator, configuration) staticmethod

Create configured Langfuse evaluator instance.

Parameters:
  • query_engine (BaseQueryEngine) –

    Engine for generating responses

  • langfuse_dataset_service (LangfuseDatasetService) –

    Service for dataset operations

  • ragas_evaluator (RagasEvaluator) –

    Evaluator for quality metrics

  • configuration (Configuration) –

    Evaluation settings

Returns:
  • LangfuseEvaluator( LangfuseEvaluator ) –

    Configured evaluator instance

Source code in src/evaluation/builders.py
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
@staticmethod
@inject
def build(
    query_engine: BaseQueryEngine,
    langfuse_dataset_service: LangfuseDatasetService,
    ragas_evaluator: RagasEvaluator,
    configuration: Configuration,
) -> LangfuseEvaluator:
    """Create configured Langfuse evaluator instance.

    Args:
        query_engine: Engine for generating responses
        langfuse_dataset_service: Service for dataset operations
        ragas_evaluator: Evaluator for quality metrics
        configuration: Evaluation settings

    Returns:
        LangfuseEvaluator: Configured evaluator instance
    """
    return LangfuseEvaluator(
        query_engine=query_engine,
        langfuse_dataset_service=langfuse_dataset_service,
        ragas_evaluator=ragas_evaluator,
        run_metadata={
            "build_name": configuration.metadata.build_name,
            "llm_configuration": configuration.pipeline.augmentation.query_engine.synthesizer.llm.name,
            "judge_llm_configuration": configuration.pipeline.evaluation.judge_llm.name,
        },
    )

RagasEvaluatorBuilder

Builder for creating Ragas evaluator instances.

Provides factory method to create configured RagasEvaluator with required LLM and embedding components.

Source code in src/evaluation/builders.py
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
class RagasEvaluatorBuilder:
    """Builder for creating Ragas evaluator instances.

    Provides factory method to create configured RagasEvaluator
    with required LLM and embedding components.
    """

    @staticmethod
    @inject
    def build(
        judge_llm: BoundJudgeLLM,
        judge_embedding_model: BoundJudgeEmbeddingModel,
    ) -> RagasEvaluator:
        """Create configured Ragas evaluator instance.

        Args:
            judge_llm: LLM for evaluation judgments
            judge_embedding_model: Model for embedding evaluations

        Returns:
            RagasEvaluator: Configured evaluator instance
        """
        return RagasEvaluator(
            judge_llm=judge_llm, embedding_model=judge_embedding_model
        )

build(judge_llm, judge_embedding_model) staticmethod

Create configured Ragas evaluator instance.

Parameters:
  • judge_llm (BoundJudgeLLM) –

    LLM for evaluation judgments

  • judge_embedding_model (BoundJudgeEmbeddingModel) –

    Model for embedding evaluations

Returns:
  • RagasEvaluator( RagasEvaluator ) –

    Configured evaluator instance

Source code in src/evaluation/builders.py
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
@staticmethod
@inject
def build(
    judge_llm: BoundJudgeLLM,
    judge_embedding_model: BoundJudgeEmbeddingModel,
) -> RagasEvaluator:
    """Create configured Ragas evaluator instance.

    Args:
        judge_llm: LLM for evaluation judgments
        judge_embedding_model: Model for embedding evaluations

    Returns:
        RagasEvaluator: Configured evaluator instance
    """
    return RagasEvaluator(
        judge_llm=judge_llm, embedding_model=judge_embedding_model
    )