.

Author: penguine-ip

deepeval/benchmarks/big_bench_hard/big_bench_hard.py

@@ -171,7 +171,7 @@ def load_benchmark_dataset(self, task: BigBenchHardTask) -> List[Golden]:
 
         goldens: List[Golden] = []
         for data in dataset["test"]:
-            golden = Golden(input=data["input"], expectedOutput=data["target"])
+            golden = Golden(input=data["input"], expected_output=data["target"])
             goldens.append(golden)
 
         return goldens

deepeval/benchmarks/drop/drop.py

@@ -181,7 +181,7 @@ def load_benchmark_dataset(self, task: DROPTask) -> List[Golden]:
             output = DELIMITER.join(tuple(data["answers_spans"]["spans"][0]))
             output_type = data["answers_spans"]["types"][0]
             golden = Golden(
-                input=input, expectedOutput=output, context=[output_type]
+                input=input, expected_output=output, context=[output_type]
             )
             goldens.append(golden)
 

deepeval/benchmarks/gsm8k/gsm8k.py

@@ -98,7 +98,7 @@ def load_benchmark_dataset(self) -> List[Golden]:
         for data in dataset["test"]:
             input = data["question"]
             output = GSM8KTemplate.format_answer(data)
-            golden = Golden(input=input, expectedOutput=output)
+            golden = Golden(input=input, expected_output=output)
             goldens.append(golden)
 
         return goldens

deepeval/benchmarks/hellaswag/hellaswag.py

@@ -192,7 +192,7 @@ def load_benchmark_dataset(self, task: HellaSwagTask) -> List[Golden]:
                 data, include_answer=False
             )
             golden = Golden(
-                input=input, expectedOutput=choices[int(data["label"])]
+                input=input, expected_output=choices[int(data["label"])]
             )
             goldens.append(golden)
         return goldens

deepeval/benchmarks/human_eval/human_eval.py

@@ -118,6 +118,6 @@ def load_benchmark_dataset(self, task: HumanEvalTask) -> List[Golden]:
         )[0]
         # Construct test set
         golden = Golden(
-            input=test_set["prompt"], expectedOutput=test_set["test"]
+            input=test_set["prompt"], expected_output=test_set["test"]
         )
         return golden

deepeval/benchmarks/mmlu/mmlu.py

@@ -179,6 +179,6 @@ def load_benchmark_dataset(self, task: MMLUTask) -> List[Golden]:
         goldens: List[Golden] = []
         for data in dataset["test"]:
             input = MMLUTemplate.format_question(data, include_answer=False)
-            golden = Golden(input=input, expectedOutput=data["target"])
+            golden = Golden(input=input, expected_output=data["target"])
             goldens.append(golden)
         return goldens

deepeval/benchmarks/truthful_qa/truthful_qa.py

@@ -195,14 +195,14 @@ def load_benchmark_dataset(
                 input, expected_output = TruthfulQATemplate.format_mc1_question(
                     data
                 )
-                golden = Golden(input=input, expectedOutput=expected_output)
+                golden = Golden(input=input, expected_output=expected_output)
                 goldens.append(golden)
             elif mode == TruthfulQAMode.MC2:
                 input, expected_output = TruthfulQATemplate.format_mc2_question(
                     data
                 )
                 golden = Golden(
-                    input=input, expectedOutput=str(expected_output)
+                    input=input, expected_output=str(expected_output)
                 )
                 goldens.append(golden)
 

deepeval/dataset/api.py

@@ -9,7 +9,7 @@ class APIDataset(BaseModel):
     overwrite: bool
     goldens: Optional[List[Golden]] = Field(default=[])
     conversational_goldens: Optional[List[ConversationalGolden]] = Field(
-        default=[], serialization_alias="conversationalGoldens"
+        default=[], alias="conversationalGoldens"
     )
 
 

deepeval/dataset/utils.py

@@ -10,10 +10,10 @@ def convert_test_cases_to_goldens(
     for test_case in test_cases:
         golden = {
             "input": test_case.input,
-            "actualOutput": test_case.actual_output,
-            "expectedOutput": test_case.expected_output,
+            "actual_output": test_case.actual_output,
+            "expected_output": test_case.expected_output,
             "context": test_case.context,
-            "retrievalContext": test_case.retrieval_context,
+            "retrieval_context": test_case.retrieval_context,
         }
         goldens.append(Golden(**golden))
     return goldens

deepeval/evaluate.py

@@ -114,12 +114,12 @@ def create_api_test_case(
             # to each individual message (test case)
             test_case.additional_metadata = additional_metadata
             test_case.comments = comments
-            traceStack = None
+            trace_stack = None
         else:
             success = True
             name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{index}")
             order = test_case._dataset_rank
-            traceStack = get_trace_stack()
+            trace_stack = get_trace_stack()
 
         return LLMApiTestCase(
             name=name,
@@ -135,7 +135,7 @@ def create_api_test_case(
             order=order,
             additionalMetadata=test_case.additional_metadata,
             comments=test_case.comments,
-            traceStack=traceStack,
+            traceStack=trace_stack,
         )
 
     elif isinstance(test_case, ConversationalTestCase):

deepeval/event/api.py

@@ -22,28 +22,22 @@ class Config:
 
 
 class APIEvent(BaseModel):
-    name: str = Field(..., serialization_alias="name")
+    name: str = Field(..., alias="name")
     input: str
     response: str
     retrieval_context: Optional[List[str]] = Field(
-        None, serialization_alias="retrievalContext"
-    )
-    completion_time: Optional[float] = Field(
-        None, serialization_alias="completionTime"
-    )
-    token_usage: Optional[float] = Field(None, serialization_alias="tokenUsage")
-    token_cost: Optional[float] = Field(None, serialization_alias="tokenCost")
-    distinct_id: Optional[str] = Field(None, serialization_alias="distinctId")
-    conversation_id: Optional[str] = Field(
-        None, serialization_alias="conversationId"
+        None, alias="retrievalContext"
     )
+    completion_time: Optional[float] = Field(None, alias="completionTime")
+    token_usage: Optional[float] = Field(None, alias="tokenUsage")
+    token_cost: Optional[float] = Field(None, alias="tokenCost")
+    distinct_id: Optional[str] = Field(None, alias="distinctId")
+    conversation_id: Optional[str] = Field(None, alias="conversationId")
     custom_properties: Optional[Dict[str, CustomProperty]] = Field(
-        None, serialization_alias="customProperties"
-    )
-    trace_stack: Optional[Dict] = Field(None, serialization_alias="traceStack")
-    trace_provider: Optional[str] = Field(
-        None, serialization_alias="traceProvider"
+        None, alias="customProperties"
     )
+    trace_stack: Optional[Dict] = Field(None, alias="traceStack")
+    trace_provider: Optional[str] = Field(None, alias="traceProvider")
     hyperparameters: Optional[Dict] = Field(None)
 
     class Config:

deepeval/event/feedback.py

@@ -21,10 +21,10 @@ def send_feedback(
             raise ValueError("'rating' must be between 1 and 5, inclusive.")
 
         api_event = APIFeedback(
-            eventId=event_id,
+            event_id=event_id,
             provider=provider,
             rating=rating,
-            expectedResponse=expected_response,
+            expected_response=expected_response,
             explanation=explanation,
         )
         api = Api()

deepeval/integrations/hugging_face/tests/test_callbacks.py

@@ -76,14 +76,13 @@ def create_deepeval_dataset(dataset, sample_size):
     random_index_list = [
         random.randint(0, total_length) for _ in range(sample_size)
     ]
-    print(random_index_list)
     eval_dataset = [dataset[row] for row in random_index_list]
     goldens = []
     for row in eval_dataset:
         context = ["; ".join(row["context"]["contexts"])]
         golden = Golden(
             input=row["question"],
-            expectedOutput=row["long_answer"],
+            expected_output=row["long_answer"],
             context=context,
             retrieval_context=context,
         )

deepeval/integrations/llama_index/callback.py

@@ -175,10 +175,12 @@ def create_trace_instance(
                 traces=[],
                 llmMetadata=LlmMetadata(
                     model=processed_payload["llm_model_name"],
-                    outputMessages=None,
-                    tokenCount=None,
-                    promptTemplate=processed_payload.get("llm_prompt_template"),
-                    promptTemplateVariables=processed_payload.get(
+                    output_messages=None,
+                    token_count=None,
+                    prompt_template=processed_payload.get(
+                        "llm_prompt_template"
+                    ),
+                    prompt_template_variables=processed_payload.get(
                         "llm_prompt_template_variables"
                     ),
                 ),
@@ -224,7 +226,7 @@ def create_trace_instance(
                 traces=[],
                 rerankingMetadata=RerankingMetadata(
                     model=processed_payload["reranker_model_name"],
-                    topK=processed_payload["reranker_top_k"],
+                    top_k=processed_payload["reranker_top_k"],
                 ),
             )
 

deepeval/synthesizer/synthesizer.py

@@ -202,7 +202,7 @@ def _generate_from_contexts(
                 source_files[index] if source_files is not None else None
             )
             golden = Golden(
-                input=evolved_input, context=context, sourceFile=source_file
+                input=evolved_input, context=context, source_file=source_file
             )
 
             if include_expected_output:

deepeval/test_run/api.py

@@ -8,45 +8,33 @@ class MetricMetadata(BaseModel):
     success: bool
     score: Optional[float] = None
     reason: Optional[str] = None
-    strict_mode: Optional[bool] = Field(False, serialization_alias="strictMode")
-    evaluation_model: Optional[str] = Field(
-        None, serialization_alias="evaluationModel"
-    )
+    strict_mode: Optional[bool] = Field(False, alias="strictMode")
+    evaluation_model: Optional[str] = Field(None, alias="evaluationModel")
     error: Optional[str] = None
-    evaluation_cost: Union[float, None] = Field(
-        None, serialization_alias="evaluationCost"
-    )
+    evaluation_cost: Union[float, None] = Field(None, alias="evaluationCost")
 
 
 class LLMApiTestCase(BaseModel):
     name: str
     input: str
-    actual_output: str = Field(..., serialization_alias="actualOutput")
-    expected_output: Optional[str] = Field(
-        None, serialization_alias="expectedOutput"
-    )
+    actual_output: str = Field(..., alias="actualOutput")
+    expected_output: Optional[str] = Field(None, alias="expectedOutput")
     context: Optional[list] = Field(None)
-    retrieval_context: Optional[list] = Field(
-        None, serialization_alias="retrievalContext"
-    )
+    retrieval_context: Optional[list] = Field(None, alias="retrievalContext")
     # make optional, not all test cases in a conversation will be evaluated
     success: Union[bool, None] = Field(None)
     # make optional, not all test cases in a conversation will be evaluated
     metrics_metadata: Union[List[MetricMetadata], None] = Field(
-        None, serialization_alias="metricsMetadata"
+        None, alias="metricsMetadata"
     )
     # make optional, not all test cases in a conversation will be evaluated
-    run_duration: Union[float, None] = Field(
-        None, serialization_alias="runDuration"
-    )
+    run_duration: Union[float, None] = Field(None, alias="runDuration")
     # make optional, not all test cases in a conversation will be evaluated
-    evaluation_cost: Union[float, None] = Field(
-        None, serialization_alias="evaluationCost"
-    )
+    evaluation_cost: Union[float, None] = Field(None, alias="evaluationCost")
     order: Union[int, None] = Field(None)
     # These should map 1 to 1 from golden
     additional_metadata: Optional[Dict] = Field(
-        None, serialization_alias="additionalMetadata"
+        None, alias="additionalMetadata"
     )
     comments: Optional[str] = Field(None)
     traceStack: Optional[dict] = Field(None)
@@ -80,14 +68,12 @@ class ConversationalApiTestCase(BaseModel):
     success: bool
     # metrics_metadata can be None when we're not evaluating using conversational metrics
     metrics_metadata: Union[List[MetricMetadata], None] = Field(
-        None, serialization_alias="metricsMetadata"
-    )
-    run_duration: float = Field(0.0, serialization_alias="runDuration")
-    evaluation_cost: Union[float, None] = Field(
-        None, serialization_alias="evaluationCost"
+        None, alias="metricsMetadata"
     )
+    run_duration: float = Field(0.0, alias="runDuration")
+    evaluation_cost: Union[float, None] = Field(None, alias="evaluationCost")
     messages: List[LLMApiTestCase] = Field(
-        default_factory=lambda: [], serialization_alias="testCases"
+        default_factory=lambda: [], alias="testCases"
     )
     order: Union[int, None] = Field(None)
 

deepeval/test_run/test_run.py

@@ -91,7 +91,7 @@ class TestRun(BaseModel):
     )
     deployment: Optional[bool] = Field(True)
     deployment_configs: Optional[DeploymentConfigs] = Field(
-        None, serialization_alias="deploymentConfigs"
+        None, alias="deploymentConfigs"
     )
     test_cases: List[LLMApiTestCase] = Field(
         alias="testCases", default_factory=lambda: []
@@ -100,19 +100,15 @@ class TestRun(BaseModel):
         alias="conversationalTestCases", default_factory=lambda: []
     )
     metrics_scores: List[MetricScores] = Field(
-        default_factory=lambda: [], serialization_alias="metricsScores"
+        default_factory=lambda: [], alias="metricsScores"
     )
     hyperparameters: Optional[Dict[Any, Any]] = Field(None)
-    test_passed: Optional[int] = Field(None, serialization_alias="testPassed")
-    test_failed: Optional[int] = Field(None, serialization_alias="testFailed")
-    run_duration: float = Field(0.0, serialization_alias="runDuration")
-    evaluation_cost: Union[float, None] = Field(
-        None, serialization_alias="evaluationCost"
-    )
-    dataset_alias: Optional[str] = Field(
-        None, serialization_alias="datasetAlias"
-    )
-    dataset_id: Optional[str] = Field(None, serialization_alias="datasetId")
+    test_passed: Optional[int] = Field(None, alias="testPassed")
+    test_failed: Optional[int] = Field(None, alias="testFailed")
+    run_duration: float = Field(0.0, alias="runDuration")
+    evaluation_cost: Union[float, None] = Field(None, alias="evaluationCost")
+    dataset_alias: Optional[str] = Field(None, alias="datasetAlias")
+    dataset_id: Optional[str] = Field(None, alias="datasetId")
 
     def add_test_case(
         self, api_test_case: Union[LLMApiTestCase, ConversationalApiTestCase]
@@ -557,8 +553,8 @@ def post_test_run(self, test_run: TestRun):
 
                 remaining_test_run = RemainingTestRun(
                     testRunId=response.testRunId,
-                    testCases=test_case_batch,
-                    conversationalTestCases=conversational_batch,
+                    test_cases=test_case_batch,
+                    conversational_test_cases=conversational_batch,
                 )
 
                 body = None

tests/test_everything.py

@@ -80,7 +80,7 @@
 """
 
 strict_mode = False
-verbose_mode = True
+verbose_mode = False
 
 
 @pytest.mark.skip(reason="openai is expensive")
@@ -89,7 +89,7 @@ def test_everything():
         threshold=0.1,
         strict_mode=strict_mode,
         async_mode=False,
-        verbose_mode=True,
+        verbose_mode=verbose_mode,
     )
     metric2 = FaithfulnessMetric(
         threshold=0.5, strict_mode=strict_mode, verbose_mode=verbose_mode
@@ -167,7 +167,7 @@ def test_everything():
     )
 
 
-@pytest.mark.skip(reason="openadi is expensive")
+# @pytest.mark.skip(reason="openadi is expensive")
 def test_everything_2():
     metric1 = AnswerRelevancyMetric(threshold=0.5, strict_mode=strict_mode)
     metric2 = FaithfulnessMetric(threshold=0.5, strict_mode=strict_mode)