Merge pull request #830 from confident-ai/features/stateless-cleanup

Features/stateless cleanup

Author: penguine-ip

a.py

@@ -0,0 +1,34 @@
+import asyncio
+from deepeval import evaluate
+from deepeval.metrics import AnswerRelevancyMetric
+from deepeval.test_case import LLMTestCase
+
+# Define the test cases
+test_case1 = LLMTestCase(
+    input="The chicken crossed the road, why?",
+    actual_output="Because he felt like it",
+)
+test_case2 = LLMTestCase(
+    input="Knock knock, who's there?", actual_output="The chicken"
+)
+
+# Define the metric
+metric1 = AnswerRelevancyMetric(verbose_mode=False)
+metric2 = AnswerRelevancyMetric(verbose_mode=True)
+
+metric1.measure(test_case1)
+metric2.measure(test_case2)
+
+# # Asynchronous function to measure relevancy concurrently
+# async def measure_relevancy():
+#     await metric1.a_measure(test_case1, _show_indicator=False)
+#     print(metric1.statements)
+#     print("All measurements are done.")
+
+# # Run the asynchronous function and print after completion
+# asyncio.run(measure_relevancy())
+# print("This is printed after all asynchronous operations are complete.")
+
+
+print(metric1.statements)
+print(metric2.statements)

deepeval/evaluate.py

@@ -23,7 +23,12 @@
     ConversationalApiTestCase,
     MetricMetadata,
 )
-from deepeval.utils import get_is_running_deepeval, set_indicator, capture_contextvars, update_contextvars
+from deepeval.utils import (
+    get_is_running_deepeval,
+    set_indicator,
+    capture_contextvars,
+    update_contextvars,
+)
 from deepeval.test_run.cache import (
     test_run_cache_manager,
     Cache,
@@ -300,7 +305,7 @@ async def a_execute_test_cases(
             await measure_metrics_with_indicator(
                 metrics, test_case, cached_test_case, ignore_errors
             )
-            
+
             for metric in metrics:
                 metric_metadata = create_metric_metadata(metric)
                 if isinstance(test_case, ConversationalTestCase):

deepeval/metrics/answer_relevancy/answer_relevancy.py

@@ -2,12 +2,13 @@
 from typing import Optional, List, Union
 from pydantic import BaseModel, Field
 
-from deepeval.utils import get_or_create_event_loop
+from deepeval.utils import get_or_create_event_loop, generate_uuid
 from deepeval.metrics.utils import (
     validate_conversational_test_case,
     trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    print_intermediate_steps,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -31,6 +32,21 @@ class AnswerRelvancyVerdict(BaseModel):
 
 
 class AnswerRelevancyMetric(BaseMetric):
+    @property
+    def statements(self) -> Optional[List[str]]:
+        return self._statements.get()
+
+    @statements.setter
+    def statements(self, value: Optional[List[str]]):
+        self._statements.set(value)
+
+    @property
+    def verdicts(self) -> Optional[List[AnswerRelvancyVerdict]]:
+        return self._verdicts.get()
+
+    @verdicts.setter
+    def verdicts(self, value: Optional[List[AnswerRelvancyVerdict]]):
+        self._verdicts.set(value)
 
     def __init__(
         self,
@@ -39,35 +55,24 @@ def __init__(
         include_reason: bool = True,
         async_mode: bool = True,
         strict_mode: bool = False,
+        verbose_mode: bool = False,
     ):
-        super().__init__()
-        self._statements: ContextVar[Optional[List[str]]] = ContextVar(f'{self.__class__.__name__}_statements', default=None)
-        self._verdicts: ContextVar[Optional[List[AnswerRelvancyVerdict]]] = ContextVar(f'{self.__class__.__name__}_verdicts', default=None)
+        self._statements: ContextVar[Optional[List[str]]] = ContextVar(
+            generate_uuid(), default=None
+        )
+        self._verdicts: ContextVar[Optional[List[AnswerRelvancyVerdict]]] = (
+            ContextVar(generate_uuid(), default=None)
+        )
         self.threshold = 1 if strict_mode else threshold
         self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
         self.include_reason = include_reason
         self.async_mode = async_mode
         self.strict_mode = strict_mode
-    
-    @property
-    def statements(self) -> Optional[List[str]]:
-        return self._statements.get()
-    @statements.setter
-    def statements(self, value: Optional[List[str]]):
-        self._statements.set(value)
-        
-    @property
-    def verdicts(self) -> Optional[List[AnswerRelvancyVerdict]]:
-        return self._verdicts.get()
-    @verdicts.setter
-    def verdicts(self, value: Optional[List[AnswerRelvancyVerdict]]):
-        self._verdicts.set(value)
+        self.verbose_mode = verbose_mode
 
     def measure(
-        self, 
-        test_case: Union[LLMTestCase, ConversationalTestCase], 
-        verbose: bool = True,
+        self, test_case: Union[LLMTestCase, ConversationalTestCase]
     ) -> float:
         if isinstance(test_case, ConversationalTestCase):
             test_case = validate_conversational_test_case(test_case, self)
@@ -78,44 +83,36 @@ def measure(
             if self.async_mode:
                 loop = get_or_create_event_loop()
                 (
-                    self.statements, 
-                    self.verdicts, 
-                    self.score, 
-                    self.reason, 
-                    self.success
-                ) = loop.run_until_complete(
-                    self._measure_async(test_case, verbose)
-                )
+                    self.statements,
+                    self.verdicts,
+                    self.score,
+                    self.reason,
+                    self.success,
+                ) = loop.run_until_complete(self._measure_async(test_case))
             else:
-                self.statements = self._generate_statements(
+                self.statements: List[str] = self._generate_statements(
                     test_case.actual_output
                 )
-                self.verdicts = self._generate_verdicts(test_case.input)
+                self.verdicts: List[AnswerRelvancyVerdict] = (
+                    self._generate_verdicts(test_case.input)
+                )
                 self.score = self._calculate_score()
                 self.reason = self._generate_reason(test_case.input)
                 self.success = self.score >= self.threshold
-                if verbose:
-                    print(f"statements: {self.statements}\nverdicts: {self.verdicts}\n")                
+                if self.verbose_mode:
+                    print_intermediate_steps(
+                        self.__name__,
+                        steps=[
+                            f"Statements:\n{self.statements}\n",
+                            f"Verdicts:\n{self.verdicts}",
+                        ],
+                    )
                 return self.score
-    
-    async def _measure_async(
-            self,
-            test_case: Union[LLMTestCase, ConversationalTestCase],
-            verbose: bool):
-        await self.a_measure(test_case, _show_indicator=False, verbose=verbose)
-        return (
-            self.statements, 
-            self.verdicts, 
-            self.score, 
-            self.reason, 
-            self.success
-            )
 
     async def a_measure(
         self,
         test_case: Union[LLMTestCase, ConversationalTestCase],
         _show_indicator: bool = True,
-        verbose: bool = True
     ) -> float:
         if isinstance(test_case, ConversationalTestCase):
             test_case = validate_conversational_test_case(test_case, self)
@@ -125,19 +122,37 @@ async def a_measure(
         with metric_progress_indicator(
             self, async_mode=True, _show_indicator=_show_indicator
         ):
-            self.statements = await self._a_generate_statements(
+            self.statements: List[str] = await self._a_generate_statements(
                 test_case.actual_output
             )
-            self.verdicts = (
+            self.verdicts: List[AnswerRelvancyVerdict] = (
                 await self._a_generate_verdicts(test_case.input)
             )
             self.score = self._calculate_score()
             self.reason = await self._a_generate_reason(test_case.input)
             self.success = self.score >= self.threshold
-            if verbose:
-                print(f"statements: {self.statements}\nverdicts: {self.verdicts}\nscore: {self.score}, success: {self.success}\n")
+            if self.verbose_mode:
+                print_intermediate_steps(
+                    self.__name__,
+                    steps=[
+                        f"Statements:\n{self.statements}\n",
+                        f"Verdicts:\n{self.verdicts}",
+                    ],
+                )
             return self.score
 
+    async def _measure_async(
+        self, test_case: Union[LLMTestCase, ConversationalTestCase]
+    ):
+        await self.a_measure(test_case, _show_indicator=False)
+        return (
+            self.statements,
+            self.verdicts,
+            self.score,
+            self.reason,
+            self.success,
+        )
+
     async def _a_generate_reason(self, input: str) -> str:
         if self.include_reason is False:
             return None
@@ -275,4 +290,4 @@ def is_successful(self) -> bool:
 
     @property
     def __name__(self):
-        return "Answer Relevancy"
+        return "Answer Relevancy"