Merge pull request #841 from confident-ai/hotfix/async-fix

.

Author: penguine-ip

deepeval/metrics/answer_relevancy/answer_relevancy.py

@@ -61,6 +61,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._statements: ContextVar[Optional[List[str]]] = ContextVar(
             generate_uuid(), default=None
         )

deepeval/metrics/base_metric.py

@@ -7,21 +7,7 @@
 from deepeval.utils import generate_uuid
 
 
-class MetricMeta(type):
-    def __new__(cls, name, bases, attrs):
-        # Create the new class
-        new_class = super().__new__(cls, name, bases, attrs)
-        # Assign new ContextVar instances with unique identifiers
-        new_class._score = ContextVar(str(uuid.uuid4()), default=None)
-        new_class._score_breakdown = ContextVar(str(uuid.uuid4()), default=None)
-        new_class._reason = ContextVar(str(uuid.uuid4()), default=None)
-        new_class._success = ContextVar(str(uuid.uuid4()), default=None)
-        new_class._error = ContextVar(str(uuid.uuid4()), default=None)
-        return new_class
-
-
-class BaseMetric(metaclass=MetricMeta):
-
+class BaseMetric:
     evaluation_model: Optional[str] = None
     strict_mode: bool = False
     async_mode: bool = True
@@ -85,6 +71,13 @@ def threshold(self) -> float:
     def threshold(self, value: float):
         self._threshold = value
 
+    def __init__(self):
+        self._score = ContextVar(str(uuid.uuid4()), default=None)
+        self._score_breakdown = ContextVar(str(uuid.uuid4()), default=None)
+        self._reason = ContextVar(str(uuid.uuid4()), default=None)
+        self._success = ContextVar(str(uuid.uuid4()), default=None)
+        self._error = ContextVar(str(uuid.uuid4()), default=None)
+
     @abstractmethod
     def measure(self, test_case: LLMTestCase, *args, **kwargs) -> float:
         raise NotImplementedError

deepeval/metrics/bias/bias.py

@@ -63,6 +63,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._opinions: ContextVar[Optional[List[str]]] = ContextVar(
             generate_uuid(), default=None
         )

deepeval/metrics/contextual_precision/contextual_precision.py

@@ -58,6 +58,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._verdicts: ContextVar[
             Optional[List[ContextualPrecisionVerdict]]
         ] = ContextVar(generate_uuid(), default=None)

deepeval/metrics/contextual_recall/contextual_recall.py

@@ -55,6 +55,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._verdicts: ContextVar[Optional[List[ContextualRecallVerdict]]] = (
             ContextVar(generate_uuid(), default=None)
         )

deepeval/metrics/contextual_relevancy/contextual_relevancy.py

@@ -57,6 +57,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._verdicts: ContextVar[
             Optional[List[ContextualRelevancyVerdict]]
         ] = ContextVar(generate_uuid(), default=None)

deepeval/metrics/faithfulness/faithfulness.py

@@ -72,6 +72,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._truths: ContextVar[Optional[List[str]]] = ContextVar(
             generate_uuid(), default=None
         )

deepeval/metrics/g_eval/g_eval.py

@@ -80,6 +80,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._evaluation_steps: ContextVar[Optional[List[str]]] = ContextVar(
             generate_uuid(), default=None
         )

deepeval/metrics/hallucination/hallucination.py

@@ -55,6 +55,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._verdicts: ContextVar[Optional[List[HallucinationVerdict]]] = (
             ContextVar(generate_uuid(), default=None)
         )

deepeval/metrics/summarization/summarization.py

@@ -117,6 +117,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._truths: ContextVar[Optional[List[str]]] = ContextVar(
             generate_uuid(), default=None
         )

deepeval/metrics/toxicity/toxicity.py

@@ -63,6 +63,7 @@ def __init__(
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        super().__init__()
         self._opinions: ContextVar[Optional[List[str]]] = ContextVar(
             generate_uuid(), default=None
         )

tests/test_everything.py

@@ -128,6 +128,18 @@ def test_everything():
         verbose_mode=verbose_mode,
     )
 
+    metric11 = GEval(
+        name="Relevancy",
+        criteria="Relevancy - determine if the actual output is relevant with the input.",
+        evaluation_params=[
+            LLMTestCaseParams.INPUT,
+            LLMTestCaseParams.ACTUAL_OUTPUT,
+        ],
+        strict_mode=strict_mode,
+        model="gpt-4-0125-preview",
+        verbose_mode=verbose_mode,
+    )
+
     test_case = LLMTestCase(
         input="What is this",
         actual_output="this is a latte",
@@ -140,15 +152,16 @@ def test_everything():
         test_case,
         [
             metric1,
-            metric2,
-            metric3,
-            metric4,
-            metric5,
+            # metric2,
+            # metric3,
+            # metric4,
+            # metric5,
             # metric6,
             # metric7,
             # metric8,
             # metric9,
-            # metric10,
+            metric10,
+            metric11,
         ],
         # run_async=False,
     )