Merge pull request #50 from athina-ai/vivek/ath-1124-bug-fixes-in-saf…

…ety-evals Vivek/ath 1124 bug fixes in safety evals
athina-ai · Apr 16, 2024 · b66c193 · b66c193
2 parents a3aeb30 + 74dde0f
commit b66c193
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 6 deletions.
diff --git a/athina/evals/eval_type.py b/athina/evals/eval_type.py
@@ -49,6 +49,8 @@ class FunctionEvalTypeId(Enum):
     NOT_GIBBERISH_TEXT = "NotGibberishText"
     CONTAINS_NO_SENSITIVE_TOPICS = "ContainsNoSensitiveTopics"
     OPENAI_CONTENT_MODERATION = "OpenAiContentModeration"
+    PII_DETECTION = "PiiDetection"
+    PROMPT_INJECTION= "PromptInjection"
 
 class GroundedEvalTypeId(Enum):
     ANSWER_SIMILARITY = "AnswerSimilarity"

diff --git a/athina/evals/safety/pii_detection/evaluator.py b/athina/evals/safety/pii_detection/evaluator.py
@@ -110,4 +110,6 @@ def detect_pii(self, text: str):
         else:
             raise Exception(f"Error occurred during PII detection: {response.text}")
 
+        if not result["pii_detected"]:
+            result["reason"] = "No PII detected"
         return result
diff --git a/athina/helpers/get_evaluator.py b/athina/helpers/get_evaluator.py
@@ -1,4 +1,4 @@
-from athina.evals import Regex, ContainsAny, ContainsAll, Contains, ContainsNone, ContainsJson, ContainsEmail, IsJson, IsEmail, NoInvalidLinks, ContainsLink, ContainsValidLink, Equals, StartsWith, EndsWith, LengthLessThan, LengthGreaterThan, ApiCall, DoesResponseAnswerQuery, Faithfulness, BaseEvaluator, ContextContainsEnoughInformation, SummaryAccuracy, Groundedness, GradingCriteria, CustomPrompt, RagasContextRelevancy, RagasAnswerRelevancy, RagasAnswerCorrectness, RagasAnswerSemanticSimilarity, RagasCoherence, RagasConciseness, RagasContextPrecision, RagasContextRecall, RagasFaithfulness, RagasHarmfulness, RagasMaliciousness, NotGibberishText, SafeForWorkText, ContainsNoSensitiveTopics
+from athina.evals import Regex, ContainsAny, ContainsAll, Contains, ContainsNone, ContainsJson, ContainsEmail, IsJson, IsEmail, NoInvalidLinks, ContainsLink, ContainsValidLink, Equals, StartsWith, EndsWith, LengthLessThan, LengthGreaterThan, ApiCall, DoesResponseAnswerQuery, Faithfulness, BaseEvaluator, ContextContainsEnoughInformation, SummaryAccuracy, Groundedness, GradingCriteria, CustomPrompt, RagasContextRelevancy, RagasAnswerRelevancy, RagasAnswerCorrectness, RagasAnswerSemanticSimilarity, RagasCoherence, RagasConciseness, RagasContextPrecision, RagasContextRecall, RagasFaithfulness, RagasHarmfulness, RagasMaliciousness, NotGibberishText, SafeForWorkText, ContainsNoSensitiveTopics, OpenAiContentModeration, PiiDetection, PromptInjection
 from athina.evals.grounded.similarity import CosineSimilarity, JaccardSimilarity, JaroWincklerSimilarity, NormalisedLevenshteinSimilarity, SorensenDiceSimilarity
 from athina.evals.grounded.wrapper import AnswerSimilarity, ContextSimilarity
 
@@ -32,6 +32,9 @@
     "SafeForWorkText": SafeForWorkText,
     "NotGibberishText": NotGibberishText,
     "ContainsNoSensitiveTopics": ContainsNoSensitiveTopics,
+    "OpenAiContentModeration" : OpenAiContentModeration,
+    "PiiDetection": PiiDetection,
+    "PromptInjection": PromptInjection
 }
 
 llm_operations = {

diff --git a/athina/scripts/guardrails.py b/athina/scripts/guardrails.py
@@ -5,22 +5,30 @@ def install():
         try:
             from guardrails import Guard
         except Exception as e:
-            subprocess.call(["pip", "install", "-U", "guardrails-ai==0.4.2"])
+            result = subprocess.call(["pip", "install", "-U", "guardrails-ai==0.4.2"])
+            if result != 0:
+                print("Guardrails installation failed")
 
         try:
             from guardrails.hub import GibberishText
         except Exception as e:
-            subprocess.call(["guardrails", "hub", "install", "hub://guardrails/gibberish_text"])
+            result = subprocess.call(["guardrails", "hub", "install", "hub://guardrails/gibberish_text"])
+            if result != 0:
+                print("Guardrails installation failed. Ensure have the latest version of pip installed")
 
         try:
             from guardrails.hub import SensitiveTopic
         except Exception as e:
-            subprocess.call(["guardrails", "hub", "install", "hub://guardrails/sensitive_topics"])
+            result = subprocess.call(["guardrails", "hub", "install", "hub://guardrails/sensitive_topics"])
+            if result != 0:
+                print("Guardrails installation failed. Ensure have the latest version of pip installed")
 
         try:
             from guardrails.hub import NSFWText
         except Exception as e:
-            subprocess.call(["guardrails", "hub", "install", "hub://guardrails/nsfw_text"])
+            result = subprocess.call(["guardrails", "hub", "install", "hub://guardrails/nsfw_text"])
+            if result != 0:
+                print("Guardrails installation failed. Ensure have the latest version of pip installed")
 
         try:
             import nltk

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "athina"
-version = "1.2.11"
+version = "1.2.12"
 description = "Python SDK to configure and run evaluations for your LLM-based application"
 authors = ["Shiv Sakhuja <[email protected]>", "Akshat Gupta <[email protected]>", "Vivek Aditya <[email protected]>", "Akhil Bisht <[email protected]>"]
 readme = "README.md"