diff --git a/pyrit/scenario/scenarios/airt/content_harms.py b/pyrit/scenario/scenarios/airt/content_harms.py index d2fb4055f6..6b806cd109 100644 --- a/pyrit/scenario/scenarios/airt/content_harms.py +++ b/pyrit/scenario/scenarios/airt/content_harms.py @@ -104,7 +104,7 @@ class ContentHarms(Scenario): with respect to certain harm categories. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: @@ -179,7 +179,7 @@ def __init__( super().__init__( name="Content Harms", - version=self.version, + version=self.VERSION, objective_scorer=self._objective_scorer, strategy_class=ContentHarmsStrategy, scenario_result_id=scenario_result_id, @@ -239,9 +239,15 @@ def _get_strategy_attacks( Returns: List[AtomicAttack]: The constructed AtomicAttack instances for each attack type. + + Raises: + ValueError: If scenario is not properly initialized. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if self._objective_target is None: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) prompt_sending_attack = PromptSendingAttack( objective_target=self._objective_target, diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index 220fac3f74..ea8ddf41df 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -60,7 +60,7 @@ class Cyber(Scenario): techniques. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -141,7 +141,7 @@ def __init__( super().__init__( name="Cyber", - version=self.version, + version=self.VERSION, strategy_class=CyberStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -241,10 +241,13 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: AtomicAttack: configured for the specified strategy. Raises: - ValueError: if an unknown CyberStrategy is passed. + ValueError: If scenario is not properly initialized or an unknown CyberStrategy is passed. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if self._objective_target is None: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) attack_strategy: Optional[AttackStrategy[Any, Any]] = None if strategy == "single_turn": attack_strategy = PromptSendingAttack( @@ -261,7 +264,8 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: raise ValueError(f"Unknown CyberStrategy: {strategy}") # _seed_groups is guaranteed to be set by _get_atomic_attacks_async before this method is called - assert self._seed_groups is not None, "_seed_groups must be resolved before creating atomic attacks" + if self._seed_groups is None: + raise ValueError("_seed_groups must be resolved before creating atomic attacks") return AtomicAttack( atomic_attack_name=f"cyber_{strategy}", diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index e28676db72..0478b4ea94 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -49,7 +49,7 @@ class Jailbreak(Scenario): scored to determine if the jailbreak was successful. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -114,7 +114,7 @@ def __init__( super().__init__( name="Jailbreak", - version=self.version, + version=self.VERSION, strategy_class=JailbreakStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -182,9 +182,15 @@ async def _get_atomic_attack_from_jailbreak_async(self, *, jailbreak_template_na Returns: AtomicAttack: An atomic attack using the specified jailbreak template. + + Raises: + ValueError: If scenario is not properly initialized. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if self._objective_target is None: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) # Create the jailbreak converter jailbreak_converter = TextJailbreakConverter( diff --git a/pyrit/scenario/scenarios/airt/leakage_scenario.py b/pyrit/scenario/scenarios/airt/leakage_scenario.py index 5137ef2eb7..0d70f9d9bb 100644 --- a/pyrit/scenario/scenarios/airt/leakage_scenario.py +++ b/pyrit/scenario/scenarios/airt/leakage_scenario.py @@ -89,7 +89,7 @@ class LeakageScenario(Scenario): attack variations designed to extract sensitive information from models. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -161,7 +161,7 @@ def __init__( super().__init__( name="Leakage Scenario", - version=self.version, + version=self.VERSION, strategy_class=LeakageStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -261,10 +261,13 @@ async def _get_atomic_attack_from_strategy_async(self, strategy: str) -> AtomicA AtomicAttack: Configured for the specified strategy. Raises: - ValueError: If an unknown LeakageStrategy is passed. + ValueError: If scenario is not properly initialized or an unknown LeakageStrategy is passed. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if self._objective_target is None: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) strategy_factories = { "first_letter": self._create_first_letter_attack, diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index e29204a715..2afc2e4b29 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -89,7 +89,7 @@ class Scam(Scenario): (e.g., phishing emails, fraudulent messages) with primarily persuasion-oriented techniques. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -167,7 +167,7 @@ def __init__( super().__init__( name="Scam", - version=self.version, + version=self.VERSION, strategy_class=ScamStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -266,10 +266,13 @@ def _get_atomic_attack_from_strategy(self, strategy: str) -> AtomicAttack: AtomicAttack: Configured for the specified strategy. Raises: - ValueError: If an unknown ScamStrategy is provided. + ValueError: If scenario is not properly initialized or an unknown ScamStrategy is provided. """ # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if self._objective_target is None: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) attack_strategy: Optional[AttackStrategy[Any, Any]] = None if strategy == "persuasive_rta": diff --git a/pyrit/scenario/scenarios/foundry/red_team_agent.py b/pyrit/scenario/scenarios/foundry/red_team_agent.py index 04ba25272d..b85242bcd1 100644 --- a/pyrit/scenario/scenarios/foundry/red_team_agent.py +++ b/pyrit/scenario/scenarios/foundry/red_team_agent.py @@ -214,7 +214,7 @@ class RedTeamAgent(Scenario): providing a consistent PyRIT contract for their integration. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> Type[ScenarioStrategy]: @@ -297,7 +297,7 @@ def __init__( # Call super().__init__() first to initialize self._memory super().__init__( name="RedTeamAgent", - version=self.version, + version=self.VERSION, strategy_class=FoundryStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, diff --git a/pyrit/scenario/scenarios/garak/encoding.py b/pyrit/scenario/scenarios/garak/encoding.py index 712d2efa6c..b4b36e3e8a 100644 --- a/pyrit/scenario/scenarios/garak/encoding.py +++ b/pyrit/scenario/scenarios/garak/encoding.py @@ -133,7 +133,7 @@ class Encoding(Scenario): By default, this uses the same dataset as Garak: slur terms and web XSS payloads. """ - version: int = 1 + VERSION: int = 1 @classmethod def get_strategy_class(cls) -> type[ScenarioStrategy]: @@ -209,7 +209,7 @@ def __init__( super().__init__( name="Encoding", - version=self.version, + version=self.VERSION, strategy_class=EncodingStrategy, objective_scorer=objective_scorer, include_default_baseline=include_baseline, @@ -332,6 +332,9 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam Returns: list[AtomicAttack]: List of atomic attacks for this encoding scheme. + + Raises: + ValueError: If scenario is not properly initialized. """ converter_configs = [ AttackConverterConfig( @@ -351,7 +354,10 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam atomic_attacks = [] for attack_converter_config in converter_configs: # objective_target is guaranteed to be non-None by parent class validation - assert self._objective_target is not None + if self._objective_target is None: + raise ValueError( + "Scenario not properly initialized. Call await scenario.initialize_async() before running." + ) attack = PromptSendingAttack( objective_target=self._objective_target, attack_converter_config=attack_converter_config, diff --git a/pyrit/score/conversation_scorer.py b/pyrit/score/conversation_scorer.py index 1b93088cb7..a390a0df2b 100644 --- a/pyrit/score/conversation_scorer.py +++ b/pyrit/score/conversation_scorer.py @@ -28,7 +28,7 @@ class ConversationScorer(Scorer, ABC): Note: This class cannot be instantiated directly. Use create_conversation_scorer() factory instead. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], enforce_all_pieces_valid=True, ) @@ -189,7 +189,7 @@ class DynamicConversationScorer(ConversationScorer, scorer_base_class): # type: def __init__(self) -> None: # Initialize with the validator and wrapped scorer - Scorer.__init__(self, validator=validator or ConversationScorer._default_validator) + Scorer.__init__(self, validator=validator or ConversationScorer._DEFAULT_VALIDATOR) self._wrapped_scorer = scorer def _get_wrapped_scorer(self) -> Scorer: diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 9b71f20a31..e40303eb0f 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -47,7 +47,7 @@ class AzureContentFilterScorer(FloatScaleScorer): MAX_TEXT_LENGTH = 10000 # Azure Content Safety API limit - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text", "image_path"], ) @@ -140,7 +140,7 @@ def __init__( else: raise ValueError("Please provide the Azure Content Safety endpoint and api_key") - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) @property def _category_values(self) -> list[str]: diff --git a/pyrit/score/float_scale/insecure_code_scorer.py b/pyrit/score/float_scale/insecure_code_scorer.py index 6086493d38..5ab98db2fa 100644 --- a/pyrit/score/float_scale/insecure_code_scorer.py +++ b/pyrit/score/float_scale/insecure_code_scorer.py @@ -20,7 +20,7 @@ class InsecureCodeScorer(FloatScaleScorer): Configuration is loaded from a YAML file for dynamic prompts and instructions. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -38,7 +38,7 @@ def __init__( Defaults to the default insecure code scoring prompt if not provided. validator (Optional[ScorerPromptValidator]): Custom validator for the scorer. Defaults to None. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index f1a36cd13b..1423654bf4 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -32,7 +32,7 @@ class PlagiarismScorer(FloatScaleScorer): 3. Word-level n-gram Jaccard similarity """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -50,7 +50,7 @@ def __init__( n (int): The n-gram size for n-gram similarity. Defaults to 5. validator (Optional[ScorerPromptValidator]): Custom validator for the scorer. Defaults to None. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self.reference_text = reference_text self.metric = metric diff --git a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py index e1d4829289..50f566effa 100644 --- a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py @@ -18,7 +18,7 @@ class SelfAskGeneralFloatScaleScorer(FloatScaleScorer): system prompt and prompt format. The final score is normalized to [0, 1]. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], is_objective_required=True, ) @@ -69,7 +69,7 @@ def __init__( ValueError: If system_prompt_format_string is not provided or empty. ValueError: If min_value is greater than max_value. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target if not system_prompt_format_string: raise ValueError("system_prompt_format_string must be provided and non-empty.") diff --git a/pyrit/score/float_scale/self_ask_likert_scorer.py b/pyrit/score/float_scale/self_ask_likert_scorer.py index c01fe65073..763484ab03 100644 --- a/pyrit/score/float_scale/self_ask_likert_scorer.py +++ b/pyrit/score/float_scale/self_ask_likert_scorer.py @@ -153,7 +153,7 @@ class SelfAskLikertScorer(FloatScaleScorer): A class that represents a "self-ask" score for text scoring for a likert scale. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -170,7 +170,7 @@ def __init__( likert_scale (LikertScalePaths): The Likert scale configuration to use for scoring. validator (Optional[ScorerPromptValidator]): Custom validator for the scorer. Defaults to None. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target self._likert_scale = likert_scale diff --git a/pyrit/score/float_scale/self_ask_scale_scorer.py b/pyrit/score/float_scale/self_ask_scale_scorer.py index 5e502681d0..05f954c3f0 100644 --- a/pyrit/score/float_scale/self_ask_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_scale_scorer.py @@ -35,7 +35,7 @@ class SystemPaths(enum.Enum): RED_TEAMER_SYSTEM_PROMPT = Path(SCORER_SCALES_PATH, "red_teamer_system_prompt.yaml").resolve() CRITERIA_SYSTEM_PROMPT = Path(SCORER_SCALES_PATH, "criteria_system_prompt.yaml").resolve() - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], is_objective_required=True, ) @@ -59,7 +59,7 @@ def __init__( Defaults to GENERAL_SYSTEM_PROMPT if not provided. validator (Optional[ScorerPromptValidator]): Custom validator for the scorer. Defaults to None. """ - super().__init__(validator=validator or self._default_validator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target diff --git a/pyrit/score/float_scale/video_float_scale_scorer.py b/pyrit/score/float_scale/video_float_scale_scorer.py index 669ee3a3b8..fc4ff9454f 100644 --- a/pyrit/score/float_scale/video_float_scale_scorer.py +++ b/pyrit/score/float_scale/video_float_scale_scorer.py @@ -36,7 +36,7 @@ class VideoFloatScaleScorer( the audio is extracted, transcribed, and scored. The audio scores are included in the aggregation. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["video_path"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["video_path"]) def __init__( self, @@ -78,7 +78,7 @@ def __init__( Raises: ValueError: If audio_scorer is provided and does not support audio_path data type. """ - FloatScaleScorer.__init__(self, validator=validator or self._default_validator) + FloatScaleScorer.__init__(self, validator=validator or self._DEFAULT_VALIDATOR) _BaseVideoScorer.__init__( self, diff --git a/pyrit/score/human/human_in_the_loop_gradio.py b/pyrit/score/human/human_in_the_loop_gradio.py index f9b3226ba9..05031bd9bb 100644 --- a/pyrit/score/human/human_in_the_loop_gradio.py +++ b/pyrit/score/human/human_in_the_loop_gradio.py @@ -21,7 +21,7 @@ class HumanInTheLoopScorerGradio(TrueFalseScorer): In the future this will not be a TrueFalseScorer. However, it is all that is supported currently. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -43,7 +43,7 @@ def __init__( # Import here to avoid importing rpyc in the main module that might not be installed from pyrit.ui.rpc import AppRPCServer - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) self._rpc_server = AppRPCServer(open_browser=open_browser) self._rpc_server.start() diff --git a/pyrit/score/true_false/decoding_scorer.py b/pyrit/score/true_false/decoding_scorer.py index 8034f5a000..17d320fe9c 100644 --- a/pyrit/score/true_false/decoding_scorer.py +++ b/pyrit/score/true_false/decoding_scorer.py @@ -24,7 +24,7 @@ class DecodingScorer(TrueFalseScorer): text matching strategy. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], supported_roles=["assistant"] ) @@ -50,7 +50,7 @@ def __init__( self._text_matcher = text_matcher if text_matcher else ExactTextMatching(case_sensitive=False) self._score_categories = categories if categories else [] - super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) + super().__init__(score_aggregator=aggregator, validator=validator or self._DEFAULT_VALIDATOR) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/gandalf_scorer.py b/pyrit/score/true_false/gandalf_scorer.py index 932bf49dfe..a4d5cf345b 100644 --- a/pyrit/score/true_false/gandalf_scorer.py +++ b/pyrit/score/true_false/gandalf_scorer.py @@ -29,7 +29,7 @@ class GandalfScorer(TrueFalseScorer): if the password is correct. Returns True if the password was successfully extracted. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -49,7 +49,7 @@ def __init__( score_aggregator (TrueFalseAggregatorFunc): Aggregator for combining scores. Defaults to TrueFalseScoreAggregator.OR. """ - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) self._prompt_target = chat_target self._defender = level.value diff --git a/pyrit/score/true_false/markdown_injection.py b/pyrit/score/true_false/markdown_injection.py index f45fc90f14..f4e87871fd 100644 --- a/pyrit/score/true_false/markdown_injection.py +++ b/pyrit/score/true_false/markdown_injection.py @@ -23,7 +23,7 @@ class MarkdownInjectionScorer(TrueFalseScorer): might be exploited. Returns True if markdown injection is detected. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -41,7 +41,7 @@ def __init__( """ self._category = "security" - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/prompt_shield_scorer.py b/pyrit/score/true_false/prompt_shield_scorer.py index edfc697b1d..d8069e44e0 100644 --- a/pyrit/score/true_false/prompt_shield_scorer.py +++ b/pyrit/score/true_false/prompt_shield_scorer.py @@ -27,7 +27,7 @@ class PromptShieldScorer(TrueFalseScorer): scorer_type: ScoreType _prompt_shield_target: PromptShieldTarget - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -47,7 +47,7 @@ def __init__( """ self._prompt_target = prompt_shield_target - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/question_answer_scorer.py b/pyrit/score/true_false/question_answer_scorer.py index 63ba8c6c17..1717c8c44f 100644 --- a/pyrit/score/true_false/question_answer_scorer.py +++ b/pyrit/score/true_false/question_answer_scorer.py @@ -22,7 +22,7 @@ class QuestionAnswerScorer(TrueFalseScorer): CORRECT_ANSWER_MATCHING_PATTERNS = ["{correct_answer_index}:", "{correct_answer}"] - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], required_metadata=["correct_answer_index", "correct_answer"] ) @@ -49,7 +49,7 @@ def __init__( self._correct_answer_matching_patterns = correct_answer_matching_patterns self._score_category = category if category is not None else [] - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/self_ask_category_scorer.py b/pyrit/score/true_false/self_ask_category_scorer.py index 5697d17c51..9c5b1dd930 100644 --- a/pyrit/score/true_false/self_ask_category_scorer.py +++ b/pyrit/score/true_false/self_ask_category_scorer.py @@ -36,7 +36,7 @@ class SelfAskCategoryScorer(TrueFalseScorer): There is also a false category that is used if the MessagePiece does not fit any of the categories. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator() + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator() def __init__( self, @@ -56,7 +56,7 @@ def __init__( Defaults to TrueFalseScoreAggregator.OR. validator (Optional[ScorerPromptValidator]): Custom validator. Defaults to None. """ - super().__init__(score_aggregator=score_aggregator, validator=validator or self._default_validator) + super().__init__(score_aggregator=score_aggregator, validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target diff --git a/pyrit/score/true_false/self_ask_general_true_false_scorer.py b/pyrit/score/true_false/self_ask_general_true_false_scorer.py index edc1cc5c08..2de626e2a1 100644 --- a/pyrit/score/true_false/self_ask_general_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_general_true_false_scorer.py @@ -22,7 +22,7 @@ class SelfAskGeneralTrueFalseScorer(TrueFalseScorer): system prompt and prompt format. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], is_objective_required=False, ) @@ -71,7 +71,7 @@ def __init__( Raises: ValueError: If system_prompt_format_string is not provided or empty. """ - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) self._prompt_target = chat_target if not system_prompt_format_string: raise ValueError("system_prompt_format_string must be provided and non-empty.") diff --git a/pyrit/score/true_false/self_ask_question_answer_scorer.py b/pyrit/score/true_false/self_ask_question_answer_scorer.py index 0cb0e704db..93a1dcfcc6 100644 --- a/pyrit/score/true_false/self_ask_question_answer_scorer.py +++ b/pyrit/score/true_false/self_ask_question_answer_scorer.py @@ -26,7 +26,7 @@ class SelfAskQuestionAnswerScorer(SelfAskTrueFalseScorer): to objective target or you need more flexibility in determining if the questions were answered correctly. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text"], is_objective_required=True, ) diff --git a/pyrit/score/true_false/self_ask_refusal_scorer.py b/pyrit/score/true_false/self_ask_refusal_scorer.py index 4341bdc33f..70b7274bed 100644 --- a/pyrit/score/true_false/self_ask_refusal_scorer.py +++ b/pyrit/score/true_false/self_ask_refusal_scorer.py @@ -32,7 +32,7 @@ class SelfAskRefusalScorer(TrueFalseScorer): evaluating whether AI systems are appropriately refusing harmful requests. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator() + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator() def __init__( self, @@ -60,7 +60,7 @@ def __init__( result_file="refusal_scorer/refusal_metrics.jsonl", ) - super().__init__(score_aggregator=score_aggregator, validator=validator or self._default_validator) + super().__init__(score_aggregator=score_aggregator, validator=validator or self._DEFAULT_VALIDATOR) self._prompt_target = chat_target self._system_prompt_with_objective = ( diff --git a/pyrit/score/true_false/self_ask_true_false_scorer.py b/pyrit/score/true_false/self_ask_true_false_scorer.py index 45f323d0ca..80cb967892 100644 --- a/pyrit/score/true_false/self_ask_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_true_false_scorer.py @@ -87,7 +87,7 @@ class SelfAskTrueFalseScorer(TrueFalseScorer): If no descriptions are provided, it defaults to the TASK_ACHIEVED scorer. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator( + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator( supported_data_types=["text", "image_path"], ) @@ -117,7 +117,7 @@ def __init__( ValueError: If both true_false_question_path and true_false_question are provided. ValueError: If required keys are missing in true_false_question. """ - super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) + super().__init__(validator=validator or self._DEFAULT_VALIDATOR, score_aggregator=score_aggregator) self._prompt_target = chat_target diff --git a/pyrit/score/true_false/substring_scorer.py b/pyrit/score/true_false/substring_scorer.py index e4548378b1..59607c07de 100644 --- a/pyrit/score/true_false/substring_scorer.py +++ b/pyrit/score/true_false/substring_scorer.py @@ -22,7 +22,7 @@ class SubStringScorer(TrueFalseScorer): Supports both exact substring matching and approximate matching. """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) def __init__( self, @@ -49,7 +49,7 @@ def __init__( self._text_matcher = text_matcher if text_matcher else ExactTextMatching(case_sensitive=False) self._score_categories = categories if categories else [] - super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) + super().__init__(score_aggregator=aggregator, validator=validator or self._DEFAULT_VALIDATOR) def _build_identifier(self) -> ScorerIdentifier: """ diff --git a/pyrit/score/true_false/video_true_false_scorer.py b/pyrit/score/true_false/video_true_false_scorer.py index c06891795d..fc312d4512 100644 --- a/pyrit/score/true_false/video_true_false_scorer.py +++ b/pyrit/score/true_false/video_true_false_scorer.py @@ -25,7 +25,7 @@ class VideoTrueFalseScorer(TrueFalseScorer, _BaseVideoScorer): - Video + Audio scoring: True only if both video frames AND audio transcript match their objectives """ - _default_validator: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["video_path"]) + _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["video_path"]) def __init__( self, @@ -67,7 +67,7 @@ def __init__( audio_objective_template=audio_objective_template, ) - TrueFalseScorer.__init__(self, validator=validator or self._default_validator) + TrueFalseScorer.__init__(self, validator=validator or self._DEFAULT_VALIDATOR) if audio_scorer is not None: self._validate_audio_scorer(audio_scorer) diff --git a/tests/unit/scenarios/test_content_harms.py b/tests/unit/scenarios/test_content_harms.py index bfd0fbf8cd..d0725beb18 100644 --- a/tests/unit/scenarios/test_content_harms.py +++ b/tests/unit/scenarios/test_content_harms.py @@ -242,7 +242,7 @@ async def test_initialization_with_minimal_parameters( # Constructor should set adversarial chat and basic metadata assert scenario._adversarial_chat == mock_adversarial_target assert scenario.name == "Content Harms" - assert scenario.version == 1 + assert scenario.VERSION == 1 # Initialization populates objective target and scenario composites await scenario.initialize_async(objective_target=mock_objective_target) @@ -391,7 +391,7 @@ def test_get_default_scorer(self, mock_objective_target): def test_scenario_version(self): """Test that scenario has correct version.""" - assert ContentHarms.version == 1 + assert ContentHarms.VERSION == 1 @patch.dict( "os.environ", diff --git a/tests/unit/scenarios/test_cyber.py b/tests/unit/scenarios/test_cyber.py index 37eed916d4..5be36d7da2 100644 --- a/tests/unit/scenarios/test_cyber.py +++ b/tests/unit/scenarios/test_cyber.py @@ -140,7 +140,7 @@ def test_init_with_custom_objectives(self, mock_objective_scorer, sample_objecti # objectives are stored as _deprecated_objectives; _seed_groups is resolved lazily assert scenario._deprecated_objectives == sample_objectives assert scenario.name == "Cyber" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_objectives(self, mock_objective_scorer, malware_prompts, mock_memory_seed_groups): """Test initialization with default objectives.""" @@ -151,7 +151,7 @@ def test_init_with_default_objectives(self, mock_objective_scorer, malware_promp # seed_groups are resolved lazily; _deprecated_objectives should be None assert scenario._deprecated_objectives is None assert scenario.name == "Cyber" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_scorer(self, mock_memory_seed_groups): """Test initialization with default scorer.""" @@ -351,7 +351,7 @@ def test_scenario_version_is_set(self, mock_objective_scorer, mock_memory_seed_g objective_scorer=mock_objective_scorer, ) - assert scenario.version == 1 + assert scenario.VERSION == 1 @pytest.mark.asyncio async def test_no_target_duplication(self, mock_objective_target, mock_memory_seed_groups, mock_dataset_config): diff --git a/tests/unit/scenarios/test_encoding.py b/tests/unit/scenarios/test_encoding.py index 699304aa14..5a2aa5be3e 100644 --- a/tests/unit/scenarios/test_encoding.py +++ b/tests/unit/scenarios/test_encoding.py @@ -108,7 +108,7 @@ def test_init_with_custom_seed_prompts(self, mock_objective_target, mock_objecti assert scenario._deprecated_seed_prompts == sample_seeds assert scenario.name == "Encoding" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_seed_prompts(self, mock_objective_target, mock_objective_scorer, mock_memory_seeds): """Test initialization with default seed prompts (Garak dataset).""" diff --git a/tests/unit/scenarios/test_foundry.py b/tests/unit/scenarios/test_foundry.py index 59bcca446b..b093b72a46 100644 --- a/tests/unit/scenarios/test_foundry.py +++ b/tests/unit/scenarios/test_foundry.py @@ -732,7 +732,7 @@ def test_scenario_version_is_set(self, mock_objective_target, mock_objective_sco attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), ) - assert scenario.version == 1 + assert scenario.VERSION == 1 @patch.dict( "os.environ", diff --git a/tests/unit/scenarios/test_jailbreak.py b/tests/unit/scenarios/test_jailbreak.py index 047334131c..88d2aabd15 100644 --- a/tests/unit/scenarios/test_jailbreak.py +++ b/tests/unit/scenarios/test_jailbreak.py @@ -259,7 +259,7 @@ def test_scenario_version_is_set( objective_scorer=mock_objective_scorer, ) - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_scenario_default_dataset(self) -> None: """Test that scenario default dataset is correct.""" diff --git a/tests/unit/scenarios/test_leakage_scenario.py b/tests/unit/scenarios/test_leakage_scenario.py index 29ec0b0bf6..f341e7929e 100644 --- a/tests/unit/scenarios/test_leakage_scenario.py +++ b/tests/unit/scenarios/test_leakage_scenario.py @@ -144,7 +144,7 @@ def test_init_with_custom_objectives(self, mock_objective_scorer, sample_objecti assert len(scenario._objectives) == len(sample_objectives) assert scenario.name == "Leakage Scenario" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_objectives(self, mock_objective_scorer, leakage_prompts, mock_memory_seeds): """Test initialization with default objectives.""" @@ -153,7 +153,7 @@ def test_init_with_default_objectives(self, mock_objective_scorer, leakage_promp assert scenario._objectives == leakage_prompts assert scenario.name == "Leakage Scenario" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_scorer(self, mock_memory_seeds): """Test initialization with default scorer.""" @@ -434,7 +434,7 @@ def test_scenario_version_is_set(self, mock_objective_scorer, sample_objectives) objective_scorer=mock_objective_scorer, ) - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_get_strategy_class_returns_leakage_strategy(self): """Test that get_strategy_class returns LeakageStrategy.""" diff --git a/tests/unit/scenarios/test_scam.py b/tests/unit/scenarios/test_scam.py index a4573f551b..67ab923828 100644 --- a/tests/unit/scenarios/test_scam.py +++ b/tests/unit/scenarios/test_scam.py @@ -148,7 +148,7 @@ def test_init_with_custom_objectives( # objectives are stored as _deprecated_objectives; _seed_groups is resolved lazily assert scenario._deprecated_objectives == sample_objectives assert scenario.name == "Scam" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_objectives( self, @@ -162,7 +162,7 @@ def test_init_with_default_objectives( # seed_groups are resolved lazily; _deprecated_objectives should be None assert scenario._deprecated_objectives is None assert scenario.name == "Scam" - assert scenario.version == 1 + assert scenario.VERSION == 1 def test_init_with_default_scorer(self, mock_memory_seed_groups) -> None: """Test initialization with default scorer.""" @@ -373,7 +373,7 @@ def test_scenario_version_is_set( objective_scorer=mock_objective_scorer, ) - assert scenario.version == 1 + assert scenario.VERSION == 1 @pytest.mark.asyncio async def test_no_target_duplication_async(