diff --git a/tests/generators/test_generators.py b/tests/generators/test_generators.py index 38e439dd..7c8e29a0 100644 --- a/tests/generators/test_generators.py +++ b/tests/generators/test_generators.py @@ -4,7 +4,6 @@ import importlib import inspect import pytest -import random from typing import List, Union @@ -12,7 +11,6 @@ from garak import _config from garak.attempt import Turn -from garak.generators.test import Blank, Repeat, Single from garak.generators.base import Generator @@ -25,112 +23,6 @@ ] -def test_generators_test_blank(): - g = Blank(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test"), generations_this_call=5) - assert output == [ - Turn(""), - Turn(""), - Turn(""), - Turn(""), - Turn(""), - ], "generators.test.Blank with generations_this_call=5 should return five Turns with empty text" - - -def test_generators_test_repeat(): - g = Repeat(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn(DEFAULT_PROMPT_TEXT)) - assert output == [ - Turn(DEFAULT_PROMPT_TEXT) - ], "generators.test.Repeat should send back a list of the posed prompt Turn" - - -def test_generators_test_single_one(): - g = Single(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test")) - assert isinstance( - output, list - ), "Single generator .generate() should send back a list" - assert ( - len(output) == 1 - ), "Single.generate() without generations_this_call should send a list of length one" - assert isinstance( - output[0], Turn - ), "Single generator output list should contain Turns" - - output = g._call_model(prompt=Turn("test")) - assert isinstance(output, list), "Single generator _call_model should return a list" - assert ( - len(output) == 1 - ), "_call_model w/ generations_this_call 1 should return a list of length 1" - assert isinstance( - output[0], Turn - ), "Single generator output list should contain Turns" - - -def test_generators_test_single_many(): - random_generations = random.randint(2, 12) - g = Single(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test"), generations_this_call=random_generations) - assert isinstance( - output, list - ), "Single generator .generate() should send back a list" - assert ( - len(output) == random_generations - ), "Single.generate() with generations_this_call should return equal generations" - for i in range(0, random_generations): - assert isinstance( - output[i], Turn - ), "Single generator output list should contain Turns (all positions)" - - -def test_generators_test_single_too_many(): - g = Single(DEFAULT_GENERATOR_NAME) - with pytest.raises(ValueError): - output = g._call_model(prompt=Turn("test"), generations_this_call=2) - assert "Single._call_model should refuse to process generations_this_call > 1" - - -def test_generators_test_blank_one(): - g = Blank(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test")) - assert isinstance( - output, list - ), "Blank generator .generate() should send back a list" - assert ( - len(output) == 1 - ), "Blank generator .generate() without generations_this_call should return a list of length 1" - assert isinstance( - output[0], Turn - ), "Blank generator output list should contain Turns" - assert output[0] == Turn( - "" - ), "Blank generator .generate() output list should contain Turns" - - -def test_generators_test_blank_many(): - g = Blank(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test"), generations_this_call=2) - assert isinstance( - output, list - ), "Blank generator .generate() should send back a list" - assert ( - len(output) == 2 - ), "Blank generator .generate() w/ generations_this_call=2 should return a list of length 2" - assert isinstance( - output[0], Turn - ), "Blank generator output list should contain Turns (first position)" - assert isinstance( - output[1], Turn - ), "Blank generator output list should contain Turns (second position)" - assert output[0] == Turn( - "" - ), "Blank generator .generate() output list should contain Turns (first position)" - assert output[1] == Turn( - "" - ), "Blank generator .generate() output list should contain Turns (second position)" - - def test_parallel_requests(): _config.system.parallel_requests = 2 diff --git a/tests/generators/test_generators_base.py b/tests/generators/test_generators_base.py index c20fe4f2..1a8431c3 100644 --- a/tests/generators/test_generators_base.py +++ b/tests/generators/test_generators_base.py @@ -1,117 +1,2 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -import pytest -import random - -from garak.attempt import Turn -from garak.generators.test import Blank, Repeat, Single - -DEFAULT_GENERATOR_NAME = "garak test" -DEFAULT_PROMPT_TEXT = "especially the lies" - - -def test_generators_test_blank(): - g = Blank(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test"), generations_this_call=5) - assert output == [ - Turn(""), - Turn(""), - Turn(""), - Turn(""), - Turn(""), - ], "generators.test.Blank with generations_this_call=5 should return five empty Turns" - - -def test_generators_test_repeat(): - g = Repeat(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn(DEFAULT_PROMPT_TEXT)) - assert output == [ - Turn(DEFAULT_PROMPT_TEXT) - ], "generators.test.Repeat should send back a list of the posed prompt string" - - -def test_generators_test_single_one(): - g = Single(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test")) - assert isinstance( - output, list - ), "Single generator .generate() should send back a list" - assert ( - len(output) == 1 - ), "Single.generate() without generations_this_call should send a list of one Turn" - assert isinstance( - output[0], Turn - ), "Single generator output list should contain strings" - - output = g._call_model(prompt=Turn("test")) - assert isinstance(output, list), "Single generator _call_model should return a list" - assert ( - len(output) == 1 - ), "_call_model w/ generations_this_call 1 should return a list of length 1" - assert isinstance( - output[0], Turn - ), "Single generator output list should contain Turns" - - -def test_generators_test_single_many(): - random_generations = random.randint(2, 12) - g = Single(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test"), generations_this_call=random_generations) - assert isinstance( - output, list - ), "Single generator .generate() should send back a list" - assert ( - len(output) == random_generations - ), "Single.generate() with generations_this_call should return equal generations" - for i in range(0, random_generations): - assert isinstance( - output[i], Turn - ), "Single generator output list should contain Turns" - - -def test_generators_test_single_too_many(): - g = Single(DEFAULT_GENERATOR_NAME) - with pytest.raises(ValueError): - output = g._call_model(prompt=Turn("test"), generations_this_call=2) - assert "Single._call_model should refuse to process generations_this_call > 1" - - -def test_generators_test_blank_one(): - g = Blank(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test")) - assert isinstance( - output, list - ), "Blank generator .generate() should send back a list" - assert ( - len(output) == 1 - ), "Blank generator .generate() without generations_this_call should return a list of length 1" - assert isinstance( - output[0], Turn - ), "Blank generator output list should contain Turns" - assert output[0] == Turn( - "" - ), "Blank generator .generate() output list should contain Turns" - - -def test_generators_test_blank_many(): - g = Blank(DEFAULT_GENERATOR_NAME) - output = g.generate(prompt=Turn("test"), generations_this_call=2) - assert isinstance( - output, list - ), "Blank generator .generate() should send back a list" - assert ( - len(output) == 2 - ), "Blank generator .generate() w/ generations_this_call=2 should return a list of length 2" - assert isinstance( - output[0], Turn - ), "Blank generator output list should contain Turns (first position)" - assert isinstance( - output[1], Turn - ), "Blank generator output list should contain Turns (second position)" - assert output[0] == Turn( - "" - ), "Blank generator .generate() output list should contain Turns (first position) w empty prompt" - assert output[1] == Turn( - "" - ), "Blank generator .generate() output list should contain Turns (second position) w empty prompt" diff --git a/tests/generators/test_test.py b/tests/generators/test_test.py index 785a2470..30466290 100644 --- a/tests/generators/test_test.py +++ b/tests/generators/test_test.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 import pytest +import random import garak._plugins from garak.attempt import Turn import garak.generators.base -import garak.generators.test +from garak.generators.test import Blank, Repeat, Single TEST_GENERATORS = [ a @@ -14,6 +15,9 @@ if b is True and a.startswith("generators.test") ] +DEFAULT_GENERATOR_NAME = "garak test" +DEFAULT_PROMPT_TEXT = "especially the lies" + @pytest.mark.parametrize("klassname", TEST_GENERATORS) def test_test_instantiate(klassname): @@ -34,3 +38,109 @@ def test_test_gen(klassname): assert ( isinstance(s, Turn) or s is None ), "generate()'s returned list's items must be Turn or None" + + +def test_generators_test_blank(): + g = Blank(DEFAULT_GENERATOR_NAME) + output = g.generate(prompt=Turn("test"), generations_this_call=5) + assert output == [ + Turn(""), + Turn(""), + Turn(""), + Turn(""), + Turn(""), + ], "generators.test.Blank with generations_this_call=5 should return five empty Turns" + + +def test_generators_test_repeat(): + g = Repeat(DEFAULT_GENERATOR_NAME) + output = g.generate(prompt=Turn(DEFAULT_PROMPT_TEXT)) + assert output == [ + Turn(DEFAULT_PROMPT_TEXT) + ], "generators.test.Repeat should send back a list of the posed prompt string" + + +def test_generators_test_single_one(): + g = Single(DEFAULT_GENERATOR_NAME) + output = g.generate(prompt=Turn("test")) + assert isinstance( + output, list + ), "Single generator .generate() should send back a list" + assert ( + len(output) == 1 + ), "Single.generate() without generations_this_call should send a list of one Turn" + assert isinstance( + output[0], Turn + ), "Single generator output list should contain strings" + + output = g._call_model(prompt=Turn("test")) + assert isinstance(output, list), "Single generator _call_model should return a list" + assert ( + len(output) == 1 + ), "_call_model w/ generations_this_call 1 should return a list of length 1" + assert isinstance( + output[0], Turn + ), "Single generator output list should contain Turns" + + +def test_generators_test_single_many(): + random_generations = random.randint(2, 12) + g = Single(DEFAULT_GENERATOR_NAME) + output = g.generate(prompt=Turn("test"), generations_this_call=random_generations) + assert isinstance( + output, list + ), "Single generator .generate() should send back a list" + assert ( + len(output) == random_generations + ), "Single.generate() with generations_this_call should return equal generations" + for i in range(0, random_generations): + assert isinstance( + output[i], Turn + ), "Single generator output list should contain Turns" + + +def test_generators_test_single_too_many(): + g = Single(DEFAULT_GENERATOR_NAME) + with pytest.raises(ValueError): + output = g._call_model(prompt=Turn("test"), generations_this_call=2) + assert "Single._call_model should refuse to process generations_this_call > 1" + + +def test_generators_test_blank_one(): + g = Blank(DEFAULT_GENERATOR_NAME) + output = g.generate(prompt=Turn("test")) + assert isinstance( + output, list + ), "Blank generator .generate() should send back a list" + assert ( + len(output) == 1 + ), "Blank generator .generate() without generations_this_call should return a list of length 1" + assert isinstance( + output[0], Turn + ), "Blank generator output list should contain Turns" + assert output[0] == Turn( + "" + ), "Blank generator .generate() output list should contain Turns" + + +def test_generators_test_blank_many(): + g = Blank(DEFAULT_GENERATOR_NAME) + output = g.generate(prompt=Turn("test"), generations_this_call=2) + assert isinstance( + output, list + ), "Blank generator .generate() should send back a list" + assert ( + len(output) == 2 + ), "Blank generator .generate() w/ generations_this_call=2 should return a list of length 2" + assert isinstance( + output[0], Turn + ), "Blank generator output list should contain Turns (first position)" + assert isinstance( + output[1], Turn + ), "Blank generator output list should contain Turns (second position)" + assert output[0] == Turn( + "" + ), "Blank generator .generate() output list should contain Turns (first position) w empty prompt" + assert output[1] == Turn( + "" + ), "Blank generator .generate() output list should contain Turns (second position) w empty prompt"