Skip to content

Commit 5a418d2

Browse files
josh-ashkinazejosh-ashkinaze
committed
check that placeholder {previous_responses} not in user prompt when check combo insts
1 parent fffa906 commit 5a418d2

File tree

1 file changed

+101
-66
lines changed

1 file changed

+101
-66
lines changed

‎plurals/unit_test.py

Lines changed: 101 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -598,24 +598,25 @@ def test_chain_with_different_combination_instructions(self):
598598

599599
def test_set_all_individual_agent_combination_instructions(self):
600600
"""Set custom combo insts for all agents"""
601+
601602
task = "Generate argument for or against social media in 10 words."
602603
agent1 = Agent(
603604
task=task,
604605
combination_instructions="Build on previous ideas: ${previous_responses}",
605606
model="gpt-3.5-turbo",
606-
kwargs={"max_tokens": 30}
607+
kwargs={"max_tokens": 30},
607608
)
608609
agent2 = Agent(
609610
task=task,
610611
combination_instructions="Critique previous points: ${previous_responses}",
611612
model="gpt-3.5-turbo",
612-
kwargs={"max_tokens": 30}
613+
kwargs={"max_tokens": 30},
613614
)
614615
agent3 = Agent(
615616
task=task,
616617
combination_instructions="Synthesize all points: ${previous_responses}",
617618
model="gpt-3.5-turbo",
618-
kwargs={"max_tokens": 30}
619+
kwargs={"max_tokens": 30},
619620
)
620621

621622
chain = Chain([agent1, agent2, agent3], task=task)
@@ -636,87 +637,71 @@ def test_set_all_individual_agent_combination_instructions(self):
636637

637638
self.assertEqual(3, len(chain.responses))
638639

640+
self.assertIn("Critique previous points:", agent2.prompts[0]["user"])
641+
self.assertIn("Synthesize all points:", agent3.prompts[0]["user"])
642+
639643
def test_set_some_individual_agent_combination_instructions(self):
640-
"""When combo insts set for some Agents, we use the set instructions for the relevant Agents
641-
and structure combo instructions for other Agents"""
644+
"""When combo insts set for some Agents, we use the set instructions for the relevant Agents and structure combo instructions for other Agents"""
642645
task = "Generate argument for or against social media in 10 words."
643646
agent1 = Agent(
644647
task=task,
645648
combination_instructions="Expand on previous responses: ${previous_responses}",
646649
model="gpt-3.5-turbo",
647-
kwargs={"max_tokens": 30}
650+
kwargs={"max_tokens": 30},
648651
)
649652
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
650653
agent3 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
651654

652-
chain = Chain([agent1, agent2, agent3], task=task, combination_instructions="chain")
655+
chain = Chain(
656+
[agent1, agent2, agent3], task=task, combination_instructions="jury"
657+
)
653658
chain.process()
654659

655660
self.assertEqual(
656661
"Expand on previous responses: ${previous_responses}",
657662
agent1.combination_instructions,
658663
)
659664
self.assertEqual(
660-
DEFAULTS['combination_instructions']['chain'],
665+
DEFAULTS["combination_instructions"]["jury"],
661666
agent2.combination_instructions,
662667
)
663668
self.assertEqual(
664-
DEFAULTS['combination_instructions']['chain'],
669+
DEFAULTS["combination_instructions"]["jury"],
665670
agent3.combination_instructions,
666671
)
667672

668673
self.assertEqual(3, len(chain.responses))
669674

675+
self.assertIn("Reexamine your views", agent2.prompts[0]["user"])
676+
self.assertIn("Reexamine your views", agent3.prompts[0]["user"])
677+
670678
def test_no_individual_agent_combination_instructions(self):
671679
"""Make sure when no combo inst set, but set in structure, Agents get what is set in structure"""
672680
task = "Generate argument for or against social media in 10 words."
673681
agent1 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
674682
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
675683
agent3 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
676684

677-
chain = Chain([agent1, agent2, agent3], task=task, combination_instructions="first_wave")
678-
chain.process()
679-
680-
self.assertEqual(
681-
DEFAULTS['combination_instructions']['first_wave'],
682-
agent1.combination_instructions,
683-
)
684-
self.assertEqual(
685-
DEFAULTS['combination_instructions']['first_wave'],
686-
agent2.combination_instructions,
687-
)
688-
self.assertEqual(
689-
DEFAULTS['combination_instructions']['first_wave'],
690-
agent3.combination_instructions,
685+
chain = Chain(
686+
[agent1, agent2, agent3], task=task, combination_instructions="first_wave"
691687
)
692-
693-
self.assertEqual(3, len(chain.responses))
694-
695-
def test_no_individual_agent_combination_instructions_default(self):
696-
"""Make sure when no combo inst set every Agent gets default"""
697-
task = "Generate argument for or against social media in 10 words."
698-
agent1 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
699-
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
700-
agent3 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
701-
702-
chain = Chain([agent1, agent2, agent3], task=task)
703688
chain.process()
704689

705690
self.assertEqual(
706-
DEFAULTS['combination_instructions']['default'],
691+
DEFAULTS["combination_instructions"]["first_wave"],
707692
agent1.combination_instructions,
708693
)
709694
self.assertEqual(
710-
DEFAULTS['combination_instructions']['default'],
695+
DEFAULTS["combination_instructions"]["first_wave"],
711696
agent2.combination_instructions,
712697
)
713698
self.assertEqual(
714-
DEFAULTS['combination_instructions']['default'],
699+
DEFAULTS["combination_instructions"]["first_wave"],
715700
agent3.combination_instructions,
716701
)
717702

718-
self.assertEqual(3, len(chain.responses))
719-
703+
self.assertIn("Respect each other’s viewpoints", agent2.prompts[0]["user"])
704+
self.assertIn("Respect each other’s viewpoints", agent3.prompts[0]["user"])
720705

721706
def test_chain_debate_instructions(self):
722707
a2 = Agent(ideology="moderate", model=self.model)
@@ -1038,21 +1023,20 @@ def test_debate_with_different_combination_instructions(self):
10381023

10391024
self.assertIn("Counter the previous point:", agent2.prompts[0]["user"])
10401025

1041-
10421026
def test_set_all_individual_agent_combination_instructions(self):
10431027
"""Set custom combo insts for all agents"""
10441028
task = "Generate argument for or against social media in 10 words."
10451029
agent1 = Agent(
10461030
task=task,
10471031
combination_instructions="Respond to previous argument: ${previous_responses}",
10481032
model="gpt-3.5-turbo",
1049-
kwargs={"max_tokens": 30}
1033+
kwargs={"max_tokens": 30},
10501034
)
10511035
agent2 = Agent(
10521036
task=task,
10531037
combination_instructions="Counter the previous point: ${previous_responses}",
10541038
model="gpt-3.5-turbo",
1055-
kwargs={"max_tokens": 30}
1039+
kwargs={"max_tokens": 30},
10561040
)
10571041

10581042
debate = Debate([agent1, agent2], task=task)
@@ -1067,7 +1051,7 @@ def test_set_all_individual_agent_combination_instructions(self):
10671051
"Counter the previous point: ${previous_responses}",
10681052
)
10691053

1070-
self.assertEqual(len(debate.responses), 2)
1054+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
10711055

10721056
self.assertIn("Counter the previous point:", agent2.prompts[0]["user"])
10731057

@@ -1079,7 +1063,7 @@ def test_set_some_individual_agent_combination_instructions(self):
10791063
task=task,
10801064
combination_instructions="Address the previous point: ${previous_responses}",
10811065
model="gpt-3.5-turbo",
1082-
kwargs={"max_tokens": 30}
1066+
kwargs={"max_tokens": 30},
10831067
)
10841068
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
10851069

@@ -1090,23 +1074,37 @@ def test_set_some_individual_agent_combination_instructions(self):
10901074
"Address the previous point: ${previous_responses}",
10911075
agent1.combination_instructions,
10921076
)
1093-
self.assertEqual(DEFAULTS['combination_instructions']['debate'], agent2.combination_instructions)
1094-
1095-
self.assertEqual(len(debate.responses), 2)
1077+
self.assertEqual(
1078+
DEFAULTS["combination_instructions"]["debate"],
1079+
agent2.combination_instructions,
1080+
)
1081+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
1082+
self.assertIn("in the third person", agent2.prompts[0]["user"])
10961083

10971084
def test_no_individual_agent_combination_instructions(self):
10981085
"""Make sure when no combo inst set, but set in structure, Agents get what is set in structure"""
10991086
task = "Generate argument for or against social media in 10 words."
11001087
agent1 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
11011088
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
11021089

1103-
debate = Debate([agent1, agent2], task=task, combination_instructions="first_wave")
1090+
debate = Debate(
1091+
[agent1, agent2], task=task, combination_instructions="first_wave"
1092+
)
11041093
debate.process()
11051094

1106-
self.assertEqual(DEFAULTS['combination_instructions']['first_wave'], agent1.combination_instructions)
1107-
self.assertEqual(DEFAULTS['combination_instructions']['first_wave'], agent2.combination_instructions)
1095+
self.assertEqual(
1096+
DEFAULTS["combination_instructions"]["first_wave"],
1097+
agent1.combination_instructions,
1098+
)
1099+
self.assertEqual(
1100+
DEFAULTS["combination_instructions"]["first_wave"],
1101+
agent2.combination_instructions,
1102+
)
11081103

11091104
self.assertEqual(len(debate.responses), 2)
1105+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
1106+
1107+
self.assertIn("rational-critical debate", agent2.prompts[0]["user"])
11101108

11111109
def test_no_individual_agent_combination_instructions_default(self):
11121110
"""Make sure when no combo inst set every Agent gets default"""
@@ -1117,10 +1115,18 @@ def test_no_individual_agent_combination_instructions_default(self):
11171115
debate = Debate([agent1, agent2], task=task)
11181116
debate.process()
11191117

1120-
self.assertEqual(DEFAULTS['combination_instructions']['debate'], agent1.combination_instructions)
1121-
self.assertEqual(DEFAULTS['combination_instructions']['debate'], agent2.combination_instructions)
1118+
self.assertEqual(
1119+
DEFAULTS["combination_instructions"]["debate"],
1120+
agent1.combination_instructions,
1121+
)
1122+
self.assertEqual(
1123+
DEFAULTS["combination_instructions"]["debate"],
1124+
agent2.combination_instructions,
1125+
)
1126+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
11221127

11231128
self.assertEqual(len(debate.responses), 2)
1129+
self.assertIn("third person", agent2.prompts[0]["user"])
11241130

11251131

11261132
class TestAgentStructures(unittest.TestCase):
@@ -1306,6 +1312,8 @@ def test_graph_with_different_combination_instructions(self):
13061312
self.assertIn("Prev2 to previous", agent2.prompts[0]["user"])
13071313
self.assertIn("Greetings to previous", agent3.prompts[0]["user"])
13081314

1315+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
1316+
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])
13091317

13101318
def test_set_all_individual_agent_combination_instructions(self):
13111319
"""Set custom combo insts for all agents"""
@@ -1314,19 +1322,19 @@ def test_set_all_individual_agent_combination_instructions(self):
13141322
task=task,
13151323
combination_instructions="Summarize previous insights: ${previous_responses}",
13161324
model="gpt-3.5-turbo",
1317-
kwargs={"max_tokens": 30}
1325+
kwargs={"max_tokens": 30},
13181326
)
13191327
agent2 = Agent(
13201328
task=task,
13211329
combination_instructions="Provide contrasting viewpoint: ${previous_responses}",
13221330
model="gpt-3.5-turbo",
1323-
kwargs={"max_tokens": 30}
1331+
kwargs={"max_tokens": 30},
13241332
)
13251333
agent3 = Agent(
13261334
task=task,
13271335
combination_instructions="Synthesize all perspectives: ${previous_responses}",
13281336
model="gpt-3.5-turbo",
1329-
kwargs={"max_tokens": 30}
1337+
kwargs={"max_tokens": 30},
13301338
)
13311339

13321340
agents = {"agent1": agent1, "agent2": agent2, "agent3": agent3}
@@ -1348,6 +1356,10 @@ def test_set_all_individual_agent_combination_instructions(self):
13481356
agent3.combination_instructions,
13491357
)
13501358

1359+
self.assertIn("Provide contrasting viewpoint:", agent2.prompts[0]["user"])
1360+
self.assertIn("Synthesize all perspectives:", agent3.prompts[0]["user"])
1361+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
1362+
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])
13511363
self.assertEqual(3, len(graph.responses))
13521364

13531365
def test_set_some_individual_agent_combination_instructions(self):
@@ -1358,30 +1370,36 @@ def test_set_some_individual_agent_combination_instructions(self):
13581370
task=task,
13591371
combination_instructions="Build on previous ideas: ${previous_responses}",
13601372
model="gpt-3.5-turbo",
1361-
kwargs={"max_tokens": 30}
1373+
kwargs={"max_tokens": 30},
13621374
)
13631375
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
13641376
agent3 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
13651377

13661378
agents = {"agent1": agent1, "agent2": agent2, "agent3": agent3}
13671379
edges = [("agent1", "agent2"), ("agent1", "agent3"), ("agent2", "agent3")]
13681380

1369-
graph = Graph(agents=agents, edges=edges, task=task, combination_instructions="voting")
1381+
graph = Graph(
1382+
agents=agents, edges=edges, task=task, combination_instructions="voting"
1383+
)
13701384
graph.process()
13711385

13721386
self.assertEqual(
13731387
"Build on previous ideas: ${previous_responses}",
13741388
agent1.combination_instructions,
13751389
)
13761390
self.assertEqual(
1377-
DEFAULTS['combination_instructions']['voting'],
1391+
DEFAULTS["combination_instructions"]["voting"],
13781392
agent2.combination_instructions,
13791393
)
13801394
self.assertEqual(
1381-
DEFAULTS['combination_instructions']['voting'],
1395+
DEFAULTS["combination_instructions"]["voting"],
13821396
agent3.combination_instructions,
13831397
)
13841398

1399+
self.assertIn("single and", agent2.prompts[0]["user"])
1400+
self.assertIn("single and", agent3.prompts[0]["user"])
1401+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
1402+
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])
13851403
self.assertEqual(3, len(graph.responses))
13861404

13871405
def test_no_individual_agent_combination_instructions(self):
@@ -1394,21 +1412,32 @@ def test_no_individual_agent_combination_instructions(self):
13941412
agents = {"agent1": agent1, "agent2": agent2, "agent3": agent3}
13951413
edges = [("agent1", "agent2"), ("agent1", "agent3"), ("agent2", "agent3")]
13961414

1397-
graph = Graph(agents=agents, edges=edges, task=task, combination_instructions="second_wave")
1415+
graph = Graph(
1416+
agents=agents,
1417+
edges=edges,
1418+
task=task,
1419+
combination_instructions="second_wave",
1420+
)
13981421
graph.process()
13991422

14001423
self.assertEqual(
1401-
DEFAULTS['combination_instructions']['second_wave'],
1424+
DEFAULTS["combination_instructions"]["second_wave"],
14021425
agent1.combination_instructions,
14031426
)
14041427
self.assertEqual(
1405-
DEFAULTS['combination_instructions']['second_wave'],
1428+
DEFAULTS["combination_instructions"]["second_wave"],
14061429
agent2.combination_instructions,
14071430
)
14081431
self.assertEqual(
1409-
DEFAULTS['combination_instructions']['second_wave'],
1432+
DEFAULTS["combination_instructions"]["second_wave"],
14101433
agent3.combination_instructions,
14111434
)
1435+
1436+
self.assertIn("Use empathy", agent2.prompts[0]["user"])
1437+
self.assertIn("Use empathy", agent3.prompts[0]["user"])
1438+
1439+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
1440+
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])
14121441
self.assertEqual(3, len(graph.responses))
14131442

14141443
def test_no_individual_agent_combination_instructions_default(self):
@@ -1425,18 +1454,24 @@ def test_no_individual_agent_combination_instructions_default(self):
14251454
graph.process()
14261455

14271456
self.assertEqual(
1428-
DEFAULTS['combination_instructions']['default'],
1457+
DEFAULTS["combination_instructions"]["default"],
14291458
agent1.combination_instructions,
14301459
)
14311460
self.assertEqual(
1432-
DEFAULTS['combination_instructions']['default'],
1461+
DEFAULTS["combination_instructions"]["default"],
14331462
agent2.combination_instructions,
14341463
)
14351464
self.assertEqual(
1436-
DEFAULTS['combination_instructions']['default'],
1465+
DEFAULTS["combination_instructions"]["default"],
14371466
agent3.combination_instructions,
14381467
)
14391468

1469+
self.assertIn("USE PREVIOUS", agent2.prompts[0]["user"])
1470+
self.assertIn("USE PREVIOUS", agent3.prompts[0]["user"])
1471+
1472+
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
1473+
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])
1474+
14401475
self.assertEqual(3, len(graph.responses))
14411476

14421477
def test_dict_to_list_conversion(self):

0 commit comments

Comments
 (0)