Run factuality benchmark with low temp

Author: peterdemin

fact.py

@@ -44,7 +44,11 @@ def main():
         for swap in (False, True):
             prompt = format_prompt(task, swap)
             print(f'{i}. {prompt}')
-            output = llm.create_completion(prompt, max_tokens=2)
+            output = llm.create_completion(
+                prompt,
+                max_tokens=20,
+                temperature=1e-6,
+            )
             answer = output['choices'][0]['text'].strip().split()[0]
             print(answer)
             result = dict(task, answer=answer, swap=swap)

requirements-llama-2.txt

@@ -0,0 +1 @@
+llama-cpp-python==0.1.77  # Works with Llama 2 GGML

requirements-wizard.txt

@@ -0,0 +1 @@
+llama-cpp-python==0.1.83  # Works with Wizard GGUF