Adding OpenAssistant support, while running Orca

Author: peterdemin

.gitignore

@@ -2,3 +2,4 @@ model.bin
 .ipynb_checkpoints
 wizard.gguf
 orca.gguf
+oasst.gguf

fact.py

@@ -1,7 +1,7 @@
 import json
 from llama_cpp import Llama
 
-MODEL_PATH = 'orca.gguf'
+MODEL_PATH = 'oasst.gguf'
 TASKS_PATH = 'fact.json'
 
 PROMPT_TMPL = """\
@@ -15,13 +15,39 @@
 Answer: The more consistent is Summary"""
 
 
+OASST_TMPL = """\
+<|im_start|>system
+You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
+<|im_end|>
+<|im_start|>user
+Decide which of the following Summary is more consistent with the Article Sentence.
+
+Note that consistency means all information in the Summary is supported by the Article Sentence.
+
+Article Sentence: {article}
+Summary Y: {option_a}
+Summary X: {option_b}
+<|im_end|>
+<|im_start|>assistant
+The more consistent is Summary
+"""
+OASST_STOP = '<|im_end|>'
+
+
 def iter_tasks(filename):
     with open(filename, 'rt', encoding='utf-8') as fobj:
         return json.load(fobj)
 
 
 def format_prompt(task, swap):
-    return PROMPT_TMPL.format(
+    template = (
+        OASST_TMPL
+        if 'oasst' in MODEL_PATH
+        else PROMPT_TMPL
+    )
+    return template.format(
         article=task['article_sent'],
         option_a=task['incorrect_sent'] if swap else task['correct_sent'],
         option_b=task['correct_sent'] if swap else task['incorrect_sent'],
@@ -63,6 +89,7 @@ def main():
                 top_k=10,
                 top_p=0.9,
                 temperature=1e-6,
+                stop=[OASST_STOP],
             )
             answer = output['choices'][0]['text'].strip().split()[0]
             print(answer)

results.jsonl

@@ -14,3 +14,10 @@
 {"article_id": "0d046cb0b05b40cdb9138f732bde87a828e6bf07", "article_sent": "' referring to the ump's electoral alliance with the centrist union of democrats and independents (udi), mr sarkozy said: 'never in the history of the fifth republic has our political family won so many departments.", "correct_sent": "sarkozy's electoral alliance has won the departmental elections.", "correct_sent_id": "95-3", "incorrect_sent": "sarkozy's electoral alliance has won departmental elections.", "incorrect_sent_id": "42-3", "answer": "X,", "swap": true}
 {"article_id": "6aa21c738f5c5c2fa7b1cad4f3dba97cc9634878", "article_sent": "'terrorist' and'd ** k' can also be included in messages.", "correct_sent": "** k' can also be included in messages.", "correct_sent_id": "82-2", "incorrect_sent": "** k'' can also be included in messages.", "incorrect_sent_id": "68-2", "answer": "Y,", "swap": false}
 {"article_id": "6aa21c738f5c5c2fa7b1cad4f3dba97cc9634878", "article_sent": "'terrorist' and'd ** k' can also be included in messages.", "correct_sent": "** k' can also be included in messages.", "correct_sent_id": "82-2", "incorrect_sent": "** k'' can also be included in messages.", "incorrect_sent_id": "68-2", "answer": "X,", "swap": true}
+{"article_id": "10f08229ca491fd0111f427c6e9b9301bdbf2435", "article_sent": "insiders say the row brought simmering tensions between the starkly contrasting pair -- both rivals for miliband's ear -- to a head.", "correct_sent": "insiders say the row brought tensions between the contrasting pair.", "correct_sent_id": "28-1", "incorrect_sent": "insiders say the row brought simmering tensions between miliband's ear.", "incorrect_sent_id": "78-1", "answer": "Y,", "swap": false}
+{"article_id": "10f08229ca491fd0111f427c6e9b9301bdbf2435", "article_sent": "insiders say the row brought simmering tensions between the starkly contrasting pair -- both rivals for miliband's ear -- to a head.", "correct_sent": "insiders say the row brought tensions between the contrasting pair.", "correct_sent_id": "28-1", "incorrect_sent": "insiders say the row brought simmering tensions between miliband's ear.", "incorrect_sent_id": "78-1", "answer": "X,", "swap": true}
+{"article_id": "9c58e5e056912c70a519faaa9c8d3d36d545c64d", "article_sent": "katarzyne pietrzycka, 42, was attacked with a stiletto by a thai transvestite jealous over her husband, leaving her bleeding heavily from her head (right) following the assault in the east coast thai city of pattaya.", "correct_sent": "katarzyne pietrzycka, 42, was attacked with a stiletto by thai transvestite jealous.", "correct_sent_id": "22-3", "incorrect_sent": "katarzyne pietrzycka, 42, was attacked with a stiletto by a thai court.", "incorrect_sent_id": "53-3", "answer": "Y.", "swap": false}
+{"article_id": "9c58e5e056912c70a519faaa9c8d3d36d545c64d", "article_sent": "katarzyne pietrzycka, 42, was attacked with a stiletto by a thai transvestite jealous over her husband, leaving her bleeding heavily from her head (right) following the assault in the east coast thai city of pattaya.", "correct_sent": "katarzyne pietrzycka, 42, was attacked with a stiletto by thai transvestite jealous.", "correct_sent_id": "22-3", "incorrect_sent": "katarzyne pietrzycka, 42, was attacked with a stiletto by a thai court.", "incorrect_sent_id": "53-3", "answer": "X.", "swap": true}
+{"article_id": "1cba358ee2a12f34b708a66177550308656741d0", "article_sent": "the man, named only as saleh, said the masked militant who appears in several beheading videos was a senior figure in the extremist organisation responsible for murdering foreign captives.", "correct_sent": "the man, named only as saleh, said the masked militant appears in several beheading videos.", "correct_sent_id": "5-0", "incorrect_sent": "the man, named only as saleh, appears in several beheading videos.", "incorrect_sent_id": "20-0", "answer": "Y.", "swap": false}
+{"article_id": "1cba358ee2a12f34b708a66177550308656741d0", "article_sent": "the man, named only as saleh, said the masked militant who appears in several beheading videos was a senior figure in the extremist organisation responsible for murdering foreign captives.", "correct_sent": "the man, named only as saleh, said the masked militant appears in several beheading videos.", "correct_sent_id": "5-0", "incorrect_sent": "the man, named only as saleh, appears in several beheading videos.", "incorrect_sent_id": "20-0", "answer": "X.", "swap": true}
+{"article_id": "33f52168d0cbc6cef4f3ed2469ff50f9712f8466", "article_sent": "the bolton boxer had hoped to take on floyd mayweather but the pound-for-pound king will instead meet manny pacquaio in a $ 300million mega-fight on may 2.", "correct_sent": "the boxer will instead meet manny pacquaio in a $ 300million mega-fight.", "correct_sent_id": "71-1", "incorrect_sent": "the bolton boxer had hoped to take the floyd mayweather in a $ 300million mega-fight.", "incorrect_sent_id": "19-1", "answer": "Y,", "swap": false}