Merge branch 'rep-pen' into unified

Author: Ph0rk0z

extensions/api/util.py

@@ -31,6 +31,7 @@ def build_parameters(body, chat=False):
         'tfs': float(body.get('tfs', 1)),
         'top_a': float(body.get('top_a', 0)),
         'repetition_penalty': float(body.get('repetition_penalty', body.get('rep_pen', 1.1))),
+        'additive_repetition_penalty': float(body.get('additive_repetition_penalty', body.get('additive_rep_pen', 0))),
         'repetition_penalty_range': int(body.get('repetition_penalty_range', 0)),
         'encoder_repetition_penalty': float(body.get('encoder_repetition_penalty', 1.0)),
         'top_k': int(body.get('top_k', 0)),

modules/loaders.py

@@ -150,6 +150,7 @@
         'tfs',
         'top_a',
         'repetition_penalty',
+        'additive_repetition_penalty',
         'repetition_penalty_range',
         'encoder_repetition_penalty',
         'no_repeat_ngram_size',
@@ -180,6 +181,7 @@
         'tfs',
         'top_a',
         'repetition_penalty',
+        'additive_repetition_penalty',
         'repetition_penalty_range',
         'encoder_repetition_penalty',
         'no_repeat_ngram_size',
@@ -219,6 +221,7 @@
         'tfs',
         'top_a',
         'repetition_penalty',
+        'additive_repetition_penalty',
         'repetition_penalty_range',
         'encoder_repetition_penalty',
         'no_repeat_ngram_size',
@@ -249,6 +252,7 @@
         'tfs',
         'top_a',
         'repetition_penalty',
+        'additive_repetition_penalty',
         'repetition_penalty_range',
         'encoder_repetition_penalty',
         'no_repeat_ngram_size',
@@ -290,6 +294,7 @@
         'tfs',
         'top_a',
         'repetition_penalty',
+        'additive_repetition_penalty',
         'repetition_penalty_range',
         'encoder_repetition_penalty',
         'no_repeat_ngram_size',

modules/presets.py

@@ -16,6 +16,7 @@ def default_preset():
         'tfs': 1,
         'top_a': 0,
         'repetition_penalty': 1,
+        'additive_repetition_penalty': 0,
         'repetition_penalty_range': 0,
         'encoder_repetition_penalty': 1,
         'no_repeat_ngram_size': 0,

modules/sampler_hijack.py

@@ -139,11 +139,12 @@ class RepetitionPenaltyLogitsProcessorWithRange(LogitsProcessor):
     Copied from the transformers library
     '''
 
-    def __init__(self, penalty: float, _range: int):
-        if not isinstance(penalty, float) or not (penalty > 0):
-            raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}")
+    def __init__(self, penalty: float, additive_penalty: float, _range: int):
+        if not (penalty > 0):
+            raise ValueError(f"`penalty` has to be strictly positive, but is {penalty}")
 
         self.penalty = penalty
+        self.additive_penalty = additive_penalty
         self._range = _range
 
     def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
@@ -153,6 +154,7 @@ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> to
 
         # if score < 0 then repetition penalty has to be multiplied to reduce the previous token probability
         score = torch.where(score < 0, score * self.penalty, score / self.penalty)
+        score -= self.additive_penalty
 
         scores.scatter_(1, input_ids, score)
         return scores
@@ -185,14 +187,22 @@ def get_logits_warper_patch(self, generation_config):
 
 
 def get_logits_processor_patch(self, **kwargs):
-    result = self._get_logits_processor_old(**kwargs)
     repetition_penalty_range = kwargs['generation_config'].repetition_penalty_range
     repetition_penalty = kwargs['generation_config'].repetition_penalty
+    additive_repetition_penalty = kwargs['generation_config'].additive_repetition_penalty
+    need_rep_pen_hijack = (repetition_penalty_range > 0) or (additive_repetition_penalty > 0)
+    if need_rep_pen_hijack:
+        # Make sure it always creates a RepetitionPenaltyLogitsProcessor
+        kwargs['generation_config'].repetition_penalty = 1.1  # must set to some value > 1
+    result = self._get_logits_processor_old(**kwargs)
+    if need_rep_pen_hijack:
+        # Now set the rep_pen back to the actual value (just in case)
+        kwargs['generation_config'].repetition_penalty = repetition_penalty
 
-    if repetition_penalty_range > 0:
+    if need_rep_pen_hijack:
         for i in range(len(result)):
             if result[i].__class__.__name__ == 'RepetitionPenaltyLogitsProcessor':
-                result[i] = RepetitionPenaltyLogitsProcessorWithRange(repetition_penalty, repetition_penalty_range)
+                result[i] = RepetitionPenaltyLogitsProcessorWithRange(repetition_penalty, additive_repetition_penalty, repetition_penalty_range)
 
     return result
 
@@ -205,6 +215,7 @@ def generation_config_init_patch(self, **kwargs):
     self.mirostat_eta = kwargs.pop("mirostat_eta", 0.1)
     self.mirostat_tau = kwargs.pop("mirostat_tau", 5)
     self.repetition_penalty_range = kwargs.pop("repetition_penalty_range", 0)
+    self.additive_repetition_penalty = kwargs.pop("additive_repetition_penalty", 0)
 
 
 def hijack_samplers():

modules/text_generation.py

@@ -240,7 +240,7 @@ def apply_stopping_strings(reply, all_stop_strings):
 
 def generate_reply_HF(question, original_question, seed, state, stopping_strings=None, is_chat=False):
     generate_params = {}
-    for k in ['max_new_tokens', 'do_sample', 'temperature', 'top_p', 'typical_p', 'repetition_penalty', 'repetition_penalty_range', 'encoder_repetition_penalty', 'top_k', 'min_length', 'no_repeat_ngram_size', 'num_beams', 'penalty_alpha', 'length_penalty', 'early_stopping', 'tfs', 'top_a', 'mirostat_mode', 'mirostat_tau', 'mirostat_eta', 'guidance_scale']:
+    for k in ['max_new_tokens', 'do_sample', 'temperature', 'top_p', 'typical_p', 'repetition_penalty', 'additive_repetition_penalty', 'repetition_penalty_range', 'encoder_repetition_penalty', 'top_k', 'min_length', 'no_repeat_ngram_size', 'num_beams', 'penalty_alpha', 'length_penalty', 'early_stopping', 'tfs', 'top_a', 'mirostat_mode', 'mirostat_tau', 'mirostat_eta', 'guidance_scale']:
         generate_params[k] = state[k]
 
     if state['negative_prompt'] != '':

modules/ui.py

@@ -112,6 +112,7 @@ def list_interface_input_elements():
         'epsilon_cutoff',
         'eta_cutoff',
         'repetition_penalty',
+        'additive_repetition_penalty',
         'repetition_penalty_range',
         'encoder_repetition_penalty',
         'no_repeat_ngram_size',

modules/ui_parameters.py

@@ -36,6 +36,7 @@ def create_ui(default_preset):
 
                             with gr.Column():
                                 shared.gradio['repetition_penalty'] = gr.Slider(1.0, 1.5, value=generate_params['repetition_penalty'], step=0.01, label='repetition_penalty')
+                                shared.gradio['additive_repetition_penalty'] = gr.Slider(0, 4, value=generate_params['additive_repetition_penalty'], step=0.05, label='additive_repetition_penalty')
                                 shared.gradio['repetition_penalty_range'] = gr.Slider(0, 4096, step=64, value=generate_params['repetition_penalty_range'], label='repetition_penalty_range')
                                 shared.gradio['encoder_repetition_penalty'] = gr.Slider(0.8, 1.5, value=generate_params['encoder_repetition_penalty'], step=0.01, label='encoder_repetition_penalty')
                                 shared.gradio['no_repeat_ngram_size'] = gr.Slider(0, 20, step=1, value=generate_params['no_repeat_ngram_size'], label='no_repeat_ngram_size')
@@ -79,7 +80,9 @@ def create_ui(default_preset):
             ### eta_cutoff
             In units of 1e-4; a reasonable value is 3. Should be used with top_p, top_k, and epsilon_cutoff set to 0.
             ### repetition_penalty
-            Exponential penalty factor for repeating prior tokens. 1 means no penalty, higher value = less repetition, lower value = more repetition.
+            Exponential penalty factor for repeating prior tokens. This is a multiplicative factor on the raw token scores. 1 means no penalty, higher value = less repetition, lower value = more repetition.
+            ### additive_repetition_penalty
+            Similar to repetition_penalty, but with an additive offset on the raw token scores instead of a multiplicative factor. 0 means no penalty, higher value = less repetition, lower value = more repetition.
             ### repetition_penalty_range
             The number of most recent tokens to consider for repetition penalty. 0 makes all tokens be used.
             ### encoder_repetition_penalty