[Feature] support output without loss in openai_map_fn (#816)

support output without loss in openai_map_fn
InternLM · Jul 9, 2024 · 8adc8d4 · 8adc8d4
1 parent 7575f2c
commit 8adc8d4
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py
@@ -32,10 +32,13 @@ def openai_map_fn(example):
         elif msg['role'] == 'user':
             input += msg['content']
         elif msg['role'] == 'assistant':
+            output_with_loss = msg.get('loss', True)
+            output_with_loss = output_with_loss.lower() == 'true'
             conversation.append({
                 'system': system,
                 'input': input,
-                'output': msg['content']
+                'output': msg['content'],
+                'output_with_loss': output_with_loss
             })
             system = ''
             input = ''