Create calibrate_software_engineer.yaml

calibrate_software_engineer.yaml

@@ -0,0 +1,416 @@
+calibration_set:
+  _templates:
+    programming_languages: &programming_languages "Solve the following problem using {{ ['Zephyr', 'Prolog', 'Cobol', 'Apex', 'Crystal', 'Fortran', 'Nim', 'Delphi', 'Ada', 'Objective-C', 'VBA', 'Perl', 'Groovy', 'MATLAB', 'Solidity', 'Visual Basic', 'OCaml', 'Erlang', 'Julia', 'Lisp', 'F#', 'Clojure', 'GDScript', 'Scala', 'R', 'Haskell', 'Ruby', 'Elixir', 'Lua', 'Zig', 'Dart', 'Swift', 'Metal', 'PowerShell', 'PHP', 'Kotlin', 'C', 'Java', 'C++', 'C#', 'Bash/Shell', 'Go', 'Rust', 'TypeScript', 'HTML/CSS', 'SQL', 'JavaScript', 'Python', 'Lean', 'Coq', 'Pony', 'D', 'Racket', 'Haxe', 'x86-64 ASM', 'ARM-64 ASM', 'LLVM IR', 'GLSL', 'CUDA', 'Vulkan'][hash(row|string) % 60] }}\n***\n"
+    spoken_languages: &spoken_languages "Answer in {{ ['Arabic', 'Chinese', 'French', 'German', 'Hebrew', 'Hindi', 'Japanese', 'Korean', 'Portuguese', 'Russian', 'Spanish', 'Turkish'][hash(row|string) % 12] }}\n***\n"
+  max_seq_length: 8192
+  shuffle: true
+  seed: 42
+  datasets:
+    
+    # Category Summary (Total: 590 samples)
+    # =====================================================
+    # General chat (24 samples - 4.07%)
+    # Instruction and Reasoning tuning (14 samples - 2.37%)
+    # Multilingual (36 samples - 6.10%)
+    # Tool use (100 samples - 16.95%)
+    # Code / Programming / Software Engineering / Devops (328 samples - 55.59%)
+    # Math (12 samples - 2.03%)
+    # Sciences (16 samples - 2.71%)
+    # Medical (8 samples - 1.36%)
+    # Finance (8 samples - 1.36%)
+    # Business (16 samples - 2.71%)
+    # Humanities and Philosophy (8 samples - 1.36%)
+    # Creative Writing, Adventure, Roleplay (13 samples - 2.20%)
+    # General Knowledge and Pop Culture (2 samples - 0.34%)
+    # Specialized skills (4 samples - 0.68%)
+    # Misc (1 sample - 0.17%)
+    # =====================================================
+    
+    # Research
+    # =====================================================
+    # According to this presentation https://minjiazhang.github.io/courses/fall24-resource/slides/awq.pdf
+    # AWQ only needs 64 samples to identify salient weights that need to be preserved.
+    # 
+    # This research predates the boom of MoE (Mixture-of-Experts) models
+    # and it's safer to assume that 64 samples of a general dataset 
+    # cannot properly identify salient weights of experts.
+    
+    # General chat (24 samples)
+    # ---------------------------------------------------------------------------
+    - dataset: HuggingFaceH4/ultrachat_200k
+      columns: [messages]
+      split: train_sft
+      formatter: chat_completion
+      num_samples: 8
+      streaming: true
+
+    - dataset: databricks/databricks-dolly-15k
+      split: train
+      columns: [instruction, response]
+      formatter: prompt_answer
+      num_samples: 8
+
+    - dataset: neuralmagic/calibration
+      subset: LLM
+      split: train
+      columns: [messages]
+      formatter: chat_completion
+      num_samples: 8
+
+    # Instruction and Reasoning tuning (14 samples)
+    # ---------------------------------------------------------------------------
+    - dataset: HuggingFaceH4/no_robots
+      split: train
+      columns: [messages]
+      formatter: chat_completion
+      num_samples: 2
+
+    - dataset: nvidia/HelpSteer
+      split: train
+      columns: [prompt, response]
+      formatter: prompt_answer
+      num_samples: 2
+      streaming: true
+
+    - dataset: garage-bAInd/Open-Platypus
+      split: train
+      columns: [instruction, output]
+      formatter: prompt_answer
+      num_samples: 2
+
+    - dataset: PJMixers/grimulkan_physical-reasoning-ShareGPT
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 4
+
+    - dataset: PJMixers/grimulkan_theory-of-mind-ShareGPT
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 4
+
+    # Multilingual (36 samples)
+    # ---------------------------------------------------------------------------
+    - dataset: HuggingFaceH4/Multilingual-Thinking
+      split: train
+      columns: [user]
+      formatter: raw_text
+      num_samples: 32
+      formatter_params:
+        prefix: *spoken_languages
+
+    - dataset: ServiceNow-AI/M2Lingual
+      subset: full_data
+      split: train
+      columns: [conversation]
+      formatter: chat_completion
+      num_samples: 4
+      streaming: true
+
+    # Tool use (include commented out ToolAce) (100 samples)
+    # ---------------------------------------------------------------------------
+
+    # Fail with minimax!
+    # jinja2.exceptions.TemplateError: Message has tool role, but there was no previous assistant message with a tool call!
+    # - dataset: Team-ACE/ToolACE
+    #   split: train
+    #   columns: [system, conversations]
+    #   formatter: chat_completion_with_sysprompt
+    #   num_samples: 100
+
+    - dataset: interstellarninja/hermes_reasoning_tool_use
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 100
+      streaming: true
+
+    # Code / Programming / Software Engineering / Devops (336 samples)
+    # ---------------------------------------------------------------------------
+
+    - dataset: deepmind/code_contests
+      split: train
+      columns: [name]
+      formatter: deepmind_code_contests
+      num_samples: 50
+      streaming: true
+
+    - dataset: dh02391735/stackoverflow-kubernetes-questions
+      split: train
+      columns: [instruction]
+      formatter: raw_text
+      num_samples: 8
+      streaming: true
+
+    - dataset: diversoailab/humaneval-rust
+      split: train
+      columns: [prompt]
+      formatter: raw_text
+      num_samples: 100
+      formatter_params: # The dataset actually doesn't hardcode the language
+        prefix: *programming_languages
+
+    - dataset: ammarnasr/the-stack-rust-clean
+      split: train
+      columns: [content]
+      formatter: raw_text
+      num_samples: 8
+      streaming: true
+      formatter_params:
+        prefix: "Explain this code and comment it for a junior dev.\n***\n"
+
+    - dataset: CSJianYang/CodeArena
+      split: test
+      columns: [messages]
+      formatter: chat_completion
+      num_samples: 8
+
+    - dataset: nvidia/OpenCodeInstruct
+      split: train
+      columns: [input, output]
+      formatter: prompt_answer
+      num_samples: 8
+      streaming: true
+
+    - dataset: nvidia/Llama-Nemotron-Post-Training-Dataset
+      split: code
+      columns: [input]
+      formatter: chat_completion
+      num_samples: 8
+      streaming: true
+
+    - dataset: nvidia/Nemotron-Competitive-Programming-v1
+      split: competitive_coding_cpp_part00
+      columns: [messages]
+      formatter: chat_completion
+      num_samples: 8
+      streaming: true
+
+    # The conversations columns has another "conversations" field :/
+    # - dataset: sr5434/CodegebraGPT_data
+    #   subset: 100k-text
+    #   split: train
+    #   columns: [conversations]
+    #   formatter: sharegpt
+    #   num_samples: 8
+
+    - dataset: rombodawg/code_bagel_hermes-2.5
+      split: train
+      columns: [input, output]
+      formatter: prompt_answer
+      num_samples: 100
+      streaming: true
+
+    - dataset: MathArena/project_euler
+      split: train
+      columns: [problem]
+      formatter: raw_text
+      num_samples: 30
+      formatter_params:
+        prefix: *programming_languages
+
+    # Math (12 samples)
+    - dataset: nvidia/Llama-Nemotron-Post-Training-Dataset
+      split: math
+      columns: [input]
+      formatter: chat_completion
+      num_samples: 4
+      streaming: true
+
+    - dataset: nvidia/Nemotron-Math-Proofs-v1
+      split: lean
+      columns: [formal_statement]
+      formatter: raw_text
+      num_samples: 4
+      streaming: true
+      formatter_params:
+        prefix: "Can you improve, document and add comment to this Lean proof for a non-mathematician?\n***\n"
+
+    - dataset: nvidia/OpenMathInstruct-2
+      split: train
+      columns: [problem, generated_solution]
+      formatter: prompt_answer
+      num_samples: 4
+      streaming: true
+
+    # Sciences (16 samples)
+    - dataset: nvidia/Llama-Nemotron-Post-Training-Dataset
+      split: science
+      columns: [input]
+      formatter: chat_completion
+      num_samples: 4
+      streaming: true
+
+    - dataset: nvidia/OpenScienceReasoning-2
+      split: train
+      columns: [input, output]
+      formatter: prompt_answer
+      num_samples: 8
+      streaming: true
+
+    - dataset: MegaScience/MegaScience
+      split: train
+      columns: [question, answer]
+      formatter: prompt_answer
+      num_samples: 4
+      streaming: true
+
+    # Medical (8 samples)
+    - dataset: OpenMed/Medical-Reasoning-SFT-GPT-OSS-120B
+      split: train
+      columns: [messages]
+      formatter: chat_completion
+      num_samples: 4
+      streaming: true
+
+    - dataset: ccdv/pubmed-summarization
+      subset: section
+      split: train
+      columns: [article]
+      formatter: raw_text
+      num_samples: 4
+      streaming: true
+      formatter_params:
+        prefix: "Summarize this:\n***\n"
+
+    # Finance (8 samples)
+    - dataset: gbharti/finance-alpaca
+      split: train
+      columns: [instruction, output]
+      formatter: prompt_answer
+      num_samples: 4
+
+    - dataset: vladlen32230/summarization-yahoo-stock-finance-article-text
+      split: train
+      columns: [text]
+      formatter: raw_text
+      num_samples: 4
+      formatter_params:
+        prefix: "Summarize this:\n***\n"
+
+    # Business (16 samples)
+    - dataset: fka/awesome-chatgpt-prompts
+      split: train
+      columns: [prompt]
+      formatter: raw_text
+      num_samples: 8
+
+    - dataset: theoldmandthesea/17k_business_book
+      split: train
+      columns: [question, answer]
+      formatter: prompt_answer
+      num_samples: 8
+
+    # Humanities and Philosophy (8 samples)
+    - dataset: ruggsea/stanford-encyclopedia-of-philosophy_instruct
+      split: train
+      columns: [question, answer]
+      formatter: prompt_answer
+      num_samples: 2
+      streaming: true
+
+    - dataset: mlfoundations-dev/stackexchange_philosophy
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 2
+
+    - dataset: FreedomIntelligence/SocraticChat
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 4
+      streaming: true
+
+    # Creative Writing, Adventure, Roleplay (13 samples)
+    - dataset: Gryphe/Opus-WritingPrompts
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 2
+
+    - dataset: anthracite-org/nopm_claude_writing_fixed
+      split: train
+      columns: [conversations]
+      formatter: sharegpt 
+      num_samples: 2
+
+    - dataset: zerofata/Roleplay-Anime-Characters
+      split: train
+      columns: [messages]
+      formatter: chat_completion
+      num_samples: 1
+
+    - dataset: zerofata/Instruct-Anime
+      split: train
+      columns: [messages]
+      formatter: chat_completion
+      num_samples: 1
+
+    - dataset: zerofata/Instruct-Anime-CreativeWriting
+      split: train
+      columns: [messages]
+      formatter: chat_completion
+      num_samples: 1
+
+    - dataset: sam-paech/gutenberg3-generalfiction-scifi-fantasy-romance-adventure-dpo
+      split: train
+      columns: [chosen]
+      formatter: chat_completion
+      num_samples: 2
+
+    - dataset: PocketDoc/Dans-Prosemaxx-Adventure
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 2
+
+    - dataset: anthracite-org/stheno-filtered-v1.1
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 2
+      streaming: true
+
+    # General Knowledge and Pop Culture (2 samples)
+    - dataset: KaraKaraWitch/TvTroper-2025
+      split: train
+      columns: [article]
+      formatter: raw_text
+      num_samples: 2
+      streaming: true
+      formatter_params:
+        prefix: "Explain this trope like I'm your grandmother\n***\n"
+
+    # Behavioral skills (8 samples)
+    - dataset: AquaV/US-Army-Survival-Sharegpt
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 1
+
+    - dataset: AquaV/Interrogation-Sharegpt
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 1
+
+    - dataset: AquaV/Multi-Environment-Operations-Sharegpt
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 1
+
+    - dataset: AquaV/Resistance-Sharegpt
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 1
+
+    # Misc (1 sample)
+    - dataset: PocketDoc/Dans-Kinomaxx-VanillaBackrooms
+      split: train
+      columns: [conversations]
+      formatter: sharegpt
+      num_samples: 1