Merge pull request #79 from ssbuild/dev

support skywork

README.md

@@ -25,7 +25,8 @@ pip install -U git+https://github.com/ssbuild/deep_training.git --no-deps --forc
     - support object detection 完整训练 https://github.com/ssbuild/detection_finetuning
     - support semantic segmentation 完整训练 https://github.com/ssbuild/semantic_segmentation
     - support chatglm3  完整训练 https://github.com/ssbuild/chatglm3_finetuning
-
+  - 0.2.7.post0
+    - support skywork 完整训练 https://github.com/ssbuild/skywork_finetuning
 
 - <strong>2023-10-16</strong>
   - 0.2.6 support muti-model

setup.py

@@ -21,7 +21,7 @@
 ]
 setup(
     name='deep_training',
-    version='0.2.7',
+    version='0.2.7.post0',
     description='an easy training architecture',
     long_description='torch_training: https://github.com/ssbuild/deep_training.git',
     license='Apache License 2.0',

src/deep_training/nlp/models/chatglm3/modeling_chatglm.py

@@ -1001,7 +1001,7 @@ def _reorder_cache(
 
     def process_response(self, output, history):
         content = ""
-        history = deepcopy(history)
+        history = copy.deepcopy(history)
         for response in output.split("<|assistant|>"):
             metadata, content = response.split("\n", maxsplit=1)
             if not metadata.strip():
@@ -1012,44 +1012,53 @@ def process_response(self, output, history):
                 history.append({"role": "assistant", "metadata": metadata, "content": content})
                 if history[0]["role"] == "system" and "tools" in history[0]:
                     content = "\n".join(content.split("\n")[1:-1])
+
                     def tool_call(**kwargs):
                         return kwargs
+
                     parameters = eval(content)
                     content = {"name": metadata.strip(), "parameters": parameters}
                 else:
                     content = {"name": metadata.strip(), "content": content}
         return content, history
 
     @torch.inference_mode()
-    def chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, role: str = "user",logits_processor=None,
-             **kwargs):
+    def chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, role: str = "user",
+             logits_processor=None, with_postprocess=True, **kwargs):
         if history is None:
             history = []
         if logits_processor is None:
             logits_processor = LogitsProcessorList()
+
+        if "eos_token_id" not in kwargs:
+            eos_token_id = [tokenizer.eos_token_id, tokenizer.get_command("<|user|>"),
+                            tokenizer.get_command("<|observation|>")]
+            kwargs["eos_token_id"] = eos_token_id
         logits_processor.append(InvalidScoreLogitsProcessor())
         gen_kwargs = {"logits_processor": logits_processor, **kwargs}
         inputs = tokenizer.build_chat_input(query, history=history, role=role)
         inputs = inputs.to(self.device)
-        eos_token_id = [tokenizer.eos_token_id, tokenizer.get_command("<|user|>"),
-                        tokenizer.get_command("<|observation|>")]
-        outputs = self.generate(**inputs, **gen_kwargs, eos_token_id=eos_token_id)
+        outputs = self.generate(**inputs, **gen_kwargs)
         outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):-1]
         response = tokenizer.decode(outputs)
         history.append({"role": role, "content": query})
-        response, history = self.process_response(response, history)
+        if with_postprocess:
+            response, history = self.process_response(response, history)
         return response, history
 
     @torch.inference_mode()
-    def stream_chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, role: str = "user",past_key_values=None,
-                    logits_processor=None, return_past_key_values=False, **kwargs):
+    def stream_chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, role: str = "user",
+                    past_key_values=None,
+                    logits_processor=None, return_past_key_values=False, with_postprocess=True, **kwargs):
         if history is None:
             history = []
         if logits_processor is None:
             logits_processor = LogitsProcessorList()
         logits_processor.append(InvalidScoreLogitsProcessor())
-        eos_token_id = [tokenizer.eos_token_id, tokenizer.get_command("<|user|>"),
-                        tokenizer.get_command("<|observation|>")]
+        if "eos_token_id" not in kwargs:
+            eos_token_id = [tokenizer.eos_token_id, tokenizer.get_command("<|user|>"),
+                            tokenizer.get_command("<|observation|>")]
+            kwargs["eos_token_id"] = eos_token_id
         gen_kwargs = {"logits_processor": logits_processor, **kwargs}
         if past_key_values is None:
             inputs = tokenizer.build_chat_input(query, history=history, role=role)
@@ -1066,14 +1075,17 @@ def stream_chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = No
             inputs['attention_mask'] = attention_mask
         history.append({"role": role, "content": query})
         for outputs in self.stream_generate(**inputs, past_key_values=past_key_values,
-                                            eos_token_id=eos_token_id, return_past_key_values=return_past_key_values,
+                                            return_past_key_values=return_past_key_values,
                                             **gen_kwargs):
             if return_past_key_values:
                 outputs, past_key_values = outputs
             outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):-1]
             response = tokenizer.decode(outputs)
             if response and response[-1] != "�":
-                response, new_history = self.process_response(response, history)
+                if with_postprocess:
+                    response, new_history = self.process_response(response, history)
+                else:
+                    new_history = history
                 if return_past_key_values:
                     yield response, new_history, past_key_values
                 else:

src/deep_training/nlp/models/skywork/__init__.py

@@ -0,0 +1,4 @@
+# coding=utf8
+# @Time    : 2023/10/31 1:51
+# @Author  : tk
+# @FileName: __init__.py

src/deep_training/nlp/models/skywork/configuration_skywork.py

@@ -0,0 +1,89 @@
+# Copyright (c) SkyworkAI and the HuggingFace Inc. team. All rights reserved.
+# This code is built upon Huggingface's transformers repository.
+
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+
+
+class SkyworkConfig(PretrainedConfig):
+
+    model_type = "skywork"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        hidden_act="silu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=1,
+        eos_token_id=2,
+        pretraining_tp=1,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.pretraining_tp = pretraining_tp
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self._rope_scaling_validation()
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    def _rope_scaling_validation(self):
+        """
+        Validate the `rope_scaling` configuration.
+        """
+        if self.rope_scaling is None:
+            return
+
+        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
+            raise ValueError(
+                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
+                f"got {self.rope_scaling}"
+            )
+        rope_scaling_type = self.rope_scaling.get("type", None)
+        rope_scaling_factor = self.rope_scaling.get("factor", None)
+        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic", "ntk"]:
+            raise ValueError(
+                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
+            )
+        if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
+            raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}")