From 0c86a4fb200d6d5e983c204be433230b7e2f9b55 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Thu, 26 Sep 2024 18:47:18 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=94=A7=20refactor:=20remove=20unused?= =?UTF-8?q?=20'Union'=20import=20in=20=5Fcost.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🔧 refactor: add exception detail in decode error log in __init__.py --- src/novelai_python/sdk/ai/_cost.py | 4 ++-- src/novelai_python/utils/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/novelai_python/sdk/ai/_cost.py b/src/novelai_python/sdk/ai/_cost.py index 353a278..8963110 100644 --- a/src/novelai_python/sdk/ai/_cost.py +++ b/src/novelai_python/sdk/ai/_cost.py @@ -1,11 +1,11 @@ import math import random -from typing import List, Optional, Union +from typing import List, Optional from pydantic import BaseModel from novelai_python.sdk.ai._const import map, initialN, initial_n, step, newN -from novelai_python.sdk.ai._enum import Sampler, Model, ModelGroups, get_model_group, ModelTypeAlias +from novelai_python.sdk.ai._enum import Sampler, ModelGroups, get_model_group, ModelTypeAlias class Args(BaseModel): diff --git a/src/novelai_python/utils/__init__.py b/src/novelai_python/utils/__init__.py index 788d268..dcebdf2 100755 --- a/src/novelai_python/utils/__init__.py +++ b/src/novelai_python/utils/__init__.py @@ -21,7 +21,7 @@ def try_jsonfy(obj: Union[str, dict, list, tuple], default_when_error=None): try: return json.loads(obj) except Exception as e: - logger.trace(f"Decode Error {obj}") + logger.trace(f"Decode Error {obj} {e}") if default_when_error is None: return f"Decode Error {type(obj)}" else: From 4c977cce2af904e8b3ed2d6e05fc3b8f2c1fe35a Mon Sep 17 00:00:00 2001 From: sudoskys Date: Thu, 26 Sep 2024 19:51:18 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E2=9C=A8=20feat:=20improve=20error=20handl?= =?UTF-8?q?ing=20and=20add=20model=20endpoint=20normalization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update status code check to include 201 in generate_stream.py - Add endpoint normalization for specific models in generate/__init__.py - Change logger level from debug to trace for out of range tokens --- .../sdk/ai/generate/__init__.py | 19 +++++++++++++------ src/novelai_python/sdk/ai/generate_stream.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/novelai_python/sdk/ai/generate/__init__.py b/src/novelai_python/sdk/ai/generate/__init__.py index 866b7b8..cd42989 100644 --- a/src/novelai_python/sdk/ai/generate/__init__.py +++ b/src/novelai_python/sdk/ai/generate/__init__.py @@ -83,6 +83,13 @@ async def necessary_headers(self, request_data) -> dict: @model_validator(mode="after") def normalize_model(self): + if self.model in [ + TextLLMModel.NEO_2B, TextLLMModel.J_6B, TextLLMModel.J_6B_V3, TextLLMModel.J_6B_V4, + TextLLMModel.GENJI_JP_6B, TextLLMModel.GENJI_JP_6B_V2, TextLLMModel.GENJI_PYTHON_6B, + TextLLMModel.EUTERPE_V0, TextLLMModel.EUTERPE_V2, TextLLMModel.KRAKE_V1, TextLLMModel.KRAKE_V2, + TextLLMModel.CASSANDRA, TextLLMModel.COMMENT_BOT, TextLLMModel.INFILL, TextLLMModel.CLIO + ]: + self.endpoint = "https://api.novelai.net" tokenizer = NaiTokenizer(get_tokenizer_model(self.model)) model_group = get_llm_group(self.model) total_tokens = len(tokenizer.encode(self.input)) @@ -238,10 +245,10 @@ def normalize_model(self): valid_sequences = [] for cell in logit_bias_group_exp: - if not any(token < 0 or token >= total_tokens for token in cell.sequence): + if any(token < 0 or token >= total_tokens for token in cell.sequence): # 超出范围的 Token - logger.debug( - f"Bias {cell} contains tokens that are out of range and will be ignored." + logger.trace( + f"Bias [{cell}] contains tokens that are out of range and will be ignored." ) else: # 将有效偏置组添加到列表 @@ -258,8 +265,8 @@ def normalize_model(self): valid_bad_words_ids = [] self.parameters.bad_words_ids = bad_words_ids for ban_word in self.parameters.bad_words_ids: - if not any(token < 0 or token >= total_tokens for token in ban_word): - logger.warning( + if any(token < 0 or token >= total_tokens for token in ban_word): + logger.trace( f"Bad word {ban_word} contains tokens that are out of range and will be ignored." ) else: @@ -358,7 +365,7 @@ async def request(self, message = response_data.get("error") or response_data.get( "message") or f"Server error with status_code {response.status_code}" status_code = response_data.get("statusCode", response.status_code) - if status_code == 200: + if status_code == 200 or status_code == 201: output = response_data.get("output", None) if not output: raise APIError( diff --git a/src/novelai_python/sdk/ai/generate_stream.py b/src/novelai_python/sdk/ai/generate_stream.py index dbb8a96..138dd1c 100644 --- a/src/novelai_python/sdk/ai/generate_stream.py +++ b/src/novelai_python/sdk/ai/generate_stream.py @@ -134,7 +134,7 @@ async def request(self, request=request_data, code=status_code, response=message ) else: - if response.status_code != 200: + if response.status_code not in [200, 201]: raise APIError( f"Server error with status code {response.status_code}", request=request_data, code=response.status_code, response=response.content From 601456cf69c0e62b37dd95857edd1ca3c48845ca Mon Sep 17 00:00:00 2001 From: sudoskys Date: Thu, 26 Sep 2024 20:24:38 +0800 Subject: [PATCH 3/3] :rocket: feat(tokenizer): add method to fetch total vocabulary size Added get_vocab method to SimpleTokenizer and total_tokens method to NaiTokenizer for retrieving vocabulary size. Updated schema validations in _schema.py for stricter type enforcement. Bumped project version to 0.5.1 in pyproject.toml. --- pyproject.toml | 2 +- .../sdk/ai/generate/__init__.py | 6 ++- src/novelai_python/sdk/ai/generate/_schema.py | 45 +++++++++---------- src/novelai_python/tokenizer/__init__.py | 7 +++ .../tokenizer/clip_simple_tokenizer.py | 8 ++++ 5 files changed, 41 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0db095c..447ca82 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "novelai-python" -version = "0.5.0" +version = "0.5.1" description = "NovelAI Python Binding With Pydantic" authors = [ { name = "sudoskys", email = "coldlando@hotmail.com" }, diff --git a/src/novelai_python/sdk/ai/generate/__init__.py b/src/novelai_python/sdk/ai/generate/__init__.py index cd42989..baea3e9 100644 --- a/src/novelai_python/sdk/ai/generate/__init__.py +++ b/src/novelai_python/sdk/ai/generate/__init__.py @@ -92,7 +92,7 @@ def normalize_model(self): self.endpoint = "https://api.novelai.net" tokenizer = NaiTokenizer(get_tokenizer_model(self.model)) model_group = get_llm_group(self.model) - total_tokens = len(tokenizer.encode(self.input)) + total_tokens = tokenizer.total_tokens() if isinstance(self.input, str): prompt = tokenizer.encode(self.input) dtype = "uint32" if self.model in [TextLLMModel.ERATO] else "uint16" @@ -284,6 +284,10 @@ def normalize_model(self): self.advanced_setting.num_logprobs = self.logprobs_count if not self.advanced_setting.max_length: self.advanced_setting.max_length = 40 + if self.parameters.repetition_penalty_range == 0: + self.parameters.repetition_penalty_range = None + if self.parameters.repetition_penalty_slope == 0: + self.parameters.repetition_penalty_slope = None return self @classmethod diff --git a/src/novelai_python/sdk/ai/generate/_schema.py b/src/novelai_python/sdk/ai/generate/_schema.py index 2d18786..c8e460a 100644 --- a/src/novelai_python/sdk/ai/generate/_schema.py +++ b/src/novelai_python/sdk/ai/generate/_schema.py @@ -2,7 +2,7 @@ from enum import Enum from typing import List, Optional, Tuple, Union -from pydantic import BaseModel, Field, model_validator, ConfigDict +from pydantic import BaseModel, Field, ConfigDict """ class StorySettings(BaseModel): @@ -77,11 +77,6 @@ class LogitBiasGroup(BaseModel): ensure_sequence_finish: bool generate_once: bool - @model_validator(mode="before") - def validate_sequence(cls, value): - print(value) - return value - class AdvanceLLMSetting(BaseModel): """ @@ -89,7 +84,7 @@ class AdvanceLLMSetting(BaseModel): """ min_length: Optional[int] = 1 max_length: Optional[int] = None - repetition_penalty: Optional[float] = None + repetition_penalty: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) generate_until_sentence: Optional[bool] = True use_cache: Optional[bool] = False use_string: Optional[bool] = False @@ -107,36 +102,36 @@ class LLMGenerationParams(BaseModel): LLM Generation Settings """ textGenerationSettingsVersion: Optional[int] = None - temperature: Optional[float] = None + temperature: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) max_length: Optional[int] = Field(default=None, ge=1, le=100) # Max 150(vip3),100(vip2) min_length: Optional[int] = Field(default=None, ge=1, le=100) # Max 150(vip3),100(vip2) - top_k: Optional[float] = Field(default=None, ge=0) - top_p: Optional[float] = Field(default=None, gt=0, le=1) - top_a: Optional[float] = Field(default=None, ge=0) - typical_p: Optional[float] = Field(default=None, ge=0, le=1) - tail_free_sampling: Optional[float] = Field(default=None, ge=0) - repetition_penalty: Optional[float] = Field(default=None, gt=0) + top_k: Optional[int] = Field(default=None, ge=0) + top_p: Optional[Union[float, int]] = Field(default=None, gt=0, le=1, allow_inf_nan=False) + top_a: Optional[Union[float, int]] = Field(default=None, ge=0, allow_inf_nan=False) + typical_p: Optional[Union[float, int]] = Field(default=None, ge=0, le=1, allow_inf_nan=False) + tail_free_sampling: Optional[Union[float, int]] = Field(default=None, ge=0, allow_inf_nan=False) + repetition_penalty: Optional[Union[float, int]] = Field(default=None, gt=0, allow_inf_nan=False) repetition_penalty_range: Optional[int] = Field(default=None, ge=0) - repetition_penalty_slope: Optional[float] = Field(default=None, ge=0) + repetition_penalty_slope: Optional[Union[float, int]] = Field(default=None, ge=0, allow_inf_nan=False) eos_token_id: int = None bad_words_ids: List[List[int]] = None logit_bias_groups: Optional[List[LogitBiasGroup]] = [] - repetition_penalty_frequency: Optional[float] = None - repetition_penalty_presence: Optional[float] = None + repetition_penalty_frequency: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) + repetition_penalty_presence: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) repetition_penalty_whitelist: Optional[List[int]] = None repetition_penalty_default_whitelist: Optional[bool] = None - cfg_scale: Optional[float] = None + cfg_scale: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) cfg_uc: Optional[str] = None phrase_rep_pen: PenStyle = PenStyle.Off - top_g: Optional[float] = None - mirostat_tau: Optional[float] = None - mirostat_lr: Optional[float] = None - math1_temp: Optional[float] = None - math1_quad: Optional[float] = None - math1_quad_entropy_scale: Optional[float] = None - min_p: Optional[float] = None + top_g: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) + mirostat_tau: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) + mirostat_lr: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) + math1_temp: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) + math1_quad: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) + math1_quad_entropy_scale: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) + min_p: Optional[Union[float, int]] = Field(default=None, allow_inf_nan=False) order: Union[List[int], List[KeyOrderEntry]] = [ KeyOrderEntry(id=Key.Cfg, enabled=False), diff --git a/src/novelai_python/tokenizer/__init__.py b/src/novelai_python/tokenizer/__init__.py index fc88a30..7f77ab2 100644 --- a/src/novelai_python/tokenizer/__init__.py +++ b/src/novelai_python/tokenizer/__init__.py @@ -187,6 +187,13 @@ def tokenize_text(self, text: str) -> List[int]: return self.tokenizer.encode(text).tokens raise NotImplementedError("Tokenizer does not support token encoding") + def total_tokens(self) -> int: + if isinstance(self.tokenizer, Tokenizer): + return len(self.tokenizer.get_vocab()) + if isinstance(self.tokenizer, SimpleTokenizer): + return len(self.tokenizer.get_vocab()) + raise NotImplementedError("Tokenizer does not support token encoding") + def encode(self, sentence: str) -> List[int]: if isinstance(self.tokenizer, SimpleTokenizer): return self.tokenizer.encode(sentence).ids diff --git a/src/novelai_python/tokenizer/clip_simple_tokenizer.py b/src/novelai_python/tokenizer/clip_simple_tokenizer.py index f0a9760..41197d2 100644 --- a/src/novelai_python/tokenizer/clip_simple_tokenizer.py +++ b/src/novelai_python/tokenizer/clip_simple_tokenizer.py @@ -144,3 +144,11 @@ def decode(self, tokens): text = ''.join([self.decoder[token] for token in tokens]) text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('', ' ') return text + + def get_vocab(self): + return self.encoder + + +if __name__ == '__main__': + tokenizer = SimpleTokenizer() + print(len(tokenizer.get_vocab())) \ No newline at end of file