From 396bfa2632073d0b4b55875b920762aec3cbaff7 Mon Sep 17 00:00:00 2001 From: Dazed Date: Fri, 3 Nov 2023 14:06:04 -0500 Subject: [PATCH] Rework how past test is injected --- modules/alltext.py | 82 ++++++++++++++++++++++++++++++++++++++++- modules/csv.py | 60 +++++++++++++++++++++--------- modules/json.py | 58 +++++++++++++++++++---------- modules/kansen.py | 51 +++++++++++++++++-------- modules/lune.py | 8 ++-- modules/rpgmakerace.py | 57 ++++++++++++++++++---------- modules/rpgmakermvmz.py | 40 +++++++++++--------- modules/txt.py | 72 +++++++++++++++++++++++------------- modules/tyrano.py | 52 ++++++++++++++++++-------- 9 files changed, 344 insertions(+), 136 deletions(-) diff --git a/modules/alltext.py b/modules/alltext.py index 424d831..1a1aaa9 100644 --- a/modules/alltext.py +++ b/modules/alltext.py @@ -89,4 +89,84 @@ def getResultString(translatedData, translationTime, filename): errorString = str(e) + '|' + translatedData[3] + Fore.RED return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\ errorString + Fore.RESET - \ No newline at end of file + +@retry(exceptions=Exception, tries=5, delay=5) +def translateGPT(t, history, fullPromptFlag): + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + enc = tiktoken.encoding_for_model("gpt-4") + tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) + return (t, tokens) + + # Sub Vars + varResponse = subVars(t) + subbedT = varResponse[0] + + # If there isn't any Japanese in the text just skip + if not re.search(r'[一-龠]+|[ぁ-?]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): + return(t, 0) + + # Characters + context = '```\ + Game Characters:\ + Character: 池ノ上 拓海 == Ikenoue Takumi - Gender: Male\ + Character: 福永 こはる == Fukunaga Koharu - Gender: Female\ + Character: 神泉 理央 == Kamiizumi Rio - Gender: Female\ + Character: 吉祥寺 アリサ == Kisshouji Arisa - Gender: Female\ + Character: 久我 友里子 == Kuga Yuriko - Gender: Female\ + ```' + + # Prompt + if fullPromptFlag: + system = PROMPT + user = 'Line to Translate = ' + subbedT + else: + system = 'Output ONLY the english translation in the following format: `Translation: `' + user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + + response = openai.ChatCompletion.create( + temperature=0.1, + frequency_penalty=0.2, + presence_penalty=0.2, + model="gpt-3.5-turbo", + messages=msg, + request_timeout=30, + ) + + # Save Translated Text + translatedText = response.choices[0].message.content + tokens = response.usage.total_tokens + + # Resub Vars + translatedText = resubVars(translatedText, varResponse[1]) + + # Remove Placeholder Text + translatedText = translatedText.replace('English Translation: ', '') + translatedText = translatedText.replace('Translation: ', '') + translatedText = translatedText.replace('Line to Translate = ', '') + translatedText = translatedText.replace('Translation = ', '') + translatedText = translatedText.replace('Translate = ', '') + translatedText = translatedText.replace('English Translation:', '') + translatedText = translatedText.replace('Translation:', '') + translatedText = translatedText.replace('Line to Translate =', '') + translatedText = translatedText.replace('Translation =', '') + translatedText = translatedText.replace('Translate =', '') + translatedText = re.sub(r'Note:.*', '', translatedText) + translatedText = translatedText.replace('っ', '') + + # Return Translation + if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: + raise Exception + else: + return [translatedText, tokens] \ No newline at end of file diff --git a/modules/csv.py b/modules/csv.py index 721bae7..171eef9 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -320,7 +320,7 @@ def resubVars(translatedText, allList): def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-4") tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, tokens) @@ -329,26 +329,44 @@ def translateGPT(t, history, fullPromptFlag): subbedT = varResponse[0] # If there isn't any Japanese in the text just skip - if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ繝ｼ]+', subbedT): + if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ]+|[\uFF00-\uFFEF]', subbedT): return(t, 0) - """Translate text using GPT""" - context = 'Eroge Names Context: 繝溘き繧ｨ繝ｫ == Mikael | Female, 繝溘き == Mika | Female, 繝吶Ν繧ｼ繝薙Η繝ｼ繝 == Beelzebuth | Female, 繝吶Ν == Bel | Female, 繧｢繧ｺ繝ｩ繧ｨ繝ｫ == Azriel | Female, 繧｢繧ｺ == Az | Female, 繝輔Ξ繧､繧｢ == Freya | Female' + # Characters + context = '```\ + Game Characters:\ + Character: 豎繝惹ｸ 諡捺ｵｷ == Ikenoue Takumi - Gender: Male\ + Character: 遖乗ｰｸ 縺薙ｯ繧 == Fukunaga Koharu - Gender: Female\ + Character: 逾樊ｳ 逅螟ｮ == Kamiizumi Rio - Gender: Female\ + Character: 蜷臥･･蟇ｺ 繧｢繝ｪ繧ｵ == Kisshouji Arisa - Gender: Female\ + Character: 荵謌 蜿矩悟ｭ == Kuga Yuriko - Gender: Female\ + ```' + + # Prompt if fullPromptFlag: - system = PROMPT - user = 'Current Text to Translate: ' + subbedT + system = PROMPT + user = 'Line to Translate = ' + subbedT else: - system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Current Text to Translate: ' + subbedT + system = 'Output ONLY the english translation in the following format: `Translation: `' + user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + response = openai.ChatCompletion.create( - temperature=0, + temperature=0.1, + frequency_penalty=0.2, + presence_penalty=0.2, model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": context}, - {"role": "user", "content": history}, - {"role": "user", "content": user} - ], + messages=msg, request_timeout=30, ) @@ -362,13 +380,19 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Current Text to Translate: ', '') + translatedText = translatedText.replace('Line to Translate = ', '') + translatedText = translatedText.replace('Translation = ', '') + translatedText = translatedText.replace('Translate = ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Current Text to Translate:', '') + translatedText = translatedText.replace('Line to Translate =', '') + translatedText = translatedText.replace('Translation =', '') + translatedText = translatedText.replace('Translate =', '') + translatedText = re.sub(r'Note:.*', '', translatedText) + translatedText = translatedText.replace('縺｣', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: - return [t, response.usage.total_tokens] + raise Exception else: return [translatedText, tokens] \ No newline at end of file diff --git a/modules/json.py b/modules/json.py index a58229f..b807d84 100644 --- a/modules/json.py +++ b/modules/json.py @@ -264,7 +264,7 @@ def resubVars(translatedText, allList): def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-4") tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, tokens) @@ -273,28 +273,44 @@ def translateGPT(t, history, fullPromptFlag): subbedT = varResponse[0] # If there isn't any Japanese in the text just skip - if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ繝ｼ]+', subbedT): + if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ]+|[\uFF00-\uFFEF]', subbedT): return(t, 0) - """Translate text using GPT""" - context = 'Eroge Names Context: Name: 遞ｲ逶 讌 == Inamori Kaede\nNicknames: 縺九∴縺｡繧繧 == Kae-chan or 縺九∴縺ｭ縺 == Kae-nee\nGender: Female,\nName: 遞ｲ逶 逵溷ｮ == Inamori Mamoru\nNicknames: 縺ｾ繝ｼ縺上ｓ == Maa-kun\nGender: Male,\nName: 闢ｮ隕 髮谺｡驛 == Hasumi Yujiro\nGender: Male,\nName: 譯占ｰｷ 諡馴ｦｬ == Kiriya Takuma\nNicknames: 縺溘▲縺上ｓ == Tak-kun\nGender: Male,\nName: 遞ｲ逶 鄒守事 == Inamori Yuki\nGender: Female,\nName: 螂･縺輔ｓ == Missus\nGender: Female' + # Characters + context = '```\ + Game Characters:\ + Character: 豎繝惹ｸ 諡捺ｵｷ == Ikenoue Takumi - Gender: Male\ + Character: 遖乗ｰｸ 縺薙ｯ繧 == Fukunaga Koharu - Gender: Female\ + Character: 逾樊ｳ 逅螟ｮ == Kamiizumi Rio - Gender: Female\ + Character: 蜷臥･･蟇ｺ 繧｢繝ｪ繧ｵ == Kisshouji Arisa - Gender: Female\ + Character: 荵謌 蜿矩悟ｭ == Kuga Yuriko - Gender: Female\ + ```' + + # Prompt if fullPromptFlag: - system = PROMPT - user = 'Line to Translate: ' + subbedT + system = PROMPT + user = 'Line to Translate = ' + subbedT else: - system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Line to Translate: ' + subbedT + system = 'Output ONLY the english translation in the following format: `Translation: `' + user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + response = openai.ChatCompletion.create( - temperature=0, + temperature=0.1, frequency_penalty=0.2, presence_penalty=0.2, model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": context}, - {"role": "user", "content": history}, - {"role": "user", "content": user} - ], + messages=msg, request_timeout=30, ) @@ -308,15 +324,19 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Line to Translate: ', '') + translatedText = translatedText.replace('Line to Translate = ', '') + translatedText = translatedText.replace('Translation = ', '') + translatedText = translatedText.replace('Translate = ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Line to Translate:', '') - translatedText = re.sub(r'\n\nPast Translated Text:.*', '', translatedText, 0, re.DOTALL) + translatedText = translatedText.replace('Line to Translate =', '') + translatedText = translatedText.replace('Translation =', '') + translatedText = translatedText.replace('Translate =', '') translatedText = re.sub(r'Note:.*', '', translatedText) + translatedText = translatedText.replace('縺｣', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: - return [t, response.usage.total_tokens] + raise Exception else: return [translatedText, tokens] \ No newline at end of file diff --git a/modules/kansen.py b/modules/kansen.py index 0f64071..6cf23ed 100644 --- a/modules/kansen.py +++ b/modules/kansen.py @@ -461,7 +461,7 @@ def resubVars(translatedText, allList): def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-4") tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, tokens) @@ -473,25 +473,41 @@ def translateGPT(t, history, fullPromptFlag): if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ]+|[\uFF00-\uFFEF]', subbedT): return(t, 0) - """Translate text using GPT""" - context = 'Eroge Names Context: (Name: 菴蝉ｼｯ 迸ｳ == Saeki Hitomi\nGender: Female, Name: 螻ｱ蟯ｸ 蜆ｪ == Yamagishi Yuu Gender: Female, Name: 莠泌香蠏 譛狗ｾ == Igarashi Tomomi Gender: Female, Name: 譁ｰ驕 繝ｪ繧ｵ == Shindou Risa Gender: Female, Name: 蟯ｸ逕ｰ 閨｡ == Kishida Satoshi Gender: Male, Name: 遶ｹ蜀 逵滉ｹ == Takeuchi Shinya Gender: Male, Name: 逕ｰ荳ｭ 逾蝉ｺ == Tanaka Yuuji Gender: Male)' + # Characters + context = '```\ + Game Characters:\ + Character: 豎繝惹ｸ 諡捺ｵｷ == Ikenoue Takumi - Gender: Male\ + Character: 遖乗ｰｸ 縺薙ｯ繧 == Fukunaga Koharu - Gender: Female\ + Character: 逾樊ｳ 逅螟ｮ == Kamiizumi Rio - Gender: Female\ + Character: 蜷臥･･蟇ｺ 繧｢繝ｪ繧ｵ == Kisshouji Arisa - Gender: Female\ + Character: 荵謌 蜿矩悟ｭ == Kuga Yuriko - Gender: Female\ + ```' + + # Prompt if fullPromptFlag: - system = PROMPT + system = PROMPT user = 'Line to Translate = ' + subbedT else: - system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Line to Translate: ' + subbedT + system = 'Output ONLY the english translation in the following format: `Translation: `' + user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + response = openai.ChatCompletion.create( - temperature=0, + temperature=0.1, frequency_penalty=0.2, presence_penalty=0.2, model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": context}, - {"role": "user", "content": history}, - {"role": "user", "content": user} - ], + messages=msg, request_timeout=30, ) @@ -505,11 +521,14 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Line to Translate: ', '') + translatedText = translatedText.replace('Line to Translate = ', '') + translatedText = translatedText.replace('Translation = ', '') + translatedText = translatedText.replace('Translate = ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Line to Translate:', '') - translatedText = re.sub(r'\n\nPast Translated Text:.*', '', translatedText, 0, re.DOTALL) + translatedText = translatedText.replace('Line to Translate =', '') + translatedText = translatedText.replace('Translation =', '') + translatedText = translatedText.replace('Translate =', '') translatedText = re.sub(r'Note:.*', '', translatedText) translatedText = translatedText.replace('縺｣', '') diff --git a/modules/lune.py b/modules/lune.py index 4e89e57..d593984 100644 --- a/modules/lune.py +++ b/modules/lune.py @@ -357,8 +357,11 @@ def translateGPT(t, history, fullPromptFlag): msg = [] msg.append({"role": "system", "content": system}) msg.append({"role": "user", "content": context}) - for line in history: - msg.append({"role": "user", "content": line}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) msg.append({"role": "user", "content": user}) response = openai.ChatCompletion.create( @@ -388,7 +391,6 @@ def translateGPT(t, history, fullPromptFlag): translatedText = translatedText.replace('Line to Translate =', '') translatedText = translatedText.replace('Translation =', '') translatedText = translatedText.replace('Translate =', '') - translatedText = re.sub(r'\n\nPast Translated Text:.*', '', translatedText, 0, re.DOTALL) translatedText = re.sub(r'Note:.*', '', translatedText) translatedText = translatedText.replace('縺｣', '') diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index f6ad602..e770a83 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -1504,7 +1504,7 @@ def resubVars(translatedText, allList): def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-4") tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, tokens) @@ -1513,28 +1513,44 @@ def translateGPT(t, history, fullPromptFlag): subbedT = varResponse[0] # If there isn't any Japanese in the text just skip - if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ]+', subbedT): + if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ]+|[\uFF00-\uFFEF]', subbedT): return(t, 0) - """Translate text using GPT""" - context = 'Eroge Names Context: Name: 繧ｻ繝ｬ繝 == Serena\nGender: Female,\n\nName: 繧ｫ繝医Μ繝ｼ繝 == Catherine\nGender: Female' + # Characters + context = '```\ + Game Characters:\ + Character: 豎繝惹ｸ 諡捺ｵｷ == Ikenoue Takumi - Gender: Male\ + Character: 遖乗ｰｸ 縺薙ｯ繧 == Fukunaga Koharu - Gender: Female\ + Character: 逾樊ｳ 逅螟ｮ == Kamiizumi Rio - Gender: Female\ + Character: 蜷臥･･蟇ｺ 繧｢繝ｪ繧ｵ == Kisshouji Arisa - Gender: Female\ + Character: 荵謌 蜿矩悟ｭ == Kuga Yuriko - Gender: Female\ + ```' + + # Prompt if fullPromptFlag: - system = PROMPT - user = 'Line to Translate: ' + subbedT + system = PROMPT + user = 'Line to Translate = ' + subbedT else: - system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Line to Translate: ' + subbedT + system = 'Output ONLY the english translation in the following format: `Translation: `' + user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + response = openai.ChatCompletion.create( - temperature=0, + temperature=0.1, frequency_penalty=0.2, presence_penalty=0.2, model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": context}, - {"role": "user", "content": history}, - {"role": "user", "content": user} - ], + messages=msg, request_timeout=30, ) @@ -1548,16 +1564,19 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Line to Translate: ', '') + translatedText = translatedText.replace('Line to Translate = ', '') + translatedText = translatedText.replace('Translation = ', '') + translatedText = translatedText.replace('Translate = ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Line to Translate:', '') - translatedText = re.sub(r'\n\nPast Translated Text:.*', '', translatedText, 0, re.DOTALL) + translatedText = translatedText.replace('Line to Translate =', '') + translatedText = translatedText.replace('Translation =', '') + translatedText = translatedText.replace('Translate =', '') translatedText = re.sub(r'Note:.*', '', translatedText) translatedText = translatedText.replace('縺｣', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: - return [t, response.usage.total_tokens] + raise Exception else: return [translatedText, tokens] \ No newline at end of file diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 53f0c68..26716cd 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -1686,7 +1686,7 @@ def resubVars(translatedText, allList): def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-4") tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, tokens) @@ -1698,36 +1698,41 @@ def translateGPT(t, history, fullPromptFlag): if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ]+|[\uFF00-\uFFEF]', subbedT): return(t, 0) - """Translate text using GPT""" + # Characters context = '```\ Game Characters:\ - Character: 螯よ怦莠憺梧 == Kisaragi Aria - Nickname: Aria - Gender: Female\ - Character: 諢帶ｴｲ鄒主ｽ悟ｭ == Aisu Miyako - Gender: Female\ - Character: 蝟憺♀蜷榊ｿ == Cocoa Kiyuna - Gender: Female\ - Character: 譟ｵ轢ｬ諢幄牡 == Ai Sakurai - Gender: Female\ - Character: 髯ｰ蟷ｳ蟆城棧 == Komari Kagehira - Gender: Female\ - Character: 險楢ｦ荳邵ｷ == Ichiru Kurube - Gender: Female\ - Character: 邱狗嚊譛 == Luna Hisube - Gender: Female\ - Character: 蛻台ｺ == Detective - Gender: Male\ + Character: 豎繝惹ｸ 諡捺ｵｷ == Ikenoue Takumi - Gender: Male\ + Character: 遖乗ｰｸ 縺薙ｯ繧 == Fukunaga Koharu - Gender: Female\ + Character: 逾樊ｳ 逅螟ｮ == Kamiizumi Rio - Gender: Female\ + Character: 蜷臥･･蟇ｺ 繧｢繝ｪ繧ｵ == Kisshouji Arisa - Gender: Female\ + Character: 荵謌 蜿矩悟ｭ == Kuga Yuriko - Gender: Female\ ```' + # Prompt if fullPromptFlag: system = PROMPT user = 'Line to Translate = ' + subbedT else: system = 'Output ONLY the english translation in the following format: `Translation: `' user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + response = openai.ChatCompletion.create( - temperature=0, + temperature=0.1, frequency_penalty=0.2, presence_penalty=0.2, model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": context}, - {"role": "user", "content": history}, - {"role": "user", "content": user} - ], + messages=msg, request_timeout=30, ) @@ -1749,7 +1754,6 @@ def translateGPT(t, history, fullPromptFlag): translatedText = translatedText.replace('Line to Translate =', '') translatedText = translatedText.replace('Translation =', '') translatedText = translatedText.replace('Translate =', '') - translatedText = re.sub(r'\n\nPast Translated Text:.*', '', translatedText, 0, re.DOTALL) translatedText = re.sub(r'Note:.*', '', translatedText) translatedText = translatedText.replace('縺｣', '') diff --git a/modules/txt.py b/modules/txt.py index c0daf2e..7e8fd49 100644 --- a/modules/txt.py +++ b/modules/txt.py @@ -303,41 +303,55 @@ def resubVars(translatedText, allList): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): - with LOCK: - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0613") - TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) - return (t, 0) + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + enc = tiktoken.encoding_for_model("gpt-4") + tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) + return (t, tokens) # Sub Vars varResponse = subVars(t) subbedT = varResponse[0] # If there isn't any Japanese in the text just skip - return (t,0) - if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ繝ｼ]+', subbedT): + if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ]+|[\uFF00-\uFFEF]', subbedT): return(t, 0) - """Translate text using GPT""" - context = 'Eroge Names Context: 螟乗ｨｹ 譏惹ｺｺ == Natsuki Akito | Male, 譛晞愆 遐ょ､懷ｭ == Asatsuyu Sayoko | Female, 逾樣 阯 == Jinno Ai | Female, 逾樣 闖ｫ | Jinno Sumire | Female, 螟乗ｨｹ 豬ｷ螟暮 == Natsuki Miyuri | Female, 豌ｴ譽ｮ 髯ｽ螟ｪ == Mizumori Youta | Male, 螟乗ｨｹ 蜥御ｺｺ == Natsuki Kazuto | Male, 驥主ｴ 蜊壻ｹ == Nozaki Hiroya | Male, 螟ｧ闖 繧ｸ繝･繝ｳ == Oogiku Jun | Male, 驟剃ｺ 邨ｹ莉｣ == Sakai Kinuyo | Female, 驟剃ｺ 雎 == Sakai Yutaka | Male, 遶懃函 譏･鄒ｩ == Tatsuki Haruyoshi | Male, 遶懃函 貎､ == Tatsuki Jun | Male, 蜷画ｲ｢ 闍ｱ邇 == Yoshizawa Eigen | Male, 蜷画ｲ｢ 豁ｦ髮 == Yoshizawa Takeo | Male' + # Characters + context = '```\ + Game Characters:\ + Character: 豎繝惹ｸ 諡捺ｵｷ == Ikenoue Takumi - Gender: Male\ + Character: 遖乗ｰｸ 縺薙ｯ繧 == Fukunaga Koharu - Gender: Female\ + Character: 逾樊ｳ 逅螟ｮ == Kamiizumi Rio - Gender: Female\ + Character: 蜷臥･･蟇ｺ 繧｢繝ｪ繧ｵ == Kisshouji Arisa - Gender: Female\ + Character: 荵謌 蜿矩悟ｭ == Kuga Yuriko - Gender: Female\ + ```' + + # Prompt if fullPromptFlag: - system = PROMPT - user = 'Line to Translate: ' + subbedT + system = PROMPT + user = 'Line to Translate = ' + subbedT else: - system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Line to Translate: ' + subbedT + system = 'Output ONLY the english translation in the following format: `Translation: `' + user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + response = openai.ChatCompletion.create( - temperature=0, - frequency_penalty=1, + temperature=0.1, + frequency_penalty=0.2, + presence_penalty=0.2, model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": context}, - {"role": "user", "content": history}, - {"role": "user", "content": user} - ], + messages=msg, request_timeout=30, ) @@ -351,13 +365,19 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Line to Translate: ', '') + translatedText = translatedText.replace('Line to Translate = ', '') + translatedText = translatedText.replace('Translation = ', '') + translatedText = translatedText.replace('Translate = ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Line to Translate:', '') + translatedText = translatedText.replace('Line to Translate =', '') + translatedText = translatedText.replace('Translation =', '') + translatedText = translatedText.replace('Translate =', '') + translatedText = re.sub(r'Note:.*', '', translatedText) + translatedText = translatedText.replace('縺｣', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: - return [t, response.usage.total_tokens] + raise Exception else: return [translatedText, tokens] \ No newline at end of file diff --git a/modules/tyrano.py b/modules/tyrano.py index 80ebc3f..477854c 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -468,7 +468,7 @@ def resubVars(translatedText, allList): def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-4") tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, tokens) @@ -480,25 +480,41 @@ def translateGPT(t, history, fullPromptFlag): if not re.search(r'[荳-鮴]+|[縺-繧脳+|[繧｡-繝ｴ]+|[\uFF00-\uFFEF]', subbedT): return(t, 0) - """Translate text using GPT""" - context = 'Eroge Names Context: (Name: 菴蝉ｼｯ 迸ｳ == Saeki Hitomi\nGender: Female, Name: 螻ｱ蟯ｸ 蜆ｪ == Yamagishi Yuu Gender: Female, Name: 莠泌香蠏 譛狗ｾ == Igarashi Tomomi Gender: Female, Name: 譁ｰ驕 繝ｪ繧ｵ == Shindou Risa Gender: Female, Name: 蟯ｸ逕ｰ 閨｡ == Kishida Satoshi Gender: Male, Name: 遶ｹ蜀 逵滉ｹ == Takeuchi Shinya Gender: Male, Name: 逕ｰ荳ｭ 逾蝉ｺ == Tanaka Yuuji Gender: Male)' + # Characters + context = '```\ + Game Characters:\ + Character: 豎繝惹ｸ 諡捺ｵｷ == Ikenoue Takumi - Gender: Male\ + Character: 遖乗ｰｸ 縺薙ｯ繧 == Fukunaga Koharu - Gender: Female\ + Character: 逾樊ｳ 逅螟ｮ == Kamiizumi Rio - Gender: Female\ + Character: 蜷臥･･蟇ｺ 繧｢繝ｪ繧ｵ == Kisshouji Arisa - Gender: Female\ + Character: 荵謌 蜿矩悟ｭ == Kuga Yuriko - Gender: Female\ + ```' + + # Prompt if fullPromptFlag: - system = PROMPT + system = PROMPT user = 'Line to Translate = ' + subbedT else: - system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Line to Translate: ' + subbedT + system = 'Output ONLY the english translation in the following format: `Translation: `' + user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + response = openai.ChatCompletion.create( - temperature=0, + temperature=0.1, frequency_penalty=0.2, presence_penalty=0.2, model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": context}, - {"role": "user", "content": history}, - {"role": "user", "content": user} - ], + messages=msg, request_timeout=30, ) @@ -512,12 +528,16 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Line to Translate: ', '') + translatedText = translatedText.replace('Line to Translate = ', '') + translatedText = translatedText.replace('Translation = ', '') + translatedText = translatedText.replace('Translate = ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Line to Translate:', '') - translatedText = re.sub(r'\n\nPast Translated Text:.*', '', translatedText, 0, re.DOTALL) + translatedText = translatedText.replace('Line to Translate =', '') + translatedText = translatedText.replace('Translation =', '') + translatedText = translatedText.replace('Translate =', '') translatedText = re.sub(r'Note:.*', '', translatedText) + translatedText = translatedText.replace('縺｣', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: