From 454f50884f2d661f197413427df952a2b6b1eb01 Mon Sep 17 00:00:00 2001 From: Dazed Date: Mon, 31 Jul 2023 17:05:32 -0500 Subject: [PATCH] Some changes to a few functions --- modules/csvtl.py | 163 ++++++++++++++++++++++++++++++---------- modules/rpgmakermvmz.py | 88 +++++++++++++--------- 2 files changed, 177 insertions(+), 74 deletions(-) diff --git a/modules/csvtl.py b/modules/csvtl.py index 85f1d7e..02c6ed2 100644 --- a/modules/csvtl.py +++ b/modules/csvtl.py @@ -94,12 +94,21 @@ def parseCSV(readFile, writeFile, filename): textHistory = [] global LOCK + format = '' + while format == '': + format = input('\n\nSelect the CSV Format:\n\n1. Translator++\n2. Translate All\n') + match format: + case '1': + format = '1' + case '2': + format = '2' + # Get total for progress bar totalLines = len(readFile.readlines()) readFile.seek(0) - reader = csv.reader(readFile, delimiter=',') - writer = csv.writer(writeFile, delimiter=',', quoting=csv.QUOTE_NONNUMERIC) + reader = csv.reader(readFile, delimiter=',',) + writer = csv.writer(writeFile, delimiter=',', doublequote=None) with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: pbar.desc=filename @@ -107,57 +116,78 @@ def parseCSV(readFile, writeFile, filename): for row in reader: try: - totalTokens += translateCSV(row, pbar, writer, textHistory) + totalTokens += translateCSV(row, pbar, writer, textHistory, format) except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) return [reader, totalTokens, e, tracebackLineNo] return [reader, totalTokens, None] -def translateCSV(row, pbar, writer, textHistory): +def translateCSV(row, pbar, writer, textHistory, format): translatedText = '' maxHistory = MAXHISTORY tokens = 0 global LOCK, ESTIMATE try: - jaString = row[0] + match format: + # Japanese Text on column 1. English on Column 2 + case '1': + jaString = row[0] - # Remove repeating characters because it confuses ChatGPT - jaString = re.sub(r'([\u3000-\uffef])\1{2,}', r'\1\1', jaString) + # Remove repeating characters because it confuses ChatGPT + jaString = re.sub(r'([\u3000-\uffef])\1{2,}', r'\1\1', jaString) - # Sub Vars - jaString = re.sub(r'(\\+[a-zA-Z]+)\[([a-zA-Z0-9]+)\]', r'[\1|\2]', jaString) + # Translate + response = translateGPT(jaString, 'Previous text for context: ' + ' '.join(textHistory), True) - # Translate - response = translateGPT(jaString, 'Previous text for context: ' + ' '.join(textHistory)) + # Check if there is an actual difference first + if response[0] != row[0]: + translatedText = response[0] + else: + translatedText = row[1] + tokens += response[1] - # Check if there is an actual difference first - if response[0] != row[0]: - translatedText = response[0] - else: - translatedText = row[1] - tokens += response[1] + # Textwrap + translatedText = textwrap.fill(translatedText, width=WIDTH) - # ReSub Vars - translatedText = re.sub(r'\[([\\a-zA-Z]+)\|([a-zA-Z0-9]+)]', r'\1[\2]', translatedText) - translatedText = re.sub('"', "'", translatedText) + # Set Data + row[1] = translatedText - # TextHistory is what we use to give GPT Context, so thats appended here. - textHistory.append(': ' + translatedText) + # Keep textHistory list at length maxHistory + with LOCK: + if len(textHistory) > maxHistory: + textHistory.pop(0) + if not ESTIMATE: + writer.writerow(row) + pbar.update(1) - # Textwrap - translatedText = textwrap.fill(translatedText, width=WIDTH) + # TextHistory is what we use to give GPT Context, so thats appended here. + textHistory.append(': ' + translatedText) + + # Translate Everything + case '2': + for i in range(len(row)): + if i not in [1, 3]: + continue + jaString = row[i] + if '_' in jaString: + continue - # Set Data - row[1] = translatedText + #Translate + response = translateGPT(jaString, 'Reply with only the English translation of the location name.', True) + translatedText = response[0] + tokens += response[1] - # Keep textHistory list at length maxHistory - with LOCK: - if len(textHistory) > maxHistory: - textHistory.pop(0) - if not ESTIMATE: + # Textwrap + translatedText = textwrap.fill(translatedText, width=WIDTH) + + # Set Data + translatedText = translatedText.replace('\"', '') + translatedText = translatedText.replace('\'', '') + translatedText = translatedText.replace(',', '') + row[i] = translatedText writer.writerow(row) - pbar.update(1) + pbar.update(1) except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) @@ -166,30 +196,87 @@ def translateCSV(row, pbar, writer, textHistory): return tokens +def subVars(jaString): + varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\.<>a-zA-Z]+' + count = 0 + + varList = re.findall(varRegex, jaString) + if len(varList) != 0: + for var in varList: + jaString = jaString.replace(var, '[v' + str(count) + ']') + count += 1 + + return [jaString, varList] + +def resubVars(translatedText, varList): + count = 0 + + if len(varList) != 0: + for var in varList: + translatedText = translatedText.replace('[v' + str(count) + ']', var) + count += 1 + + # Remove Color Variables Spaces + if '\\c' in translatedText: + translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r'\1', translatedText) + translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) + return translatedText + @retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(t, history): +def translateGPT(t, history, fullPromptFlag): with LOCK: # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0613") TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, 0) + # Sub Vars + varResponse = subVars(t) + subbedT = varResponse[0] + # If there isn't any Japanese in the text just skip - if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+', t): + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', subbedT): return(t, 0) """Translate text using GPT""" - system = PROMPT + history + context = 'Character Context: クレア == Clea | Female, ノラ == Nora | Female, リルム == Relm | Female, ソフィア == Sophia | Female, セリス == Celis | Female, ピステ == Piste | Female, イクト == Ect | Female, カロン == Caron | Female, エモニ == Emoni | Female, アミナ == Amina | Female, アルマダ == Armada | Female, フォニ == Phoni | Female, エレオ == Eleo | Female, ペルノ == Perno | Female' + if fullPromptFlag: + system = PROMPT + user = 'Text to Translate: ' + subbedT + else: + system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' + user = 'Text to Translate: ' + subbedT response = openai.ChatCompletion.create( temperature=0, model="gpt-3.5-turbo", messages=[ {"role": "system", "content": system}, - {"role": "user", "content": t} + {"role": "user", "content": context}, + {"role": "user", "content": history}, + {"role": "user", "content": user} ], request_timeout=30, ) - return [response.choices[0].message.content, response.usage.total_tokens] \ No newline at end of file + # Save Translated Text + translatedText = response.choices[0].message.content + tokens = response.usage.total_tokens + + # Resub Vars + translatedText = resubVars(translatedText, varResponse[1]) + + # Remove Placeholder Text + translatedText = translatedText.replace('English Translation: ', '') + translatedText = translatedText.replace('Translation: ', '') + translatedText = translatedText.replace('Text to Translate: ', '') + translatedText = translatedText.replace('English Translation:', '') + translatedText = translatedText.replace('Translation:', '') + translatedText = translatedText.replace('Text to Translate:', '') + + # Return Translation + if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: + return [t, response.usage.total_tokens] + else: + return [translatedText, tokens] \ No newline at end of file diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index c0532d8..e190a5a 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -26,7 +26,7 @@ PROMPT = Path('prompt.txt').read_text(encoding='utf-8') THREADS = 20 # For GPT4 rate limit will be hit if you have more than 1 thread. LOCK = threading.Lock() WIDTH = 60 -LISTWIDTH = 32 +LISTWIDTH = 60 MAXHISTORY = 10 ESTIMATE = '' TOTALCOST = 0 @@ -207,9 +207,9 @@ def translateNote(event, regex): # Regex that only matches text inside LB. jaString = event['note'] - match = re.search(regex, jaString) + match = re.findall(regex, jaString, re.DOTALL) if match: - jaString = match.group(1) + jaString = match[0] # Need to remove outside code jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】]+', '', jaString) jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？]+$', '', jaString) @@ -354,26 +354,26 @@ def parseSystem(data, filename): def searchThings(name, pbar): tokens = 0 - # Set the context of what we are translating - responseList = [] - responseList.append(translateGPT(name['name'], 'Reply with only the English translation of the RPG Item name.', False)) - responseList.append(translateGPT(name['description'], 'Reply with only the English translation of the description.', False)) + # Name + nameResponse = translateGPT(name['name'], 'Reply with only the english translation of the RPG item name.', False) if 'name' in name else '' + + # Description + descriptionResponse = translateGPT(name['description'], 'Reply with only the english translation of the description.', False) if 'description' in name else '' if '') + tokens += translateNote(name, r'') - # Extract all our translations in a list from response - for i in range(len(responseList)): - tokens += responseList[i][1] - responseList[i] = responseList[i][0] + # Count Tokens + tokens += nameResponse[1] if nameResponse != '' else 0 + tokens += descriptionResponse[1] if descriptionResponse != '' else 0 # Set Data - name['name'] = responseList[0].strip('.\"') - responseList[1] = textwrap.fill(responseList[1], LISTWIDTH) - name['description'] = responseList[1].strip('\"') - # name['note'] = responseList[2] - pbar.update(1) + if 'name' in name: + name['name'] = nameResponse[0].strip('\"') + if 'description' in name: + name['description'] = descriptionResponse[0].strip('\"') + pbar.update(1) return tokens def searchNames(name, pbar, context): @@ -383,7 +383,7 @@ def searchNames(name, pbar, context): if 'Actors' in context: newContext = 'Reply with only the english translation of the NPC name' if 'Armors' in context: - newContext = 'Reply with only the english translation of the RPG armor/clothing name' + newContext = 'Reply with only the english translation of the RPG equipment name' if 'Classes' in context: newContext = 'Reply with only the english translation of the RPG class name' if 'MapInfos' in context: @@ -401,7 +401,10 @@ def searchNames(name, pbar, context): responseList.append(translateGPT(name['nickname'], 'Reply with ONLY the english translation of the NPC nickname', False)) if 'Armors' in context or 'Weapons' in context: - responseList.append(translateGPT(name['description'], '', True)) + if 'description' in name: + responseList.append(translateGPT(name['description'], '', True)) + else: + responseList.append(['', 0]) if 'Enemies' in context: if 'desc1' in name['note']: @@ -430,9 +433,10 @@ def searchNames(name, pbar, context): if 'Armors' in context or 'Weapons' in context: translatedText = textwrap.fill(responseList[1], LISTWIDTH) - name['description'] = translatedText.strip('\"') - if ']*)>') + if 'description' in name: + name['description'] = translatedText.strip('\"') + if ']*)>') pbar.update(1) return tokens @@ -467,7 +471,6 @@ def searchCodes(page, pbar): # if jaString.startswith('　'): # jaString = jaString + ': ' - oldjaString = jaString jaString = re.sub(r'([\u3000-\uffef])\1{3,}', r'\1\1\1', jaString) # Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior) @@ -484,6 +487,7 @@ def searchCodes(page, pbar): # Join up 401 groups for better translation. if len(currentGroup) > 0: finalJAString = ''.join(currentGroup) + oldjaString = finalJAString # Check for speaker if '\\N' in finalJAString: @@ -910,20 +914,20 @@ def searchCodes(page, pbar): jaString = jaString.replace(' 。', '.') # Need to remove outside code and put it back later - startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】（）Ａ-Ｚ０-９]+', jaString) - jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】（）Ａ-Ｚ０-９]+', '', jaString) - endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー【】（）Ａ-Ｚ０-９ 。！？]+$', jaString) - jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー【】（）Ａ-Ｚ０-９ 。！？]+$', '', jaString) + startString = re.search(r'^en.+\)\s|^en.+\)|^if.+\)\s|^if.+\)', jaString) + jaString = re.sub(r'^en.+\)\s|^en.+\)|^if.+\)\s|^if.+\)', '', jaString) + endString = re.search(r'\sen.+$|en.+$|\sif.+$|if.+$', jaString) + jaString = re.sub(r'\sen.+$|en.+$|\sif.+$|if.+$', '', jaString) if startString is None: startString = '' else: startString = startString.group() if endString is None: endString = '' else: endString = endString.group() if len(textHistory) > 0: - response = translateGPT(jaString, 'Previous text for context: ' + textHistory[len(textHistory)-1], False) + response = translateGPT(jaString, 'Previous text for context: ' + textHistory[len(textHistory)-1], True) translatedText = response[0] else: - response = translateGPT(jaString, '', False) + response = translateGPT(jaString, '', True) translatedText = response[0] # Remove characters that may break scripts @@ -1003,7 +1007,6 @@ def searchCodes(page, pbar): # This is part of the logic so we just pass it tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) # raise Exception(str(e) + '|Line:' + tracebackLineNo) - pass except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) raise Exception(str(e) + '|Line:' + tracebackLineNo + '| Failed to translate: ' + oldjaString) @@ -1044,20 +1047,20 @@ def searchSS(state, pbar): tokens = 0 # Name - nameResponse = translateGPT(state['name'], 'Reply with only the english translation of the RPG item name.', False) if 'name' in state else '' + nameResponse = translateGPT(state['name'], 'Reply with only the english translation of the RPG Skill name.', False) if 'name' in state else '' # Description descriptionResponse = translateGPT(state['description'], 'Reply with only the english translation of the description.', False) if 'description' in state else '' # Messages - message1Response = translateGPT(state['message1'], 'reply with only the english translation of the text.', False) if 'message1' in state else '' + message1Response = translateGPT('Taro' + state['message1'], 'reply with only the gender neutral english translation of the text.', False) if 'message1' in state else '' message2Response = translateGPT(state['message2'], 'reply with only the english translation of the text.', False) if 'message2' in state else '' message3Response = translateGPT(state['message3'], 'reply with only the english translation of the text.', False) if 'message3' in state else '' message4Response = translateGPT(state['message4'], 'reply with only the english translation of the text.', False) if 'message4' in state else '' # if 'note' in state: - # if 'raceDesc' in state['note']: - # tokens += translateNote(state, r']*)>') + if 'DOBBY' in state['note']: + tokens += translateNote(state, r'\n(.+)\n') # Count Tokens tokens += nameResponse[1] if nameResponse != '' else 0 @@ -1071,9 +1074,12 @@ def searchSS(state, pbar): if 'name' in state: state['name'] = nameResponse[0].strip('\"') if 'description' in state: - state['description'] = descriptionResponse[0].strip('\"') + # Textwrap + translatedText = descriptionResponse[0] + translatedText = textwrap.fill(translatedText, width=LISTWIDTH) + state['description'] = translatedText.strip('\"') if 'message1' in state: - state['message1'] = message1Response[0].strip('\"') + state['message1'] = message1Response[0].strip('\"').replace('Taro', '') if 'message2' in state: state['message2'] = message2Response[0].strip('\"') if 'message3' in state: @@ -1170,6 +1176,10 @@ def resubVars(translatedText, varList): translatedText = translatedText.replace('[v' + str(count) + ']', var) count += 1 + # Remove Color Variables Spaces + if '\\c' in translatedText: + translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r'\1', translatedText) + translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) return translatedText @retry(exceptions=Exception, tries=5, delay=5) @@ -1210,16 +1220,22 @@ def translateGPT(t, history, fullPromptFlag): request_timeout=30, ) + # Save Translated Text translatedText = response.choices[0].message.content tokens = response.usage.total_tokens # Resub Vars translatedText = resubVars(translatedText, varResponse[1]) + # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') translatedText = translatedText.replace('Text to Translate: ', '') + translatedText = translatedText.replace('English Translation:', '') + translatedText = translatedText.replace('Translation:', '') + translatedText = translatedText.replace('Text to Translate:', '') + # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: return [t, response.usage.total_tokens] else: