Create nscript parser

modules/main.py

@@ -24,6 +24,7 @@ from modules.kansen import handleKansen
 from modules.lune import handleLune
 from modules.atelier import handleAtelier
 from modules.anim import handleAnim
+from modules.nscript import handleNScript
 
 # For GPT4 rate limit will be hit if you have more than 1 thread.
 # 1 Thread for each file. Controls how many files are worked on at once.
@@ -41,6 +42,7 @@ MODULES = [
     ["Lune", "json", handleLune],
     ["Atelier", "txt", handleAtelier],
     ["Anim", "json", handleAnim],
+    ["NScript", "txt", handleNScript],
 ]
 
 # Info Message

modules/nscript.py

@@ -0,0 +1,647 @@
+# Libraries
+import os, re, textwrap, threading, time, traceback, tiktoken, openai
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv('api').replace(' ', '') != '':
+    openai.api_base = os.getenv('api')
+openai.organization = os.getenv('org')
+openai.api_key = os.getenv('key')
+
+#Globals
+MODEL = os.getenv('model')
+TIMEOUT = int(os.getenv('timeout'))
+LANGUAGE = os.getenv('language').capitalize()
+PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
+VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
+THREADS = int(os.getenv('threads'))
+LOCK = threading.Lock()
+WIDTH = int(os.getenv('width'))
+LISTWIDTH = int(os.getenv('listWidth'))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ''
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False    # Output a list of all the character names found
+BRFLAG = False   # If the game uses <br> instead
+FIXTEXTWRAP = True  # Overwrites textwrap
+IGNORETLTEXT = False    # Ignores all translated text.
+MISMATCH = []   # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+#tqdm Globals
+BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if 'gpt-3.5' in MODEL:
+    INPUTAPICOST = .002 
+    OUTPUTAPICOST = .002
+    BATCHSIZE = 10
+elif 'gpt-4' in MODEL:
+    INPUTAPICOST = .01
+    OUTPUTAPICOST = .03
+    BATCHSIZE = 10
+
+def handleNScript(filename, estimate):
+    global ESTIMATE
+    ESTIMATE = estimate
+
+    if ESTIMATE:
+        start = time.time()
+        translatedData = openFiles(filename)
+
+        # Print Result
+        end = time.time()
+        tqdm.write(getResultString(translatedData, end - start, filename))
+        with LOCK:
+            TOKENS[0] += translatedData[1][0]
+            TOKENS[1] += translatedData[1][1]
+
+        # Print Total
+        totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+
+        # Print any errors on maps
+        if len(MISMATCH) > 0:
+            return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+        else:
+            return totalString
+    
+    else:
+        try:
+            with open('translated/' + filename, 'w', encoding='shift_jis', errors='ignore') as outFile:
+                start = time.time()
+                translatedData = openFiles(filename)
+
+                # Print Result
+                end = time.time()
+                outFile.writelines(translatedData[0])
+                tqdm.write(getResultString(translatedData, end - start, filename))
+                with LOCK:
+                    TOKENS[0] += translatedData[1][0]
+                    TOKENS[1] += translatedData[1][1]
+        except Exception as e:
+            traceback.print_exc()
+            return 'Fail'
+
+    return getResultString(['', TOKENS, None], end - start, 'TOTAL')
+
+def getResultString(translatedData, translationTime, filename):
+    # File Print String
+    totalTokenstring =\
+        Fore.YELLOW +\
+        '[Input: ' + str(translatedData[1][0]) + ']'\
+        '[Output: ' + str(translatedData[1][1]) + ']'\
+        '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
+        (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
+    timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+
+    if translatedData[2] == None:
+        # Success
+        return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+
+    else:
+        # Fail
+        try:
+            raise translatedData[2]
+        except Exception as e:
+            traceback.print_exc()
+            errorString = str(e) + Fore.RED
+            return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
+                errorString + Fore.RESET
+
+def openFiles(filename):
+    with open('files/' + filename, 'r', encoding='cp932') as readFile:
+        translatedData = parseNScript(readFile, filename)
+
+        # Delete lines marked for deletion
+        finalData = []
+        for line in translatedData[0]:
+            if line != '\\d\n':
+                finalData.append(line)
+        translatedData[0] = finalData
+    
+    return translatedData
+
+def parseNScript(readFile, filename):
+    totalTokens = [0,0]
+    totalLines = 0
+
+    # Get total for progress bar
+    data = readFile.readlines()
+    totalLines = len(data)
+
+    with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+        pbar.desc=filename
+        pbar.total=totalLines
+
+        try:
+            result = translateNScript(data, pbar, totalLines)
+            totalTokens[0] += result[0]
+            totalTokens[1] += result[1]
+        except Exception as e:
+            traceback.print_exc()
+            return [data, totalTokens, e]
+    return [data, totalTokens, None]
+
+def translateNScript(data, pbar, totalLines):
+    textHistory = []
+    batch = []
+    currentGroup = []
+    maxHistory = MAXHISTORY
+    tokens = [0,0]
+    speaker = ''
+    insertBool = False
+    global LOCK, ESTIMATE
+    i = 0
+    batchStartIndex = 0
+
+    while i < len(data):
+        # Speaker
+        matchList = re.findall(r'^【\s+(.*?)\s+】$', data[i])
+        if len(matchList) != 0:
+            response = getSpeaker(matchList[0])
+            speaker = response[0]
+            tokens[0] += response[1][0]
+            tokens[1] += response[1][1]
+            data[i] = '【　' + speaker + '　】\n'
+            i += 1
+        else:
+            speaker = ''
+
+        # Choices
+        matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i])
+        if len(matchList) != 0:
+            originalText = matchList[0]
+            if len(textHistory) > 0:
+                response = translateGPT(matchList[0], 'Keep your translation as brief as possible. Previous text for context: ' + textHistory[len(textHistory)-1] + '\n\nReply in the style of a dialogue option.', False)
+            else:
+                response = translateGPT(matchList[0], '\n\nReply in the style of a dialogue option.', False)
+            translatedText = response[0]
+            tokens[0] += response[1][0]
+            tokens[1] += response[1][1]
+
+            # Remove characters that may break scripts
+            charList = ['.', '\"', '\\n']
+            for char in charList:
+                translatedText = translatedText.replace(char, '')
+
+            # Escape all '
+            translatedText = translatedText.replace('\\', '')
+            # translatedText = translatedText.replace("'", "\\\'")
+
+            # Set Data
+            translatedText = data[i].replace(originalText, translatedText)
+            data[i] = translatedText                
+
+        # Lines
+        matchList = re.findall(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９「」　>].*', data[i]) 
+        if len(matchList) > 0:
+            currentGroup.append(matchList[0])
+            if len(data) > i+1:
+                if speaker == '':
+                    while '\n' != data[i+1] and '【' not in data[i+1]:
+                        if insertBool is True:
+                            data[i] = '\d\n'
+                            pbar.update(1)
+                        i += 1
+                        matchList = re.findall(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９「」　>].*', data[i])
+                        if len(matchList) > 0:
+                            currentGroup.append(matchList[0])
+                else:
+                    while '　' in data[i+1][0] or '"' in data[i+1]:
+                        if insertBool is True:
+                            data[i] = '\d\n'
+                            pbar.update(1)
+                        i += 1
+                        matchList = re.findall(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９「」　>].*', data[i])
+                        if len(matchList) > 0:
+                            currentGroup.append(matchList[0])
+                        
+            # Join up 401 groups for better translation.
+            if len(currentGroup) > 0:
+                finalJAString = ' '.join(currentGroup)
+                oldjaString = finalJAString
+
+            # Remove any textwrap
+            if FIXTEXTWRAP == True:
+                finalJAString = finalJAString.replace('>', '')
+
+            # Remove Extra Stuff bad for translation.
+            finalJAString = finalJAString.replace('ﾞ', '')
+            finalJAString = finalJAString.replace('・', '.')
+            finalJAString = finalJAString.replace('‶', '')
+            finalJAString = finalJAString.replace('”', '')
+            finalJAString = finalJAString.replace('―', '-')
+            finalJAString = finalJAString.replace('ー', '-')
+            finalJAString = finalJAString.replace('…', '...')
+            finalJAString = re.sub(r'(\.{3}\.+)', '...', finalJAString)
+            finalJAString = finalJAString.replace('　', ' ')
+
+            # Furigana Removal
+            matchList = re.findall(r'『\((.+)/.*?』', finalJAString)
+            if len(matchList) > 0:
+                finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1])
+
+            # Add Speaker (If there is one)
+            if speaker != '':
+                finalJAString = f'{speaker}: {finalJAString}'
+
+            # [Passthrough 1] Pulling From File
+            if insertBool is False:
+                # Append to List and Clear Values
+                batch.append(finalJAString)
+                speaker = ''
+
+                # Translate Batch if Full
+                if len(batch) == BATCHSIZE:
+                    # Translate
+                    response = translateGPT(batch, textHistory, True)
+                    tokens[0] += response[1][0]
+                    tokens[1] += response[1][1]
+                    translatedBatch = response[0]
+                    textHistory = translatedBatch[-10:]
+
+                    # Set Values
+                    if len(batch) == len(translatedBatch):
+                        i = batchStartIndex
+                        insertBool = True
+
+                    # Mismatch
+                    else:
+                        pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+                        MISMATCH.append(batch)
+                        batchStartIndex = i
+                        batch.clear()
+
+                i += 1
+                if insertBool is True:
+                    pbar.update(1)
+                currentGroup = []
+
+            # [Passthrough 2] Setting Data
+            else:
+                # Get Text
+                translatedText = translatedBatch[0]
+                translatedText = translatedText.replace('\\"', '\"')
+                translatedText = translatedText.replace('[', '(')
+                translatedText = translatedText.replace(']', ')')
+
+                # Remove added speaker
+                translatedText = re.sub(r'^.+?:\s', '', translatedText)
+
+                # Textwrap
+                translatedText = textwrap.fill(translatedText, width=WIDTH)
+                textList = translatedText.split('\n')
+                    
+                # Set Text
+                data[i] = '\d\n'
+                for line in textList:
+                    # Wordwrap Text
+                    line = textwrap.fill(line, width=WIDTH)
+                    
+                    # Set
+                    data.insert(i, '>' + line.strip() + '\n')
+                    i+=1
+                translatedBatch.pop(0)
+                speaker = ''
+                currentGroup = []
+
+                # If Batch is empty. Move on.
+                if len(translatedBatch) == 0:
+                    insertBool = False
+                    batchStartIndex = i
+                    batch.clear()
+
+        # Nothing relevant. Skip Line.
+        else:
+            i += 1
+            if insertBool is True:
+                pbar.update(1)
+
+        # Translate Batch if not empty and EOF
+        if len(batch) != 0 and i >= len(data):
+            # Translate
+            response = translateGPT(batch, textHistory, True)
+            tokens[0] += response[1][0]
+            tokens[1] += response[1][1]
+            translatedBatch = response[0]
+            textHistory = translatedBatch[-10:]
+
+            # Set Values
+            if len(batch) == len(translatedBatch):
+                i = batchStartIndex
+                insertBool = True
+
+            # Mismatch
+            else:
+                pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+                MISMATCH.append(batch)
+                batchStartIndex = i
+                batch.clear()
+
+            currentGroup = []
+    return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+    match speaker:
+        case '央':
+            return ['Akira', [0,0]]
+        case '累':
+            return ['Rui', [0,0]]
+        case '梨里':
+            return ['Riri', [0,0]]
+        case '純':
+            return ['Jun', [0,0]]
+        case '美鈴':
+            return ['Misuzu', [0,0]]
+        case '須田':
+            return ['Suda', [0,0]]
+        case '高橋':
+            return ['Takahashi', [0,0]]
+        case '勇二':
+            return ['Yuuji', [0,0]]
+        case _:
+            return translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False)
+        
+def subVars(jaString):
+    jaString = jaString.replace('\u3000', ' ')
+
+    # Nested
+    count = 0
+    nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+    nestedList = set(nestedList)
+    if len(nestedList) != 0:
+        for icon in nestedList:
+            jaString = jaString.replace(icon, '{Nested_' + str(count) + '}')
+            count += 1
+
+    # Icons
+    count = 0
+    iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+    iconList = set(iconList)
+    if len(iconList) != 0:
+        for icon in iconList:
+            jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}')
+            count += 1
+
+    # Colors
+    count = 0
+    colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+    colorList = set(colorList)
+    if len(colorList) != 0:
+        for color in colorList:
+            jaString = jaString.replace(color, '{Color_' + str(count) + '}')
+            count += 1
+
+    # Names
+    count = 0
+    nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+    nameList = set(nameList)
+    if len(nameList) != 0:
+        for name in nameList:
+            jaString = jaString.replace(name, '{Noun_' + str(count) + '}')
+            count += 1
+
+    # Variables
+    count = 0
+    varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+    varList = set(varList)
+    if len(varList) != 0:
+        for var in varList:
+            jaString = jaString.replace(var, '{Var_' + str(count) + '}')
+            count += 1
+
+    # Formatting
+    count = 0
+    formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
+    formatList = set(formatList)
+    if len(formatList) != 0:
+        for var in formatList:
+            jaString = jaString.replace(var, '{FCode_' + str(count) + '}')
+            count += 1
+
+    # Put all lists in list and return
+    allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+    return [jaString, allList]
+
+def resubVars(translatedText, allList):
+    # Fix Spacing and ChatGPT Nonsense
+    matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+    if len(matchList) > 0:
+        for match in matchList:
+            text = match.strip()
+            translatedText = translatedText.replace(match, text)
+
+    # Nested
+    count = 0
+    if len(allList[0]) != 0:
+        for var in allList[0]:
+            translatedText = translatedText.replace('{Nested_' + str(count) + '}', var)
+            count += 1
+
+    # Icons
+    count = 0
+    if len(allList[1]) != 0:
+        for var in allList[1]:
+            translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var)
+            count += 1
+
+    # Colors
+    count = 0
+    if len(allList[2]) != 0:
+        for var in allList[2]:
+            translatedText = translatedText.replace('{Color_' + str(count) + '}', var)
+            count += 1
+
+    # Names
+    count = 0
+    if len(allList[3]) != 0:
+        for var in allList[3]:
+            translatedText = translatedText.replace('{Noun_' + str(count) + '}', var)
+            count += 1
+
+    # Vars
+    count = 0
+    if len(allList[4]) != 0:
+        for var in allList[4]:
+            translatedText = translatedText.replace('{Var_' + str(count) + '}', var)
+            count += 1
+    
+    # Formatting
+    count = 0
+    if len(allList[5]) != 0:
+        for var in allList[5]:
+            translatedText = translatedText.replace('{FCode_' + str(count) + '}', var)
+            count += 1
+
+    return translatedText
+
+def batchList(input_list, batch_size):
+    if not isinstance(batch_size, int) or batch_size <= 0:
+        raise ValueError("batch_size must be a positive integer")
+        
+    return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+def createContext(fullPromptFlag, subbedT):
+    characters = 'Game Characters:\n\
+渋江 央 (Shibue Akira) - Male\n\
+蘆名 累 (Ashina Rui) - Female\n\
+清原 梨里 (Kiyohara Riri) - Female\n\
+五十嵐 純 (Igarashi Jun) - Female\n\
+子野日 美鈴 (Nenohi Misuzu) - Female\n\
+須田 (Suda) - Male\n\
+高橋 (Takahashi) - Female\n\
+勇二 (Yuuji) - Male\n\
+'
+    
+    system = PROMPT + VOCAB if fullPromptFlag else \
+        f"\
+You are an expert Eroge Game translator who translates Japanese text to English.\n\
+You are going to be translating text from a videogame.\n\
+I will give you lines of text, and you must translate each line to the best of your ability.\n\
+{VOCAB}\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\
+"
+    user = f'{subbedT}'
+    return characters, system, user
+
+def translateText(characters, system, user, history):
+    # Prompt
+    msg = [{"role": "system", "content": system + characters}]
+
+    # Characters
+    msg.append({"role": "system", "content": characters})
+
+    # History
+    if isinstance(history, list):
+        msg.extend([{"role": "system", "content": h} for h in history])
+    else:
+        msg.append({"role": "system", "content": history})
+    
+    # Content to TL
+    msg.append({"role": "user", "content": f'{user}'})
+    response = openai.chat.completions.create(
+        temperature=0.1,
+        frequency_penalty=0.1,
+        presence_penalty=0.1,
+        model=MODEL,
+        messages=msg,
+    )
+    return response
+
+def cleanTranslatedText(translatedText, varResponse):
+    placeholders = {
+        f'{LANGUAGE} Translation: ': '',
+        'Translation: ': '',
+        'っ': '',
+        '〜': '~',
+        'ー': '-',
+        'ッ': '',
+        '。': '.',
+        'Placeholder Text': ''
+        # Add more replacements as needed
+    }
+    for target, replacement in placeholders.items():
+        translatedText = translatedText.replace(target, replacement)
+
+    translatedText = resubVars(translatedText, varResponse[1])
+    return [line for line in translatedText.replace('\\n', '\n').split('\n') if line]
+
+def extractTranslation(translatedTextList, is_list):
+    pattern = r'`?<Line(\d+)>([\\]*.*?[\\]*?)<\/?Line\d+>`?'
+    # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+    if is_list:
+        return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
+    else:
+        matchList = re.findall(pattern, translatedTextList)
+        return matchList[0][1] if matchList else translatedTextList
+
+def countTokens(characters, system, user, history):
+    inputTotalTokens = 0
+    outputTotalTokens = 0
+    enc = tiktoken.encoding_for_model(MODEL)
+    
+    # Input
+    if isinstance(history, list):
+        for line in history:
+            inputTotalTokens += len(enc.encode(line))
+    else:
+        inputTotalTokens += len(enc.encode(history))
+    inputTotalTokens += len(enc.encode(system))
+    inputTotalTokens += len(enc.encode(characters))
+    inputTotalTokens += len(enc.encode(user))
+
+    # Output
+    outputTotalTokens += round(len(enc.encode(user))/1.5)
+
+    return [inputTotalTokens, outputTotalTokens]
+
+def combineList(tlist, text):
+    if isinstance(text, list):
+        return [t for sublist in tlist for t in sublist]
+    return tlist[0]
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(text, history, fullPromptFlag):
+    totalTokens = [0, 0]
+    if isinstance(text, list):
+        tList = batchList(text, BATCHSIZE)
+    else:
+        tList = [text]
+
+    for index, tItem in enumerate(tList):
+        # Before sending to translation, if we have a list of items, add the formatting
+        if isinstance(tItem, list):
+            payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)])
+            payload = payload.replace('``', '`Placeholder Text`')
+            varResponse = subVars(payload)
+            subbedT = varResponse[0]
+        else:
+            varResponse = subVars(tItem)
+            subbedT = varResponse[0]
+
+        # Things to Check before starting translation
+        if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+            continue
+
+        # Create Message
+        characters, system, user = createContext(fullPromptFlag, subbedT)
+
+        # Calculate Estimate
+        if ESTIMATE:
+            estimate = countTokens(characters, system, user, history)
+            totalTokens[0] += estimate[0]
+            totalTokens[1] += estimate[1]
+            continue
+
+        # Translating
+        response = translateText(characters, system, user, history)
+        translatedText = response.choices[0].message.content
+        totalTokens[0] += response.usage.prompt_tokens
+        totalTokens[1] += response.usage.completion_tokens
+
+        # Formatting
+        translatedTextList = cleanTranslatedText(translatedText, varResponse)
+        if isinstance(tItem, list):
+            extractedTranslations = extractTranslation(translatedTextList, True)
+            tList[index] = extractedTranslations
+            if len(tItem) != len(translatedTextList):
+                mismatch = True     # Just here so breakpoint can be set
+            history = extractedTranslations[-10:]  # Update history if we have a list
+        else:
+            # Ensure we're passing a single string to extractTranslation
+            extractedTranslations = extractTranslation('\n'.join(translatedTextList), False)
+            tList[index] = extractedTranslations
+
+    finalList = combineList(tList, text)
+    return [finalList, totalTokens]

modules/rpgmakermvmz.py

@@ -47,7 +47,7 @@ if 'gpt-3.5' in MODEL:
 elif 'gpt-4' in MODEL:
     INPUTAPICOST = .01
     OUTPUTAPICOST = .03
-    BATCHSIZE = 20
+    BATCHSIZE = 40
     FREQUENCY_PENALTY = 0.1
 
 #tqdm Globals
@@ -56,11 +56,11 @@ POSITION = 0
 LEAVE = False
 
 # Dialogue / Scroll
-CODE401 = False
-CODE405 = False
+CODE401 = True
+CODE405 = True
 
 # Choices
-CODE102 = False
+CODE102 = True
 
 # Variables
 CODE122 = False
@@ -69,7 +69,7 @@ CODE122 = False
 CODE101 = False
 
 # Other
-CODE355655 = True
+CODE355655 = False
 CODE357 = False
 CODE657 = False
 CODE356 = False
@@ -1708,9 +1708,6 @@ def searchCodes(page, pbar, fillList, filename):
                 for choice in range(len(codeList[i]['parameters'][0])):
                     jaString = codeList[i]['parameters'][0][choice]
                     jaString = jaString.replace(' 。', '.')
-                    # Things to Check before starting translation
-                    if re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', jaString):
-                        print('Test')
 
                     # Avoid Empty Strings
                     if jaString == '':
@@ -2161,9 +2158,13 @@ def batchList(input_list, batch_size):
 
 def createContext(fullPromptFlag, subbedT):
     characters = 'Game Characters:\n\
-神田 瑠唯 (Kanda Rui) - Female\n\
-ルイ (Rui) - Female\n\
-チュベロス (Tuberose) - Male\n\
+ルース (Ruth) - Male\n\
+リーザ (Reeza) - Female\n\
+エリル (Eril) - Female\n\
+シアン (Cyan) - Female\n\
+トリス (Tris) - Female\n\
+エリル (Eril) - Female\n\
+エリル (Eril) - Female\n\
 '
     
     system = PROMPT + VOCAB if fullPromptFlag else \

modules/txt.py

@@ -1,390 +0,0 @@
-from concurrent.futures import ThreadPoolExecutor, as_completed
-import json
-import os
-from pathlib import Path
-import re
-import sys
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-
-from colorama import Fore
-from dotenv import load_dotenv
-import openai
-from retry import retry
-from tqdm import tqdm
-
-#Globals
-load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
-    openai.api_base = os.getenv('api')
-
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE=os.getenv('language').capitalize()
-
-APICOST = .002 # Depends on the model https://openai.com/pricing
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
-MAXHISTORY = 10
-ESTIMATE = ''
-TOTALCOST = 0
-TOKENS = 0
-TOTALTOKENS = 0
-
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
-POSITION=0
-LEAVE=False
-
-# Flags
-CODE401 = True
-CODE102 = True
-CODE122 = False
-CODE101 = False
-CODE355655 = False
-CODE357 = False
-CODE356 = False
-CODE320 = False
-CODE111 = False
-
-def handleTXT(filename, estimate):
-    global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
-    ESTIMATE = estimate
-
-    if estimate:
-        start = time.time()
-        translatedData = openFiles(filename)
-
-        # Print Result
-        end = time.time()
-        tqdm.write(getResultString(['', TOKENS, None], end - start, filename))
-        with LOCK:
-            TOTALCOST += TOKENS * .001 * APICOST
-            TOTALTOKENS += TOKENS
-            TOKENS = 0
-
-        return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
-    
-    else:
-        with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
-            start = time.time()
-            translatedData = openFiles(filename)
-
-            # Print Result
-            end = time.time()
-            outFile.writelines(translatedData[0])
-            tqdm.write(getResultString(translatedData, end - start, filename))
-            with LOCK:
-                TOTALCOST += translatedData[1] * .001 * APICOST
-                TOTALTOKENS += translatedData[1]
-
-    return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
-
-def openFiles(filename):
-    with open('files/' + filename, 'r', encoding='UTF-8') as f:
-        translatedData = parseText(f, filename)
-    
-    return translatedData
-
-def getResultString(translatedData, translationTime, filename):
-    # File Print String
-    tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \
-        ' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']'
-    timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
-
-    if translatedData[2] == None:
-        # Success
-        return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
-
-    else:
-        # Fail
-        try:
-            raise translatedData[2]
-        except Exception as e:
-            errorString = str(e) + Fore.RED
-            return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
-                errorString + Fore.RESET
-        
-def parseText(data, filename):
-    totalTokens = 0
-    totalLines = 0
-    global LOCK
-
-    # Get total for progress bar
-    linesList = data.readlines()
-    totalLines = len(linesList)
-    
-    with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
-        pbar.desc=filename
-        pbar.total=totalLines
-        try:
-            response = translateText(linesList, pbar)
-        except Exception as e:
-            traceback.print_exc()
-            return [linesList, 0, e]
-    return [response[0], response[1], None]
-
-def translateText(data, pbar):
-    textHistory = []
-    maxHistory = MAXHISTORY
-    tokens = 0
-    speaker = ''
-    speakerFlag = False
-    currentGroup = []
-    syncIndex = 0
-
-    for i in range(len(data)):
-        if i != syncIndex:
-            continue
-
-        match = re.findall(r'm\[[0-9]+\] = \"(.*)\"', data[i])
-        if len(match) > 0:
-            jaString = match[0]
-
-            ### Translate
-            # Remove any textwrap
-            jaString = re.sub(r'\\n', ' ', jaString)
-
-            # Grab Speaker
-            speakerMatch = re.findall(r's\[[0-9]+\] = \"(.+?)[／\"]', data[i-1])
-            if len(speakerMatch) > 0:
-                # If there isn't any Japanese in the text just skip
-                if re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString) and '_' not in speakerMatch[0]:
-                    speaker = ''
-                else:
-                    speaker = ''
-            else:
-                speaker = ''
-
-            # Grab rest of the messages
-            currentGroup.append(jaString)
-            start = i
-            data[i] = re.sub(r'(m\[[0-9]+\]) = \"(.+)\"', rf'\1 = ""', data[i])
-            while (len(data) > i+1 and re.search(r'm\[[0-9]+\] = \"(.*)\"', data[i+1]) != None):
-                i+=1
-                match = re.findall(r'm\[[0-9]+\] = \"(.*)\"', data[i])
-                currentGroup.append(match[0])
-                data[i] = re.sub(r'(m\[[0-9]+\]) = \"(.+)\"', rf'\1 = ""', data[i])
-            finalJAString = ' '.join(currentGroup)
-            
-            # Translate
-            if speaker != '':
-                response = translateGPT(f'{speaker}: {finalJAString}', 'Previous Text for Context: ' + ' '.join(textHistory), True)
-            else:
-                response = translateGPT(finalJAString, 'Previous Text for Context: ' + ' '.join(textHistory), True)
-            tokens += response[1]
-            translatedText = response[0]
-            
-            # Remove added speaker and quotes
-            translatedText = re.sub(r'^.+?:\s', '', translatedText)
-
-            # TextHistory is what we use to give GPT Context, so thats appended here.
-            # rawTranslatedText = re.sub(r'[\\<>]+[a-zA-Z]+\[[a-zA-Z0-9]+\]', '', translatedText)
-            if speaker != '':
-                textHistory.append(speaker + ': ' + translatedText)
-            elif speakerFlag == False:
-                textHistory.append('\"' + translatedText + '\"')
-
-            # Keep textHistory list at length maxHistory
-            if len(textHistory) > maxHistory:
-                textHistory.pop(0)
-            currentGroup = []  
-
-            # Textwrap
-            translatedText = translatedText.replace('\"', '\\"')
-            translatedText = textwrap.fill(translatedText, width=WIDTH)
-
-            # Write
-            textList = translatedText.split("\n")
-            for t in textList:
-                data[start] = re.sub(r'(m\[[0-9]+\]) = \"(.*)\"', rf'\1 = "{t}"', data[start])
-                start+=1
-                
-        syncIndex = i + 1
-        pbar.update()
-    return [data, tokens]
-        
-def subVars(jaString):
-    jaString = jaString.replace('\u3000', ' ')
-
-    # Icons
-    count = 0
-    iconList = re.findall(r'[\\]+[iI]\[[0-9]+\]', jaString)
-    iconList = set(iconList)
-    if len(iconList) != 0:
-        for icon in iconList:
-            jaString = jaString.replace(icon, '<I' + str(count) + '>')
-            count += 1
-
-    # Colors
-    count = 0
-    colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
-    colorList = set(colorList)
-    if len(colorList) != 0:
-        for color in colorList:
-            jaString = jaString.replace(color, '<C' + str(count) + '>')
-            count += 1
-
-    # Names
-    count = 0
-    nameList = re.findall(r'[\\]+[nN]\[[0-9]+\]', jaString)
-    nameList = set(nameList)
-    if len(nameList) != 0:
-        for name in nameList:
-            jaString = jaString.replace(name, '<N' + str(count) + '>')
-            count += 1
-
-    # Variables
-    count = 0
-    varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
-    varList = set(varList)
-    if len(varList) != 0:
-        for var in varList:
-            jaString = jaString.replace(var, '<V' + str(count) + '>')
-            count += 1
-
-    # Formatting
-    count = 0
-    formatList = re.findall(r'[\\]+[!.]', jaString)
-    formatList = set(formatList)
-    if len(formatList) != 0:
-        for format in formatList:
-            jaString = jaString.replace(format, '<F' + str(count) + '>')
-            count += 1
-
-    # Put all lists in list and return
-    allList = [iconList, colorList, nameList, varList, formatList]
-    return [jaString, allList]
-
-def resubVars(translatedText, allList):
-    # Fix Spacing and ChatGPT Nonsense
-    matchList = re.findall(r'<\s?.+?\s?>', translatedText)
-    if len(matchList) > 0:
-        for match in matchList:
-            text = match.strip()
-            translatedText = translatedText.replace(match, text)
-
-    # Icons
-    count = 0
-    if len(allList[0]) != 0:
-        for var in allList[0]:
-            translatedText = translatedText.replace('<I' + str(count) + '>', var)
-            count += 1
-
-    # Colors
-    count = 0
-    if len(allList[1]) != 0:
-        for var in allList[1]:
-            translatedText = translatedText.replace('<C' + str(count) + '>', var)
-            count += 1
-
-    # Names
-    count = 0
-    if len(allList[2]) != 0:
-        for var in allList[2]:
-            translatedText = translatedText.replace('<N' + str(count) + '>', var)
-            count += 1
-
-    # Vars
-    count = 0
-    if len(allList[3]) != 0:
-        for var in allList[3]:
-            translatedText = translatedText.replace('<V' + str(count) + '>', var)
-            count += 1
-    
-    # Formatting
-    count = 0
-    if len(allList[4]) != 0:
-        for var in allList[4]:
-            translatedText = translatedText.replace('<F' + str(count) + '>', var)
-            count += 1
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateGPT(t, history, fullPromptFlag):
-    # If ESTIMATE is True just count this as an execution and return.
-    if ESTIMATE:
-        enc = tiktoken.encoding_for_model(MODEL)
-        tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
-        return (t, tokens)
-    
-    # Sub Vars
-    varResponse = subVars(t)
-    subbedT = varResponse[0]
-
-    # If there isn't any Japanese in the text just skip
-    if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
-        return(t, 0)
-
-    # Characters
-    context = '```\
-        Game Characters:\
-        Character: 池ノ上 拓海 == Ikenoue Takumi - Gender: Male\
-        Character: 福永 こはる == Fukunaga Koharu - Gender: Female\
-        Character: 神泉 理央 == Kamiizumi Rio - Gender: Female\
-        Character: 吉祥寺 アリサ == Kisshouji Arisa - Gender: Female\
-        Character: 久我 友里子 == Kuga Yuriko - Gender: Female\
-        ```'
-
-    # Prompt
-    if fullPromptFlag:
-        system = PROMPT
-        user = 'Line to Translate = ' + subbedT
-    else:
-        system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`' 
-        user = 'Line to Translate = ' + subbedT
-
-     # Create Message List
-    msg = []
-    msg.append({"role": "system", "content": system})
-    msg.append({"role": "user", "content": context})
-    if isinstance(history, list):
-        for line in history:
-            msg.append({"role": "user", "content": line})
-    else:
-        msg.append({"role": "user", "content": history})
-    msg.append({"role": "user", "content": user})
-
-    response = openai.ChatCompletion.create(
-        temperature=0.1,
-        frequency_penalty=0.2,
-        presence_penalty=0.2,
-        model=MODEL,
-        messages=msg,
-        request_timeout=TIMEOUT,
-    )
-
-    # Save Translated Text
-    translatedText = response.choices[0].message.content
-    tokens = response.usage.total_tokens
-
-    # Resub Vars
-    translatedText = resubVars(translatedText, varResponse[1])
-
-    # Remove Placeholder Text
-    translatedText = translatedText.replace(LANGUAGE +' Translation: ', '')
-    translatedText = translatedText.replace('Translation: ', '')
-    translatedText = translatedText.replace('Line to Translate = ', '')
-    translatedText = translatedText.replace('Translation = ', '')
-    translatedText = translatedText.replace('Translate = ', '')
-    translatedText = translatedText.replace(LANGUAGE +' Translation:', '')
-    translatedText = translatedText.replace('Translation:', '')
-    translatedText = translatedText.replace('Line to Translate =', '')
-    translatedText = translatedText.replace('Translation =', '')
-    translatedText = translatedText.replace('Translate =', '')
-    translatedText = re.sub(r'Note:.*', '', translatedText)
-    translatedText = translatedText.replace('っ', '')
-
-    # Return Translation
-    if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
-        raise Exception
-    else:
-        return [translatedText, tokens]

vocab.txt

@@ -34,6 +34,8 @@ Here are some vocabulary and terms so that you know the proper spelling and tran
 魔法力 (M. Power)
 命中率 (Accuracy)
 %1 の%2を獲得！ (Gained %1 %2!)
+持っている数 (Owned)
+ME 音量 (ME Volume)
 
 # Other
 ソル (Sol)