Fix some errors and TL UI

.gitignore

@@ -8,4 +8,5 @@
 !requirements.txt
 *.csv
 *.ks
+*.cid
 __pycache__

modules/tyrano.py

@@ -14,6 +14,7 @@ openai.organization = os.getenv('org')
 openai.api_key = os.getenv('key')
 
 #Globals
+PBAR = None
 MODEL = os.getenv('model')
 TIMEOUT = int(os.getenv('timeout'))
 LANGUAGE = os.getenv('language').capitalize()
@@ -49,7 +50,7 @@ if 'gpt-3.5' in MODEL:
 elif 'gpt-4' in MODEL:
     INPUTAPICOST = .01
     OUTPUTAPICOST = .03
-    BATCHSIZE = 10
+    BATCHSIZE = 40
 
 def handleTyrano(filename, estimate):
     global ESTIMATE
@@ -137,14 +138,12 @@ def parseTyrano(readFile, filename):
 
     # Get total for progress bar
     data = readFile.readlines()
-    totalLines = len(data)
 
     with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
         pbar.desc=filename
-        pbar.total=totalLines
 
         try:
-            result = translateTyrano(data, pbar, totalLines)
+            result = translateTyrano(data, pbar, filename, False, [[],[]])
             totalTokens[0] += result[0]
             totalTokens[1] += result[1]
         except Exception as e:
@@ -152,86 +151,82 @@ def parseTyrano(readFile, filename):
             return [data, totalTokens, e]
     return [data, totalTokens, None]
 
-def translateTyrano(data, pbar, totalLines):
+def translateTyrano(data, pbar, filename, setData, jobList):
     textHistory = []
-    batch = []
+    lineList = jobList[0]
-    currentGroup = []
+    totalTokens = [0,0]
-    maxHistory = MAXHISTORY
-    tokens = [0,0]
     speaker = ''
-    insertBool = False
     global LOCK, ESTIMATE
     i = 0
-    batchStartIndex = 0
+
+    # Set Progress Bar
+    global PBAR 
+    PBAR = pbar
 
     while i < len(data):
+        # Choices
+        choiceList = []
+        choiceRegex = r'\[font.+\](.+)\/$'
+        if 'font' in data[i]:
+            match = re.search(choiceRegex, data[i])
+            if match != None:
+                choiceList.append(match.group(1))
+                i += 1
+
+                # Grab them all up for list
+                while(i < len(data) and 'font' in data[i]):
+                    match = re.search(choiceRegex, data[i])
+                    if match != None:
+                        choiceList.append(match.group(1))
+                    i += 1
+                
+                # Translate
+                if len(choiceList) != 0:
+                    response = translateGPT(choiceList, 'This will be a dialogue option', True)
+                    choiceListTL = response[0]
+                    totalTokens[0] += response[1][0]
+                    totalTokens[1] += response[1][1]
+                
+                # Set Data
+                if len(choiceList) == len(choiceListTL):
+                    i = i - len(choiceListTL)
+                    for j in range(len(choiceListTL)):
+                        data[i] = data[i].replace(choiceList[j], choiceListTL[j])
+                        i += 1
+                else:
+                    with LOCK:
+                        if filename not in MISMATCH:
+                            MISMATCH.append(filename)
+                   
         # Speaker
-        if '#' in data[i]:
+        if '[@]' in data[i]:
-            matchList = re.findall(r'^#(.*)', data[i])
+            if 'FACE' not in data[i]:
-            if len(matchList) != 0:
+                matchList = re.findall(r'd^\[(.*?)\].+\[.+\]$', data[i])
+            else:
+                matchList = re.findall(r'd^\[.+\]\[(.*?)\].+\[.+\]$', data[i])
+            if len(matchList) != 0 and '=' not in matchList[0] and re.search(r'\[.+\]', matchList[0]) == None:
                 response = getSpeaker(matchList[0])
                 speaker = response[0]
-                tokens[0] += response[1][0]
+                totalTokens[0] += response[1][0]
-                tokens[1] += response[1][1]
+                totalTokens[1] += response[1][1]
-                data[i] = '#' + speaker + '\n'
+                # data[i] = data[i].replace(matchList[0], f'{speaker}')
             else:
                 speaker = ''
-
+                   
-        # Choices
-        elif '[sel' in data[i]:
-            matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i])
-            if len(matchList) != 0:
-                originalText = matchList[0]
-                if len(textHistory) > 0:
-                    response = translateGPT(matchList[0], 'Keep your translation as brief as possible. Previous text for context: ' + textHistory[len(textHistory)-1] + '\n\nReply in the style of a dialogue option.', False)
-                else:
-                    response = translateGPT(matchList[0], '\n\nReply in the style of a dialogue option.', False)
-                translatedText = response[0]
-                tokens[0] += response[1][0]
-                tokens[1] += response[1][1]
-
-                # Remove characters that may break scripts
-                charList = ['.', '\"', '\\n']
-                for char in charList:
-                    translatedText = translatedText.replace(char, '')
-
-                # Escape all '
-                translatedText = translatedText.replace('\\', '')
-                # translatedText = translatedText.replace("'", "\\\'")
-
-                # Set Data
-                translatedText = data[i].replace(originalText, translatedText)
-                data[i] = translatedText                
-
         # Lines
-        matchList = re.findall(r'(.+?)\[[rpc]+\]$', data[i]) 
+        if '-FACE' not in data[i]:
-        if len(matchList) > 0:
+            matchList = re.findall(r'd^\[.*?\](.+)\[.+\]$', data[i])
-            if 'hisout' in matchList[0]:
+        else:
-                i += 1
+            matchList = re.findall(r'd^\[.+\]\[.*?\](.+)\[.+\]$', data[i]) 
-                continue
+        if len(matchList) > 0 and '=' not in matchList[0] and re.search(r'\[.+\]', matchList[0]) == None:
-            currentGroup.append(matchList[0])
+            # Remove [r] and [l]
-            if len(data) > i+1:
+            oldjaString = matchList[0]
-                while '[r]' in data[i+1]:
+            jaString = oldjaString
-                    if insertBool is True:
+            jaString = jaString.replace('[r]', ' ')
-                        data[i] = '\d\n'
+            jaString = jaString.replace('[l]', '')
-                        pbar.update(1)
-                    i += 1
-                    matchList = re.findall(r'(.+?)\[r\]', data[i])
-                    if len(matchList) > 0:
-                        currentGroup.append(matchList[0])
-                while '[p]' in data[i+1]:
-                    if insertBool is True:
-                        data[i] = '\d\n'
-                        pbar.update(1)
-                    i += 1
-            # Join up 401 groups for better translation.
-            if len(currentGroup) > 0:
-                finalJAString = ' '.join(currentGroup)
-                oldjaString = finalJAString
 
-            # Remove any textwrap
+            # Join up 401 groups for better translation.
-            if FIXTEXTWRAP == True:
+            finalJAString = jaString
-                finalJAString = finalJAString.replace('[r]', ' ')
 
             # Remove Extra Stuff bad for translation.
             finalJAString = finalJAString.replace('ﾞ', '')
@@ -243,6 +238,8 @@ def translateTyrano(data, pbar, totalLines):
             finalJAString = finalJAString.replace('…', '...')
             finalJAString = re.sub(r'(\.{3}\.+)', '...', finalJAString)
             finalJAString = finalJAString.replace('　', ' ')
+            finalJAString = finalJAString.replace('】', ')')
+            finalJAString = finalJAString.replace('【　', '(')
 
             # Furigana Removal
             matchList = re.findall(r'(\[ruby\stext=.+text=\"(.+)\"\])', finalJAString)
@@ -253,104 +250,53 @@ def translateTyrano(data, pbar, totalLines):
             if speaker != '':
                 finalJAString = f'{speaker}: {finalJAString}'
 
-            # [Passthrough 1] Pulling From File
+            # [Passthrough 1] Append To List
-            if insertBool is False:
+            if setData is False:
-                # Append to List and Clear Values
+                lineList.append(finalJAString)
-                batch.append(finalJAString)
+            
-                speaker = ''
+            # [Passthrough 2] Set Data
-
-                # Translate Batch if Full
-                if len(batch) == BATCHSIZE:
-                    # Translate
-                    response = translateGPT(batch, textHistory, True)
-                    tokens[0] += response[1][0]
-                    tokens[1] += response[1][1]
-                    translatedBatch = response[0]
-                    textHistory = translatedBatch[-10:]
-
-                    # Set Values
-                    if len(batch) == len(translatedBatch):
-                        i = batchStartIndex
-                        insertBool = True
-
-                    # Mismatch
-                    else:
-                        pbar.write(f'Mismatch: {batchStartIndex} - {i}')
-                        MISMATCH.append(batch)
-                        batchStartIndex = i
-                        batch.clear()
-
-                i += 1
-                if insertBool is True:
-                    pbar.update(1)
-                currentGroup = []
-
-            # [Passthrough 2] Setting Data
             else:
-                # Get Text
+                # Grab and Pop
-                translatedText = translatedBatch[0]
+                translatedText = lineList[0]
-                translatedText = translatedText.replace('\\"', '\"')
+                lineList.pop(0)
-                translatedText = translatedText.replace('[', '(')
-                translatedText = translatedText.replace(']', ')')
 
-                # Remove added speaker
+                # Remove speaker
-                translatedText = re.sub(r'^.+?:\s', '', translatedText)
+                translatedText = re.sub(r'^\[?(.+?)\]?\s?[|:]\s?', '', translatedText)
 
                 # Textwrap
-                translatedText = textwrap.fill(translatedText, width=WIDTH)
+                translatedText = textwrap.fill(translatedText, WIDTH)
-                textList = translatedText.split('\n')
+                translatedText = translatedText.replace('\n', '[r]')
-                    
-                # Set Text
-                data[i] = '\d\n'
-                for line in textList:
-                    # Wordwrap Text
-                    if '[r]' not in line:
-                        line = textwrap.fill(line, width=WIDTH)
-                        line = line.replace('\n', '[r]')
-                    
-                    # Set
-                    data.insert(i, line.strip() + '[r]\n')
-                    i+=1
-                data[i-1] = data[i-1].replace('[r]', '[pcms]')
-                translatedBatch.pop(0)
-                speaker = ''
-                currentGroup = []
 
-                # If Batch is empty. Move on.
+                # Set Data
-                if len(translatedBatch) == 0:
+                data[i] = data[i].replace(oldjaString, translatedText)
-                    insertBool = False
-                    batchStartIndex = i
-                    batch.clear()
 
-        # Nothing relevant. Skip Line.
+        # Next Line
+        i += 1
+    
+    # Translate Data
+    lineListTL = []
+    setData = False
+    
+    # Line List
+    if len(lineList) > 0:
+        pbar.total = len(lineList)
+        pbar.refresh()
+        response = translateGPT(lineList, textHistory, True)
+        lineListTL = response[0]
+        totalTokens[0] += response[1][0]
+        totalTokens[1] += response[1][1]
+        if len(lineListTL) != len(lineList):
+            with LOCK:
+                if filename not in MISMATCH:
+                    MISMATCH.append(filename)
         else:
-            i += 1
+            setData = True
-            if insertBool is True:
-                pbar.update(1)
 
-        # Translate Batch if not empty and EOF
+        # Start Pass 2
-        if len(batch) != 0 and i >= len(data):
+        if setData:
-            # Translate
+            translateTyrano(data, pbar, filename, True, [lineListTL])
-            response = translateGPT(batch, textHistory, True)
-            tokens[0] += response[1][0]
-            tokens[1] += response[1][1]
-            translatedBatch = response[0]
-            textHistory = translatedBatch[-10:]
 
-            # Set Values
+    return totalTokens
-            if len(batch) == len(translatedBatch):
-                i = batchStartIndex
-                insertBool = True
-
-            # Mismatch
-            else:
-                pbar.write(f'Mismatch: {batchStartIndex} - {i}')
-                MISMATCH.append(batch)
-                batchStartIndex = i
-                batch.clear()
-
-            currentGroup = []
-    return tokens
 # Save some money and enter the character before translation
 def getSpeaker(speaker):
     match speaker:
@@ -424,7 +370,7 @@ def subVars(jaString):
 
     # Formatting
     count = 0
-    formatList = re.findall(r'[\\]+[\w]*\[[\w\\\[\]]+\]', jaString)
+    formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
     formatList = set(formatList)
     if len(formatList) != 0:
         for var in formatList:
@@ -495,37 +441,27 @@ def batchList(input_list, batch_size):
 
 def createContext(fullPromptFlag, subbedT):
     characters = 'Game Characters:\n\
-ファイン (Fine) - Female\n\
+眠り姫 (Sleeping Princess) - Female\n\
-ウェンティ (Wendy) - Female\n\
+迷子 (Lost Child) - Male\n\
-クレア (Claire) - Female\n\
-ミナ (Mina) - Female\n\
-サーラ (Sarah) - Female\n\
-ミリェル (Miriel) - Female\n\
-カタリナ (Catalina) - Female\n\
-リリィ (Lily) - Female\n\
-ヴァネット (Vanette) - Female\n\
-セラス (Ceras) - Female\n\
 '
     
     system = PROMPT + VOCAB if fullPromptFlag else \
         f"\
 You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
 Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- Maintain Japanese Honorifics. For example: 'サクラねえちゃん' == 'Sakura Onee-san'\n\
 - You translate everything, including erotic and explicit content.\n\
 - 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in the output must be in English even if it may be hard to translate.\n\
+- All text in your response must be in English even if it is hard to translate.\n\
 - Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
 - Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, etc)\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
 - `...` can be a part of the dialogue. Translate it as it is.\n\
-- Do not include a speaker if there isn't one in the original line of text.\n\
 {VOCAB}\n\
 "
     user = f'{subbedT}'
     return characters, system, user
 
-def translateText(characters, system, user, history):
+def translateText(characters, system, user, history, penalty):
     # Prompt
     msg = [{"role": "system", "content": system + characters}]
 
@@ -541,9 +477,8 @@ def translateText(characters, system, user, history):
     # Content to TL
     msg.append({"role": "user", "content": f'{user}'})
     response = openai.chat.completions.create(
-        temperature=0.1,
+        temperature=0,
-        frequency_penalty=0.1,
+        frequency_penalty=penalty,
-        presence_penalty=0.1,
         model=MODEL,
         messages=msg,
     )
@@ -555,7 +490,6 @@ def cleanTranslatedText(translatedText, varResponse):
         'Translation: ': '',
         'っ': '',
         '〜': '~',
-        'ー': '-',
         'ッ': '',
         '。': '.',
         'Placeholder Text': ''
@@ -564,9 +498,25 @@ def cleanTranslatedText(translatedText, varResponse):
     for target, replacement in placeholders.items():
         translatedText = translatedText.replace(target, replacement)
 
+    # Elongate Long Dashes (Since GPT Ignores them...)
+    translatedText = elongateCharacters(translatedText)
     translatedText = resubVars(translatedText, varResponse[1])
     return translatedText
 
+def elongateCharacters(text):
+    # Define a pattern to match one character followed by one or more `ー` characters
+    # Using a positive lookbehind assertion to capture the preceding character
+    pattern = r'(?<=(.))ー+'
+    
+    # Define a replacement function that elongates the captured character
+    def repl(match):
+        char = match.group(1)  # The character before the ー sequence
+        count = len(match.group(0)) - 1  # Number of ー characters
+        return char * count  # Replace ー sequence with the character repeated
+
+    # Use re.sub() to replace the pattern in the text
+    return re.sub(pattern, repl, text)
+
 def extractTranslation(translatedTextList, is_list):
     pattern = r'`?<Line\d+>([\\]*.*?[\\]*?)<\/?Line\d+>`?'
     # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
@@ -604,6 +554,7 @@ def combineList(tlist, text):
 
 @retry(exceptions=Exception, tries=5, delay=5)
 def translateGPT(text, history, fullPromptFlag):
+    global PBAR
     mismatch = False
     totalTokens = [0, 0]
     if isinstance(text, list):
@@ -615,7 +566,7 @@ def translateGPT(text, history, fullPromptFlag):
         # Before sending to translation, if we have a list of items, add the formatting
         if isinstance(tItem, list):
             payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)])
-            payload = payload.replace('><', '>Placeholder Text<')
+            payload = re.sub(r'(<Line\d+)(><)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload)
             varResponse = subVars(payload)
             subbedT = varResponse[0]
         else:
@@ -637,7 +588,7 @@ def translateGPT(text, history, fullPromptFlag):
             continue
 
         # Translating
-        response = translateText(characters, system, user, history)
+        response = translateText(characters, system, user, history, 0.02)
         translatedText = response.choices[0].message.content
         totalTokens[0] += response.usage.prompt_tokens
         totalTokens[1] += response.usage.completion_tokens
@@ -649,7 +600,7 @@ def translateGPT(text, history, fullPromptFlag):
             tList[index] = extractedTranslations
             if len(tItem) != len(extractedTranslations):
                 # Mismatch. Try Again
-                response = translateText(characters, system, user, history)
+                response = translateText(characters, system, user, history, 0.1)
                 translatedText = response.choices[0].message.content
                 totalTokens[0] += response.usage.prompt_tokens
                 totalTokens[1] += response.usage.completion_tokens
@@ -667,10 +618,11 @@ def translateGPT(text, history, fullPromptFlag):
                 history = extractedTranslations[-10:]  # Update history if we have a list
             else:
                 history = text[-10:]
+            PBAR.update(len(tItem))
         else:
             # Ensure we're passing a single string to extractTranslation
             extractedTranslations = extractTranslation(translatedText, False)
             tList[index] = extractedTranslations
 
     finalList = combineList(tList, text)
-    return [finalList, totalTokens]
+    return [finalList, totalTokens]