diff --git a/modules/anim.py b/modules/anim.py
index aa37497..1b6dbda 100644
--- a/modules/anim.py
+++ b/modules/anim.py
@@ -1,51 +1,54 @@
-import json
-import os
+# Libraries
+import json, os, re, textwrap, threading, time, traceback, tiktoken, openai
from pathlib import Path
-import re
-import sys
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-
from colorama import Fore
from dotenv import load_dotenv
-import openai
from retry import retry
from tqdm import tqdm
-#Globals
+# Open AI
load_dotenv()
if os.getenv('api').replace(' ', '') != '':
openai.api_base = os.getenv('api')
-
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
+
+#Globals
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE=os.getenv('language').capitalize()
-
-INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
-OUTPUTAPICOST = .002
+LANGUAGE = os.getenv('language').capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
+THREADS = int(os.getenv('threads'))
LOCK = threading.Lock()
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
-NOTEWIDTH = 50
+NOTEWIDTH = 70
MAXHISTORY = 10
ESTIMATE = ''
-totalTokens = [0, 0]
+TOKENS = [0, 0]
NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
-POSITION=0
-LEAVE=False
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True
-IGNORETLTEXT = True
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if 'gpt-3.5' in MODEL:
+ INPUTAPICOST = .002
+ OUTPUTAPICOST = .002
+ BATCHSIZE = 10
+elif 'gpt-4' in MODEL:
+ INPUTAPICOST = .01
+ OUTPUTAPICOST = .03
+ BATCHSIZE = 50
def handleAnim(filename, estimate):
global ESTIMATE, totalTokens
@@ -129,64 +132,61 @@ def parseJSON(data, filename):
pbar.desc=filename
pbar.total=totalLines
try:
- result = translateJSON(data, pbar)
+ keys = list(data.keys())
+ batches = [keys[i:i + 20] for i in range(0, len(keys), 20)]
+ result = translateJSON(batches, data, pbar)
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
return [data, totalTokens, e]
return [data, totalTokens, None]
-def translateJSON(data, pbar):
+def translateJSON(keys, data, pbar):
textHistory = []
- maxHistory = MAXHISTORY
tokens = [0, 0]
- for key, value in data.items():
- # Text
- if value == "":
- jaString = key
- else:
- jaString = value
+ for batch in keys:
+ # Save Batch
+ originalBatch = batch
- # Check if TLed
# If there isn't any Japanese in the text just skip
if IGNORETLTEXT is True:
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', str(batch)):
pbar.update(1)
continue
# Remove any textwrap
if FIXTEXTWRAP == True:
- jaString = jaString.replace('\n', ' ')
+ for i in range(len(batch)):
+ data[originalBatch[i]] = data[originalBatch[i]].replace('\n', ' ')
# Translate
- if jaString != '':
- response = translateGPT(f'{jaString}', textHistory, True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedText = response[0]
- textHistory.append('\"' + translatedText + '\"')
+ response = translateGPT(batch, textHistory, True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+
+ # Format and Set Text
+ if len(batch) == len(translatedBatch):
+ for i in range(len(translatedBatch)):
+
+ # Remove added speaker
+ translatedText = translatedBatch[i]
+ translatedText = re.sub(r'^.+?\s\|\s?', '', translatedText)
+
+ # Textwrap
+ if '\n' not in translatedText:
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+
+ # Set Data
+ data[originalBatch[i]] = translatedText
+ pbar.update(1)
+ # Mismatch, Skip Batch
else:
- translatedText = jaString
- textHistory.append('\"' + translatedText + '\"')
+ MISMATCH.append(batch)
+ continue
- # Remove added speaker
- translatedText = re.sub(r'^.+?\s\|\s?', '', translatedText)
-
- # Textwrap
- if '\n' not in translatedText:
- translatedText = textwrap.fill(translatedText, width=WIDTH)
-
- # Set Data
- data[key] = translatedText
-
- # Keep textHistory list at length maxHistory
- if len(textHistory) > maxHistory:
- textHistory.pop(0)
- currentGroup = []
- pbar.update(1)
-
- return tokens
+ return tokens
def subVars(jaString):
jaString = jaString.replace('\u3000', ' ')