dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add More Variables and Start scripts

Browse source
This commit is contained in:
m5kro 2023-11-15 21:21:25 -08:00
parent 7fa94430a0
commit c1f480e93b
16 changed files with 119 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
.DS_Store vendored Normal file
View file

Binary file not shown.

9
.env.example
View file

@ -1,2 +1,7 @@
key="<OPEN_AI_KEY>"
organization="<ORG_KEY>"
key="<OPEN_AI_KEY_OR_OTHER_KEY>"
organization="<ORG_KEY_OR_RANDOM>"
proxy="<PROXY_LINK>"
model="<LLM_MODEL_NAME>"
timeout="<TIMEOUT_SECONDS>"
fileThreads="<NUMBER_OF_FILES_PER_TIME>"
threads="<NUMBER_OF_THREADS_PER_FILE>"

BIN
modules/.DS_Store vendored Normal file
View file

Binary file not shown.

27
modules/alltext.py
View file

@ -18,12 +18,15 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
LOCK = threading.Lock()
WIDTH = 60
LISTWIDTH = 60
@ -94,7 +97,7 @@ def getResultString(translatedData, translationTime, filename):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -103,17 +106,17 @@ def translateGPT(t, history, fullPromptFlag):
subbedT = varResponse[0]
# If there isn't any Japanese in the text just skip
if not re.search(r'[一-龠]+|[ぁ-?]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
if not re.search(r'[<EFBFBD><EFBFBD>-<2D>]+|[<5B><>-?]+|[<5B>@-<2D><>]+|[\uFF00-\uFFEF]', subbedT):
return(t, 0)
# Characters
context = '```\
Game Characters:\
Character: 池ノ上 拓海 == Ikenoue Takumi - Gender: Male\
Character: 福永 こはる == Fukunaga Koharu - Gender: Female\
Character: 神泉 理央 == Kamiizumi Rio - Gender: Female\
Character: 吉祥寺 アリサ == Kisshouji Arisa - Gender: Female\
Character: 久我 友里子 == Kuga Yuriko - Gender: Female\
Character: <EFBFBD>r<EFBFBD>m<EFBFBD><EFBFBD> <EFBFBD><EFBFBD>C == Ikenoue Takumi - Gender: Male\
Character: <EFBFBD><EFBFBD><EFBFBD>i <EFBFBD><EFBFBD><EFBFBD>͂<EFBFBD> == Fukunaga Koharu - Gender: Female\
Character: <EFBFBD>_<EFBFBD><EFBFBD> <EFBFBD><EFBFBD><EFBFBD><EFBFBD> == Kamiizumi Rio - Gender: Female\
Character: <EFBFBD>g<EFBFBD>ˎ<EFBFBD> <EFBFBD>A<EFBFBD><EFBFBD><EFBFBD>T == Kisshouji Arisa - Gender: Female\
Character: <EFBFBD>v<EFBFBD><EFBFBD> <EFBFBD>F<EFBFBD><EFBFBD><EFBFBD>q == Kuga Yuriko - Gender: Female\
```'
# Prompt
@ -139,9 +142,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -163,10 +166,10 @@ def translateGPT(t, history, fullPromptFlag):
translatedText = translatedText.replace('Translation =', '')
translatedText = translatedText.replace('Translate =', '')
translatedText = re.sub(r'Note:.*', '', translatedText)
translatedText = translatedText.replace('っ', '')
translatedText = translatedText.replace('<EFBFBD><EFBFBD>', '')
# Return Translation
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

13
modules/csv.py
View file

@ -18,12 +18,15 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 20
THREADS = int(os.getenv('threads'))
LOCK = threading.Lock()
WIDTH = 60
MAXHISTORY = 10
@ -320,7 +323,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -365,9 +368,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -395,4 +398,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

13
modules/json.py
View file

@ -17,13 +17,16 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # Controls how many threads are working on a single file (May have to drop this)
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = 50
LISTWIDTH = 90
@ -293,7 +296,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
historyRaw = ''
if isinstance(history, list):
for line in history:
@ -346,9 +349,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo-1106",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -376,4 +379,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]
return [translatedText, totalTokens]

13
modules/kansen.py
View file

@ -16,12 +16,15 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
LOCK = threading.Lock()
WIDTH = 60
LISTWIDTH = 60
@ -461,7 +464,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -506,9 +509,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -536,4 +539,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

13
modules/lune.py
View file

@ -17,13 +17,16 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # Controls how many threads are working on a single file (May have to drop this)
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = 50
LISTWIDTH = 90
@ -293,7 +296,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
historyRaw = ''
if isinstance(history, list):
for line in history:
@ -346,9 +349,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo-1106",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -376,4 +379,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]
return [translatedText, totalTokens]

13
modules/lune2.py
View file

@ -18,12 +18,15 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 20
THREADS = int(os.getenv('threads'))
LOCK = threading.Lock()
WIDTH = 60
LISTWIDTH = 75
@ -347,7 +350,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -392,9 +395,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -422,4 +425,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

2
modules/main.py
View file

@ -17,7 +17,7 @@ from modules.lune2 import handleLuneTxt
# For GPT4 rate limit will be hit if you have more than 1 thread.
# 1 Thread for each file. Controls how many files are worked on at once.
THREADS = 5
THREADS = int(os.getenv('fileThreads'))
# Info Message
tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once a translation starts do not close it unless you want to lose your\

13
modules/rpgmakerace.py
View file

@ -19,13 +19,16 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # Controls how many threads are working on a single file (May have to drop this)
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = 50
LISTWIDTH = 90
@ -1757,7 +1760,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
historyRaw = ''
if isinstance(history, list):
for line in history:
@ -1811,9 +1814,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -1841,4 +1844,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]
return [translatedText, totalTokens]

13
modules/rpgmakermvmz.py
View file

@ -18,13 +18,16 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # Controls how many threads are working on a single file (May have to drop this)
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = 40
LISTWIDTH = 50
@ -1758,7 +1761,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
historyRaw = ''
if isinstance(history, list):
for line in history:
@ -1813,9 +1816,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo-1106",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -1843,4 +1846,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]
return [translatedText, totalTokens]

13
modules/txt.py
View file

@ -18,12 +18,15 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 20
THREADS = int(os.getenv('threads'))
LOCK = threading.Lock()
WIDTH = 75
LISTWIDTH = 75
@ -305,7 +308,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -350,9 +353,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -380,4 +383,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

13
modules/tyrano.py
View file

@ -16,12 +16,15 @@ from tqdm import tqdm
#Globals
load_dotenv()
openai.api_base = os.getenv('proxy')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
LOCK = threading.Lock()
WIDTH = 80
LISTWIDTH = 60
@ -475,7 +478,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -517,9 +520,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -547,4 +550,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

13
start.bat Normal file
View file

@ -0,0 +1,13 @@
@echo off
set /p choice=Do you want to edit the .env file? (y/n):
if /i "%choice%"=="y" (
notepad .env
echo Press enter when you are done
pause
echo Continuing...
) else (
echo Continuing without editing .env file...
)
python3 -m pip install -r requirements.txt
python3 start.py
pause

11
start.sh Executable file
View file

@ -0,0 +1,11 @@
#!/bin/bash
read -p "Do you want to edit the .env file? (y/n): " choice
if [ "$choice" == "y" ] || [ "$choice" == "Y" ]; then
nano .env
echo "Continuing..."
else
echo "Continuing without editing .env file..."
fi
python3 -m pip install -r requirements.txt
python3 start.py