dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #9 from m5kro/Other-API

Browse source 
Add Support for Other APIs and some other QOL changes
This commit is contained in:
dazedanon 2023-11-19 05:16:19 -06:00 committed by GitHub
commit c9c4df9d38
14 changed files with 202 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
.env.example
View file

@ -1,4 +1,30 @@
key="<OPEN_AI_KEY>"
organization="<ORG_KEY>"
#API link, leave blank to use OpenAI API
api="<API_LINK>"
#API key
key="<OPEN_AI_KEY_OR_OTHER_KEY>"
#Oranization key, make something up for self hosted or other API
organization="<ORG_KEY_OR_RANDOM>"
#LLM model name, use gpt-3.5-turbo-1106 or gpt-3.5-turbo or gpt-4 for OpenAI API
#For text generation webui use gpt-3.5-turbo, for other API's consult their documentation
model="<LLM_MODEL_NAME>"
#The language to translate TO, Don't forget to change the prompt
language="<LANGUAGE_NAME>"
#The timeout before disconnect error, 30 to 120 recommended
timeout="<TIMEOUT_SECONDS>"
#The number of files to translate at the same time, 1 recommended for free or self hosted API or gpt-4
fileThreads="<NUMBER_OF_FILES_PER_TIME>"
#The number of threads per file, 1 recommended for free or self hosted API or gpt-4
threads="<NUMBER_OF_THREADS_PER_FILE>"
#The wordwrap of dialogue text
width="<DIALOGUE_TEXT_WORDWRAP>"
#The wordwap of items and help text
listWidth="<ITEM_TEXT_WORDWRAP>"

22
modules/alltext.py
View file

@ -18,16 +18,24 @@ from tqdm import tqdm
# Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE = os.getenv('language').capitalize()
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
LOCK = threading.Lock()
WIDTH = 60
LISTWIDTH = 60
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
MAXHISTORY = 10
ESTIMATE = ''
TOTALCOST = 0
@ -95,7 +103,7 @@ def getResultString(translatedData, translationTime, filename):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -139,9 +147,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -164,4 +172,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

20
modules/csv.py
View file

@ -18,15 +18,23 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 20
THREADS = int(os.getenv('threads'))
LOCK = threading.Lock()
WIDTH = 60
WIDTH = int(os.getenv('width'))
MAXHISTORY = 10
ESTIMATE = ''
TOTALCOST = 0
@ -321,7 +329,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -366,9 +374,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -396,4 +404,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

22
modules/json.py
View file

@ -17,17 +17,25 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # Controls how many threads are working on a single file (May have to drop this)
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = 50
LISTWIDTH = 90
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
NOTEWIDTH = 50
MAXHISTORY = 10
ESTIMATE = ''
@ -294,7 +302,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
historyRaw = ''
if isinstance(history, list):
for line in history:
@ -347,9 +355,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo-1106",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -377,4 +385,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]
return [translatedText, totalTokens]

22
modules/kansen.py
View file

@ -16,16 +16,24 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
LOCK = threading.Lock()
WIDTH = 60
LISTWIDTH = 60
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
MAXHISTORY = 10
ESTIMATE = ''
TOTALCOST = 0
@ -462,7 +470,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -507,9 +515,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -537,4 +545,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

22
modules/lune.py
View file

@ -17,17 +17,25 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # Controls how many threads are working on a single file (May have to drop this)
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = 50
LISTWIDTH = 90
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
NOTEWIDTH = 50
MAXHISTORY = 10
ESTIMATE = ''
@ -294,7 +302,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
historyRaw = ''
if isinstance(history, list):
for line in history:
@ -347,9 +355,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo-1106",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -377,4 +385,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]
return [translatedText, totalTokens]

22
modules/lune2.py
View file

@ -18,16 +18,24 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 20
THREADS = int(os.getenv('threads'))
LOCK = threading.Lock()
WIDTH = 60
LISTWIDTH = 75
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
MAXHISTORY = 10
ESTIMATE = ''
TOTALCOST = 0
@ -348,7 +356,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -393,9 +401,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -423,4 +431,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

2
modules/main.py
View file

@ -17,7 +17,7 @@ from modules.lune2 import handleLuneTxt
# For GPT4 rate limit will be hit if you have more than 1 thread.
# 1 Thread for each file. Controls how many files are worked on at once.
THREADS = 5
THREADS = int(os.getenv('fileThreads'))
# Info Message
tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once a translation starts do not close it unless you want to lose your\

22
modules/rpgmakerace.py
View file

@ -19,17 +19,25 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # Controls how many threads are working on a single file (May have to drop this)
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = 50
LISTWIDTH = 90
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
NOTEWIDTH = 50
MAXHISTORY = 10
ESTIMATE = ''
@ -1795,7 +1803,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
historyRaw = ''
if isinstance(history, list):
for line in history:
@ -1850,9 +1858,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo-1106",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -1879,4 +1887,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]
return [translatedText, totalTokens]

22
modules/rpgmakermvmz.py
View file

@ -18,17 +18,25 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # Controls how many threads are working on a single file (May have to drop this)
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = 40
LISTWIDTH = 40
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
NOTEWIDTH = 40
MAXHISTORY = 10
ESTIMATE = ''
@ -1790,7 +1798,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
historyRaw = ''
if isinstance(history, list):
for line in history:
@ -1845,9 +1853,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo-1106",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -1874,4 +1882,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]
return [translatedText, totalTokens]

22
modules/txt.py
View file

@ -18,16 +18,24 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 20
THREADS = int(os.getenv('threads'))
LOCK = threading.Lock()
WIDTH = 75
LISTWIDTH = 75
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
MAXHISTORY = 10
ESTIMATE = ''
TOTALCOST = 0
@ -306,7 +314,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -351,9 +359,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -381,4 +389,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

22
modules/tyrano.py
View file

@ -16,16 +16,24 @@ from tqdm import tqdm
#Globals
load_dotenv()
if not os.getenv('api').replace(" ", ""):
print('No API given, defaulting to OpenAI API')
else:
openai.api_base = os.getenv('api')
print('Using ' + os.getenv('api') + ' as API')
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
LOCK = threading.Lock()
WIDTH = 80
LISTWIDTH = 60
WIDTH = int(os.getenv('width'))
LISTWIDTH = int(os.getenv('listWidth'))
MAXHISTORY = 10
ESTIMATE = ''
TOTALCOST = 0
@ -476,7 +484,7 @@ def resubVars(translatedText, allList):
def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
enc = tiktoken.encoding_for_model(MODEL)
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
return (t, tokens)
@ -518,9 +526,9 @@ def translateGPT(t, history, fullPromptFlag):
temperature=0.1,
frequency_penalty=0.2,
presence_penalty=0.2,
model="gpt-3.5-turbo",
model=MODEL,
messages=msg,
request_timeout=30,
request_timeout=TIMEOUT,
)
# Save Translated Text
@ -548,4 +556,4 @@ def translateGPT(t, history, fullPromptFlag):
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, tokens]
return [translatedText, tokens]

13
start.bat Normal file
View file

@ -0,0 +1,13 @@
@echo off
set /p choice=Do you want to edit the .env file? (y/n):
if /i "%choice%"=="y" (
notepad .env
echo Press enter when you are done
pause
echo Continuing...
) else (
echo Continuing without editing .env file...
)
python3 -m pip install -r requirements.txt
python3 start.py
pause

11
start.sh Executable file
View file

@ -0,0 +1,11 @@
#!/bin/bash
read -p "Do you want to edit the .env file? (y/n): " choice
if [ "$choice" == "y" ] || [ "$choice" == "Y" ]; then
nano .env
echo "Continuing..."
else
echo "Continuing without editing .env file..."
fi
python3 -m pip install -r requirements.txt
python3 start.py