Remove outdated alltext.py
This commit is contained in:
parent
ddb3d9196a
commit
178a09367f
1 changed files with 0 additions and 172 deletions
|
|
@ -1,172 +0,0 @@
|
|||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import tiktoken
|
||||
|
||||
from colorama import Fore
|
||||
from dotenv import load_dotenv
|
||||
import openai
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
|
||||
# Globals
|
||||
load_dotenv()
|
||||
if os.getenv('api').replace(' ', '') != '':
|
||||
openai.api_base = os.getenv('api')
|
||||
|
||||
openai.organization = os.getenv('org')
|
||||
openai.api_key = os.getenv('key')
|
||||
MODEL = os.getenv('model')
|
||||
TIMEOUT = int(os.getenv('timeout'))
|
||||
LANGUAGE = os.getenv('language').capitalize()
|
||||
|
||||
APICOST = .002 # Depends on the model https://openai.com/pricing
|
||||
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
|
||||
THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
|
||||
LOCK = threading.Lock()
|
||||
WIDTH = int(os.getenv('width'))
|
||||
LISTWIDTH = int(os.getenv('listWidth'))
|
||||
MAXHISTORY = 10
|
||||
ESTIMATE = ''
|
||||
TOTALCOST = 0
|
||||
TOKENS = 0
|
||||
TOTALTOKENS = 0
|
||||
NAMESLIST = []
|
||||
|
||||
# tqdm Globals
|
||||
BAR_FORMAT = '{l_bar}{bar:10}{r_bar}{bar:-10b}'
|
||||
POSITION = 0
|
||||
LEAVE = False
|
||||
|
||||
def handleAllText(filename, estimate):
|
||||
global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
|
||||
ESTIMATE = estimate
|
||||
|
||||
with open('translated/' + filename, 'w+t', newline='', encoding='utf-16-le') as writeFile:
|
||||
start = time.time()
|
||||
translatedData = openFiles(filename, writeFile)
|
||||
|
||||
if estimate:
|
||||
# Print Result
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(['', TOKENS, None], end - start, filename))
|
||||
TOTALCOST += TOKENS * .001 * APICOST
|
||||
TOTALTOKENS += TOKENS
|
||||
TOKENS = 0
|
||||
os.remove('translated/' + filename)
|
||||
|
||||
else:
|
||||
# Print Result
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(translatedData, end - start, filename))
|
||||
TOTALCOST += translatedData[1] * .001 * APICOST
|
||||
TOTALTOKENS += translatedData[1]
|
||||
|
||||
return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
|
||||
|
||||
def openFiles(filename, writeFile):
|
||||
with open('files/' + filename, 'r', encoding='utf-16-le') as readFile, writeFile:
|
||||
translatedData = parseText(readFile, writeFile, filename)
|
||||
|
||||
return translatedData
|
||||
|
||||
def getResultString(translatedData, translationTime, filename):
|
||||
# File Print String
|
||||
tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \
|
||||
' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']'
|
||||
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
|
||||
|
||||
if translatedData[2] == None:
|
||||
# Success
|
||||
return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
|
||||
|
||||
else:
|
||||
# Fail
|
||||
try:
|
||||
raise translatedData[2]
|
||||
except Exception as e:
|
||||
errorString = str(e) + '|' + translatedData[3] + Fore.RED
|
||||
return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' + \
|
||||
errorString + Fore.RESET
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(t, history, fullPromptFlag):
|
||||
# If ESTIMATE is True just count this as an execution and return.
|
||||
if ESTIMATE:
|
||||
enc = tiktoken.encoding_for_model(MODEL)
|
||||
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
|
||||
return (t, tokens)
|
||||
|
||||
# Sub Vars
|
||||
varResponse = subVars(t)
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
|
||||
return (t, 0)
|
||||
|
||||
# Characters
|
||||
context = '```\
|
||||
Character: 池ノ上 拓海 == Ikenoue Takumi - Gender: Male\
|
||||
Character: 福永 こはる == Fukunaga Koharu - Gender: Female\
|
||||
Character: 神泉 理央 == Kamiizumi Rio - Gender: Female\
|
||||
Character: 吉祥寺 アリサ == Kisshouji Arisa - Gender: Female\
|
||||
Character: 久我 友里子 == Kuga Yuriko - Gender: Female\
|
||||
```'
|
||||
|
||||
# Prompt
|
||||
if fullPromptFlag:
|
||||
system = PROMPT
|
||||
user = 'Line to Translate = ' + subbedT
|
||||
else:
|
||||
system = 'Output ONLY the ' + LANGUAGE + ' translation in the following format: `Translation: <' + LANGUAGE.upper() + '_TRANSLATION>`'
|
||||
user = 'Line to Translate = ' + subbedT
|
||||
|
||||
# Create Message List
|
||||
msg = []
|
||||
msg.append({"role": "system", "content": system})
|
||||
msg.append({"role": "user", "content": context})
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
msg.append({"role": "user", "content": line})
|
||||
else:
|
||||
msg.append({"role": "user", "content": history})
|
||||
msg.append({"role": "user", "content": user})
|
||||
|
||||
response = openai.ChatCompletion.create(
|
||||
temperature=0.1,
|
||||
frequency_penalty=0.2,
|
||||
presence_penalty=0.2,
|
||||
model=MODEL,
|
||||
messages=msg,
|
||||
request_timeout=TIMEOUT,
|
||||
)
|
||||
|
||||
# Save Translated Text
|
||||
translatedText = response.choices[0].message.content
|
||||
tokens = response.usage.total_tokens
|
||||
|
||||
# Resub Vars
|
||||
translatedText = resubVars(translatedText, varResponse[1])
|
||||
|
||||
# Remove Placeholder Text
|
||||
translatedText = translatedText.replace(LANGUAGE + ' Translation: ', '')
|
||||
translatedText = translatedText.replace('Translation: ', '')
|
||||
translatedText = translatedText.replace('Line to Translate = ', '')
|
||||
translatedText = translatedText.replace(LANGUAGE + ' Translation = ', '')
|
||||
translatedText = translatedText.replace('Translate = ', '')
|
||||
translatedText = re.sub(r'Note:.*', '', translatedText)
|
||||
translatedText = translatedText.replace('っ', '')
|
||||
|
||||
# Return Translation
|
||||
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
|
||||
raise Exception
|
||||
else:
|
||||
return [translatedText, tokens]
|
||||
Loading…
Reference in a new issue