from concurrent.futures import ThreadPoolExecutor, as_completed import json import os from pathlib import Path import re import sys import textwrap import threading import time import traceback import tiktoken from colorama import Fore from dotenv import load_dotenv import openai from retry import retry from tqdm import tqdm #Globals load_dotenv() openai.organization = os.getenv('org') openai.api_key = os.getenv('key') APICOST = .002 # Depends on the model https://openai.com/pricing PROMPT = Path('prompt.txt').read_text(encoding='utf-8') THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread. LOCK = threading.Lock() WIDTH = 60 LISTWIDTH = 60 MAXHISTORY = 10 ESTIMATE = '' TOTALCOST = 0 TOKENS = 0 TOTALTOKENS = 0 NAMESLIST = [] #tqdm Globals BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' POSITION=0 LEAVE=False def handleAllText(filename, estimate): global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST ESTIMATE = estimate with open('translated/' + filename, 'w+t', newline='', encoding='utf-16-le') as writeFile: start = time.time() translatedData = openFiles(filename, writeFile) if estimate: # Print Result end = time.time() tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) TOTALCOST += TOKENS * .001 * APICOST TOTALTOKENS += TOKENS TOKENS = 0 os.remove('translated/' + filename) else: # Print Result end = time.time() tqdm.write(getResultString(translatedData, end - start, filename)) TOTALCOST += translatedData[1] * .001 * APICOST TOTALTOKENS += translatedData[1] return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') def openFiles(filename, writeFile): with open('files/' + filename, 'r', encoding='utf-16-le') as readFile, writeFile: translatedData = parseText(readFile, writeFile, filename) return translatedData def getResultString(translatedData, translationTime, filename): # File Print String tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \ ' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']' timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' if translatedData[2] == None: # Success return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET else: # Fail try: raise translatedData[2] except Exception as e: errorString = str(e) + '|' + translatedData[3] + Fore.RED return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\ errorString + Fore.RESET