dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Updated requirements and a fix to Alice

Browse source
This commit is contained in:
Dazed 2023-12-10 01:29:44 -06:00
parent 309957545a
commit de29ee1ba5
2 changed files with 19 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
modules/alice.py
View file

@ -48,7 +48,7 @@ if 'gpt-3.5' in MODEL:
elif 'gpt-4' in MODEL: elif 'gpt-4' in MODEL:
INPUTAPICOST = .01 INPUTAPICOST = .01
OUTPUTAPICOST = .03 OUTPUTAPICOST = .03
BATCHSIZE = 10 BATCHSIZE = 5
def handleAlice(filename, estimate): def handleAlice(filename, estimate):
global ESTIMATE global ESTIMATE
@ -89,6 +89,7 @@ def handleAlice(filename, estimate):
totalTokens[0] += translatedData[1][0] totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1] totalTokens[1] += translatedData[1][1]
except Exception as e: except Exception as e:
traceback.print_exc()
return 'Fail' return 'Fail'
return getResultString(['', totalTokens, None], end - start, 'TOTAL') return getResultString(['', totalTokens, None], end - start, 'TOTAL')
@ -178,19 +179,22 @@ def translateLines(linesList, pbar):
# Grab rest of the messages # Grab rest of the messages
currentGroup.append(jaString) currentGroup.append(jaString)
# Check if next line should be merged
if insertBool is True: if insertBool is True:
linesList[i] = re.sub(r'(m\[[0-9]+\]) = \"(.+)\"', rf'\1 = ""', linesList[i]) linesList[i] = re.sub(r'(m\[[0-9]+\]) = \"(.+)\"', rf'\1 = ""', linesList[i])
linesList[i] = linesList[i].replace(';', '') linesList[i] = linesList[i].replace(';', '')
start = i start = i
while (len(linesList) > i+1 and re.search(r'm\[[0-9]+\] = \"\s(.*)\"', linesList[i+1]) != None): while (len(linesList) > i+1 and re.search(r'm\[[0-9]+\] = \"\s+(.*)\"', linesList[i+1]) != None):
multiLine = True multiLine = True
i += 1 i += 1
match = re.findall(r'm\[[0-9]+\] = \"\s(.*)\"', linesList[i]) match = re.findall(r'm\[[0-9]+\] = \"\s+(.*)\"', linesList[i])
currentGroup.append(match[0]) currentGroup.append(match[0])
if insertBool is True: if insertBool is True:
pbar.update(1) pbar.update(1)
linesList[i] = re.sub(r'(m\[[0-9]+\]) = \"\s(.+)\"', rf'\1 = ""', linesList[i]) linesList[i] = re.sub(r'(m\[[0-9]+\]) = \"\s+(.+)\"', rf'\1 = ""', linesList[i])
linesList[i] = linesList[i].replace(';', '') linesList[i] = linesList[i].replace(';', '')
i += 1
# Combine Groups and Add Speaker # Combine Groups and Add Speaker
finalJAString = ' '.join(currentGroup) finalJAString = ' '.join(currentGroup)
@ -222,11 +226,8 @@ def translateLines(linesList, pbar):
else: else:
pbar.write(f'Mismatch: {batchStartIndex} - {i}') pbar.write(f'Mismatch: {batchStartIndex} - {i}')
MISMATCH.append(batch) MISMATCH.append(batch)
i += 1
batchStartIndex = i batchStartIndex = i
batch.clear() batch.clear()
else:
i += 1
multiLine = False multiLine = False
currentGroup = [] currentGroup = []
@ -243,6 +244,9 @@ def translateLines(linesList, pbar):
# Textwrap # Textwrap
translatedText = translatedText.replace('\"', '\\"') translatedText = translatedText.replace('\"', '\\"')
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
if 'the village women...' in translatedText:
print('t')
# Write (Skip First) # Write (Skip First)
if firstRun is True: if firstRun is True:
@ -257,7 +261,6 @@ def translateLines(linesList, pbar):
translatedText = translatedText.replace(';', '') translatedText = translatedText.replace(';', '')
linesList[start] = translatedText linesList[start] = translatedText
start+=1 start+=1
i += 1
multiLine = False multiLine = False
translatedText = translatedText.replace(';', '') translatedText = translatedText.replace(';', '')
translatedBatch.pop(0) translatedBatch.pop(0)
@ -267,7 +270,6 @@ def translateLines(linesList, pbar):
translatedText = re.sub(r'(m\[[0-9]+\]) = \"(.*)\"', rf'\1 = "{translatedText}"', linesList[i]) translatedText = re.sub(r'(m\[[0-9]+\]) = \"(.*)\"', rf'\1 = "{translatedText}"', linesList[i])
translatedText = translatedText.replace(';', '') translatedText = translatedText.replace(';', '')
linesList[i] = translatedText linesList[i] = translatedText
i += 1
translatedBatch.pop(0) translatedBatch.pop(0)
# If Batch is empty. Move on. # If Batch is empty. Move on.
@ -283,6 +285,7 @@ def translateLines(linesList, pbar):
return [linesList, tokens] return [linesList, tokens]
except Exception: except Exception:
traceback.print_exc()
return [linesList, tokens] return [linesList, tokens]
def subVars(jaString): def subVars(jaString):
@ -431,24 +434,23 @@ def translateText(characters, system, user, history):
msg = [{"role": "system", "content": system}] msg = [{"role": "system", "content": system}]
# Characters # Characters
msg.append({"role": "user", "content": characters}) msg.append({"role": "system", "content": characters})
# History # History
if isinstance(history, list): if isinstance(history, list):
msg.extend([{"role": "user", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "user", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": user}) msg.append({"role": "user", "content": f'{user}'})
response = openai.ChatCompletion.create( response = openai.chat.completions.create(
temperature=0.1, temperature=0.1,
top_p = 0.2, top_p = 0.2,
frequency_penalty=0, frequency_penalty=0,
presence_penalty=0, presence_penalty=0,
model=MODEL, model=MODEL,
messages=msg, messages=msg,
request_timeout=TIMEOUT,
) )
return response return response

4
requirements.txt
View file

@ -1,7 +1,7 @@
colorama==0.4.6 colorama==0.4.6
openai==0.27.4 openai==1.3.8
python-dotenv==1.0.0 python-dotenv==1.0.0
retry==0.9.2 retry==0.9.2
ruamel.yaml==0.17.32 ruamel.yaml==0.17.32
tiktoken==0.3.3 tiktoken==0.5.2
tqdm==4.65.0 tqdm==4.65.0