Adjust estimate to account for mismatches
This commit is contained in:
parent
99ace95d78
commit
6083b25165
13 changed files with 26 additions and 25 deletions
|
|
@ -497,7 +497,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -538,7 +538,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -415,7 +415,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -501,7 +501,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -594,7 +594,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -481,7 +481,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -600,7 +600,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -2254,7 +2254,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ LEAVE = False
|
|||
|
||||
# Dialogue / Scroll
|
||||
CODE401 = True
|
||||
CODE405 = False
|
||||
CODE405 = True
|
||||
CODE408 = False
|
||||
|
||||
# Choices
|
||||
|
|
@ -67,7 +67,7 @@ CODE102 = True
|
|||
CODE122 = False
|
||||
|
||||
# Names
|
||||
CODE101 = False
|
||||
CODE101 = True
|
||||
|
||||
# Other
|
||||
CODE355655 = False
|
||||
|
|
@ -843,10 +843,10 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
### \\n<Speaker>
|
||||
nCase = None
|
||||
if finalJAString[0] != '\\':
|
||||
regex = r'(.*?)([\\]+[kKnN][wWcC]?[<\[](.*?)[>\]].*)'
|
||||
regex = r'(.*?)([\\]+[kKnN][wWcC]?[<](.*?)[>].*)'
|
||||
nCase = 0
|
||||
else:
|
||||
regex = r'(.*[\\]+[kKnN][wWcC]?[<\[](.*?)[\]>])(.*)'
|
||||
regex = r'(.*[\\]+[kKnN][wWcC]?[<](.*?)[>])(.*)'
|
||||
nCase = 1
|
||||
matchList = re.findall(regex, finalJAString)
|
||||
if len(matchList) > 0:
|
||||
|
|
@ -971,7 +971,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if speaker == '' and finalJAString != '':
|
||||
docList.append(finalJAString)
|
||||
elif finalJAString != '':
|
||||
docList.append(f'{speaker}: {finalJAString}')
|
||||
docList.append(f'[{speaker}]: {finalJAString}')
|
||||
else:
|
||||
docList.append(speaker)
|
||||
speaker = ''
|
||||
|
|
@ -1946,11 +1946,12 @@ def batchList(input_list, batch_size):
|
|||
|
||||
def createContext(fullPromptFlag, subbedT):
|
||||
characters = 'Game Characters:\n\
|
||||
リゼ (Rize) - Female\n\
|
||||
ティア (Tia) - Female\n\
|
||||
リィナ (Riina) - Female\n\
|
||||
クェス (Ques) - Female\n\
|
||||
たこ (Tako) - Female\n\
|
||||
皆月 (Minazuki)\n\
|
||||
さやか (Sayaka)\n\
|
||||
皆月 さやか (Minazuki Sayaka) - Female\n\
|
||||
広瀬 (Hirose)\n\
|
||||
智恵 (Chie) - Female\n\
|
||||
広瀬 智恵 (Hirose Chie) - Femal\n\
|
||||
'
|
||||
|
||||
system = PROMPT + VOCAB if fullPromptFlag else \
|
||||
|
|
@ -2051,7 +2052,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
@ -2095,7 +2096,7 @@ def translateGPT(text, history, fullPromptFlag):
|
|||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(characters, system, user, history, 0)
|
||||
response = translateText(characters, system, user, history, 0.02)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
|
@ -2107,7 +2108,7 @@ def translateGPT(text, history, fullPromptFlag):
|
|||
tList[index] = extractedTranslations
|
||||
if len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(characters, system, user, history, 0.05)
|
||||
response = translateText(characters, system, user, history, 0.1)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
|
|
|||
|
|
@ -593,7 +593,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -1215,7 +1215,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -499,7 +499,7 @@ def countTokens(characters, system, user, history):
|
|||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user))/1.5)
|
||||
outputTotalTokens += round(len(enc.encode(user))*3)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
|
|
|||
|
|
@ -40,5 +40,5 @@ ME 音量 (ME Volume)
|
|||
回想する (Recollection)
|
||||
|
||||
# Other
|
||||
ジュピターノ (Jupiterino)
|
||||
パパ (papa)
|
||||
```
|
||||
Loading…
Reference in a new issue