Only keep recent 10 log files and only write if actual text translated
This commit is contained in:
parent
ccb7384360
commit
45685f6da1
4 changed files with 229 additions and 199 deletions
|
|
@ -138,11 +138,9 @@ class LogViewer(QWidget):
|
|||
if interval_ms:
|
||||
self._tail_interval = interval_ms
|
||||
|
||||
# Ensure directory exists
|
||||
# Ensure directory exists but don't create the file
|
||||
try:
|
||||
self._tail_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Ensure file exists
|
||||
self._tail_path.touch(exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
@ -154,14 +152,17 @@ class LogViewer(QWidget):
|
|||
pass
|
||||
self._tail_f = None
|
||||
|
||||
try:
|
||||
self._tail_f = open(self._tail_path, 'r', encoding='utf-8', errors='ignore')
|
||||
# Seek to end so we only read new lines
|
||||
self._tail_f.seek(0, os.SEEK_END)
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"Log error: {str(e)}")
|
||||
self._tail_f = None
|
||||
return
|
||||
# Only open the file if it exists
|
||||
# If it doesn't exist yet, the timer will keep checking
|
||||
if self._tail_path.exists():
|
||||
try:
|
||||
self._tail_f = open(self._tail_path, 'r', encoding='utf-8', errors='ignore')
|
||||
# Seek to end so we only read new lines
|
||||
self._tail_f.seek(0, os.SEEK_END)
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"Log error: {str(e)}")
|
||||
self._tail_f = None
|
||||
return
|
||||
|
||||
self._tail_timer.start(self._tail_interval)
|
||||
|
||||
|
|
@ -180,6 +181,15 @@ class LogViewer(QWidget):
|
|||
|
||||
def _poll_tail(self):
|
||||
"""Timer callback: read any new data from the tailed file and append it."""
|
||||
# If file handle doesn't exist yet, try to open it if the file was created
|
||||
if not self._tail_f and self._tail_path.exists():
|
||||
try:
|
||||
self._tail_f = open(self._tail_path, 'r', encoding='utf-8', errors='ignore')
|
||||
# Seek to end so we only read new lines
|
||||
self._tail_f.seek(0, os.SEEK_END)
|
||||
except Exception:
|
||||
pass # Will try again next poll
|
||||
|
||||
if not self._tail_f:
|
||||
return
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1675,12 +1675,24 @@ class TranslationTab(QWidget):
|
|||
try:
|
||||
history_dir = self.project_root / 'log' / 'history'
|
||||
history_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Clean up old log files, keeping only the 10 most recent
|
||||
try:
|
||||
log_files = sorted(history_dir.glob("translationHistory_*.txt"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
# Keep only the 10 most recent, delete the rest
|
||||
for old_log in log_files[10:]:
|
||||
try:
|
||||
old_log.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Use timestamp (safe filename) for sorting
|
||||
fname = datetime.datetime.now().strftime('translationHistory_%Y%m%d_%H%M%S.txt')
|
||||
run_log_path = history_dir / fname
|
||||
# Create/touch the file so tailer can open and seek to end
|
||||
run_log_path.touch(exist_ok=True)
|
||||
|
||||
# Don't create the file yet - it will be created when first log is written
|
||||
|
||||
# Export env var so subprocess workers inherit the path
|
||||
try:
|
||||
os.environ['TRANSLATION_RUN_LOG'] = str(run_log_path)
|
||||
|
|
@ -1689,25 +1701,19 @@ class TranslationTab(QWidget):
|
|||
|
||||
# Try to create a hard link at legacy location so modules that
|
||||
# still write to log/translationHistory.txt end up in this file.
|
||||
# This will be created when the run_log_path file is first written to
|
||||
legacy = self.project_root / 'log' / 'translationHistory.txt'
|
||||
try:
|
||||
# Remove any existing legacy file and create hard link
|
||||
# Remove any existing legacy file
|
||||
if legacy.exists():
|
||||
try:
|
||||
legacy.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
os.link(str(run_log_path), str(legacy))
|
||||
except Exception:
|
||||
# If hard link fails (Windows permissions or cross-device),
|
||||
# fallback to ensuring the legacy file exists but do not fail.
|
||||
try:
|
||||
legacy.parent.mkdir(parents=True, exist_ok=True)
|
||||
legacy.touch(exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
pass
|
||||
|
||||
# Clear UI log and start tailing the per-run file
|
||||
# Clear UI log and start tailing the per-run file (tailer will handle non-existent files)
|
||||
self.translation_log_viewer.clear_log()
|
||||
self.translation_log_viewer.start_tail(run_log_path)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -143,16 +143,30 @@ files to translate are in the /files folder and that you picked the right game e
|
|||
try:
|
||||
hist_dir = Path("log") / "history"
|
||||
hist_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Clean up old log files, keeping only the 10 most recent
|
||||
try:
|
||||
log_files = sorted(hist_dir.glob("translationHistory_*.txt"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
# Keep only the 10 most recent, delete the rest
|
||||
for old_log in log_files[10:]:
|
||||
try:
|
||||
old_log.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
fname = datetime.datetime.now().strftime("translationHistory_%Y%m%d_%H%M%S.txt")
|
||||
run_log_path = hist_dir / fname
|
||||
run_log_path.touch(exist_ok=True)
|
||||
# Export env var so other modules/util functions pick it up
|
||||
# Don't create the file yet - it will be created when first log is written
|
||||
# Store the path in environment variable
|
||||
try:
|
||||
os.environ['TRANSLATION_RUN_LOG'] = str(run_log_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Try to create a hard link from legacy path to this run file for compatibility
|
||||
# This will be created when the run_log_path file is first written to
|
||||
legacy = Path("log") / "translationHistory.txt"
|
||||
try:
|
||||
if legacy.exists():
|
||||
|
|
@ -160,14 +174,8 @@ files to translate are in the /files folder and that you picked the right game e
|
|||
legacy.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
os.link(str(run_log_path), str(legacy))
|
||||
except Exception:
|
||||
# Fallback: ensure legacy file exists so modules writing to it won't fail
|
||||
try:
|
||||
legacy.parent.mkdir(parents=True, exist_ok=True)
|
||||
legacy.touch(exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
with ThreadPoolExecutor(max_workers=THREADS) as executor:
|
||||
|
|
|
|||
|
|
@ -728,176 +728,182 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
with open(config.logFilePath, "a+", encoding="utf-8") as logFile:
|
||||
totalTokens = [0, 0]
|
||||
|
||||
if isinstance(text, list):
|
||||
formatType = "json"
|
||||
tList = batchList(text, config.batchSize)
|
||||
else:
|
||||
formatType = "text"
|
||||
tList = [text]
|
||||
# Don't open log file here - we'll open it only when we need to write
|
||||
totalTokens = [0, 0]
|
||||
|
||||
if isinstance(text, list):
|
||||
formatType = "json"
|
||||
tList = batchList(text, config.batchSize)
|
||||
else:
|
||||
formatType = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Check if text contains target language
|
||||
if not re.search(config.langRegex, str(tItem)):
|
||||
if pbar is not None:
|
||||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j], config.language)
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem, config.language)
|
||||
history = tItem[-config.maxHistory:] if isinstance(tItem, list) else tItem
|
||||
continue
|
||||
|
||||
# Format for translation
|
||||
for index, tItem in enumerate(tList):
|
||||
# Check if text contains target language
|
||||
if not re.search(config.langRegex, str(tItem)):
|
||||
if pbar is not None:
|
||||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j] or not str(tItem[j]).strip():
|
||||
tItem[j] = "Placeholder Text"
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
subbedT = payload
|
||||
else:
|
||||
# Check for empty/whitespace strings in non-list items
|
||||
if not tItem or not str(tItem).strip():
|
||||
subbedT = "Placeholder Text"
|
||||
else:
|
||||
subbedT = tItem
|
||||
|
||||
# Check cache for this exact payload
|
||||
cached_result = get_cached_translation(subbedT, config.language)
|
||||
if cached_result is not None:
|
||||
if isinstance(tItem, list):
|
||||
tList[index] = cached_result
|
||||
history = cached_result[-config.maxHistory:]
|
||||
else:
|
||||
tList[index] = cached_result
|
||||
history = cached_result
|
||||
|
||||
if lock and pbar is not None:
|
||||
with lock:
|
||||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
|
||||
continue
|
||||
|
||||
# Create context
|
||||
system, user = createContext(config, fullPromptFlag, subbedT, formatType, history)
|
||||
|
||||
# Calculate estimate if in estimate mode
|
||||
if config.estimateMode:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
|
||||
# Cache the payload with original text as placeholder for future estimates
|
||||
if isinstance(tItem, list):
|
||||
cache_translation(subbedT, tItem, config.language)
|
||||
else:
|
||||
cache_translation(subbedT, [tItem], config.language)
|
||||
|
||||
continue
|
||||
|
||||
# --- Translation and Validation Retry Block ---
|
||||
max_retries = 2 # 1 initial attempt + 2 retries
|
||||
final_translations = None
|
||||
last_raw_translation = ""
|
||||
numLines = len(tItem) if isinstance(tItem, list) else None
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
is_valid = True
|
||||
|
||||
# On retries, add a note to the system prompt
|
||||
current_system = system
|
||||
if attempt > 0:
|
||||
current_system += f"\n\nIMPORTANT: Your previous attempt was incorrect or incomplete. Please ensure the entire output is translated to {config.language} and contains no untranslated characters. Translate the following text again, ensuring the JSON structure is correct."
|
||||
if pbar:
|
||||
pbar.write(f"Retrying translation... (Attempt {attempt + 1}/{max_retries + 1})")
|
||||
|
||||
# Translate
|
||||
response = translateText(current_system, user, history, 0.05, formatType, config.model, numLines)
|
||||
translatedText = response.choices[0].message.content
|
||||
last_raw_translation = translatedText
|
||||
|
||||
# Update token count for this attempt
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Clean the translation first for consistency
|
||||
cleaned_text = cleanTranslatedText(translatedText, config.language)
|
||||
|
||||
# Process and validate translation result
|
||||
if cleaned_text:
|
||||
if isinstance(tItem, list):
|
||||
extracted = extractTranslation(cleaned_text, True, pbar)
|
||||
|
||||
# Check 1: Mismatch in length -> still a hard failure
|
||||
if extracted is None or len(tItem) != len(extracted):
|
||||
is_valid = False
|
||||
else:
|
||||
# Set translations (line count matches)
|
||||
final_translations = extracted
|
||||
else:
|
||||
# Single string: accept output even if it contains characters
|
||||
# matching langRegex (allow names or untranslated tokens).
|
||||
final_translations = cleaned_text.replace("Placeholder Text", "")
|
||||
else:
|
||||
is_valid = False
|
||||
if pbar: pbar.write(f"AI Refused: {tItem}\n")
|
||||
|
||||
# If translation is valid, break the retry loop
|
||||
if is_valid:
|
||||
break
|
||||
|
||||
# --- End of Retry Block ---
|
||||
|
||||
# After the loop, handle the final result
|
||||
if final_translations is not None: # Success case
|
||||
formatted_output = last_raw_translation
|
||||
try:
|
||||
parsed_json = json.loads(last_raw_translation)
|
||||
formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{formatted_output}\n")
|
||||
|
||||
# Cache the entire payload and its translation
|
||||
if not config.estimateMode:
|
||||
cache_translation(subbedT, final_translations, config.language)
|
||||
|
||||
if isinstance(tItem, list):
|
||||
tList[index] = final_translations
|
||||
history = final_translations[-config.maxHistory:]
|
||||
else:
|
||||
tList[index] = final_translations
|
||||
history = final_translations
|
||||
|
||||
if lock and pbar is not None:
|
||||
with lock:
|
||||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
|
||||
else: # Failure case after all retries
|
||||
if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.")
|
||||
|
||||
formatted_mismatch_output = last_raw_translation
|
||||
try:
|
||||
parsed_json = json.loads(last_raw_translation)
|
||||
formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Failed after retries: {filename}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Final Output:\n{formatted_mismatch_output}\n")
|
||||
|
||||
if filename and mismatchList is not None and filename not in mismatchList:
|
||||
mismatchList.append(filename)
|
||||
|
||||
tItem[j] = cleanTranslatedText(tItem[j], config.language)
|
||||
tList[index] = tItem
|
||||
history = text[-config.maxHistory:] if isinstance(text, list) else text
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem, config.language)
|
||||
history = tItem[-config.maxHistory:] if isinstance(tItem, list) else tItem
|
||||
continue
|
||||
|
||||
# Format for translation
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j] or not str(tItem[j]).strip():
|
||||
tItem[j] = "Placeholder Text"
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
subbedT = payload
|
||||
else:
|
||||
# Check for empty/whitespace strings in non-list items
|
||||
if not tItem or not str(tItem).strip():
|
||||
subbedT = "Placeholder Text"
|
||||
else:
|
||||
subbedT = tItem
|
||||
|
||||
# Check cache for this exact payload
|
||||
cached_result = get_cached_translation(subbedT, config.language)
|
||||
if cached_result is not None:
|
||||
if isinstance(tItem, list):
|
||||
tList[index] = cached_result
|
||||
history = cached_result[-config.maxHistory:]
|
||||
else:
|
||||
tList[index] = cached_result
|
||||
history = cached_result
|
||||
|
||||
if lock and pbar is not None:
|
||||
with lock:
|
||||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
|
||||
continue
|
||||
|
||||
# Create context
|
||||
system, user = createContext(config, fullPromptFlag, subbedT, formatType, history)
|
||||
|
||||
# Calculate estimate if in estimate mode
|
||||
if config.estimateMode:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
|
||||
# Cache the payload with original text as placeholder for future estimates
|
||||
if isinstance(tItem, list):
|
||||
cache_translation(subbedT, tItem, config.language)
|
||||
else:
|
||||
cache_translation(subbedT, [tItem], config.language)
|
||||
|
||||
continue
|
||||
|
||||
# --- Translation and Validation Retry Block ---
|
||||
max_retries = 2 # 1 initial attempt + 2 retries
|
||||
final_translations = None
|
||||
last_raw_translation = ""
|
||||
numLines = len(tItem) if isinstance(tItem, list) else None
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
is_valid = True
|
||||
|
||||
# On retries, add a note to the system prompt
|
||||
current_system = system
|
||||
if attempt > 0:
|
||||
current_system += f"\n\nIMPORTANT: Your previous attempt was incorrect or incomplete. Please ensure the entire output is translated to {config.language} and contains no untranslated characters. Translate the following text again, ensuring the JSON structure is correct."
|
||||
if pbar:
|
||||
pbar.write(f"Retrying translation... (Attempt {attempt + 1}/{max_retries + 1})")
|
||||
|
||||
# Translate
|
||||
response = translateText(current_system, user, history, 0.05, formatType, config.model, numLines)
|
||||
translatedText = response.choices[0].message.content
|
||||
last_raw_translation = translatedText
|
||||
|
||||
# Update token count for this attempt
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Clean the translation first for consistency
|
||||
cleaned_text = cleanTranslatedText(translatedText, config.language)
|
||||
|
||||
# Process and validate translation result
|
||||
if cleaned_text:
|
||||
if isinstance(tItem, list):
|
||||
extracted = extractTranslation(cleaned_text, True, pbar)
|
||||
|
||||
# Check 1: Mismatch in length -> still a hard failure
|
||||
if extracted is None or len(tItem) != len(extracted):
|
||||
is_valid = False
|
||||
else:
|
||||
# Set translations (line count matches)
|
||||
final_translations = extracted
|
||||
else:
|
||||
# Single string: accept output even if it contains characters
|
||||
# matching langRegex (allow names or untranslated tokens).
|
||||
final_translations = cleaned_text.replace("Placeholder Text", "")
|
||||
else:
|
||||
is_valid = False
|
||||
if pbar: pbar.write(f"AI Refused: {tItem}\n")
|
||||
|
||||
# If translation is valid, break the retry loop
|
||||
if is_valid:
|
||||
break
|
||||
|
||||
# --- End of Retry Block ---
|
||||
|
||||
# After the loop, handle the final result
|
||||
if final_translations is not None: # Success case
|
||||
formatted_output = last_raw_translation
|
||||
try:
|
||||
parsed_json = json.loads(last_raw_translation)
|
||||
formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
|
||||
# Only open and write to log file when we have something to log
|
||||
try:
|
||||
with open(config.logFilePath, "a", encoding="utf-8") as logFile:
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{formatted_output}\n")
|
||||
except Exception:
|
||||
pass # Don't fail if logging fails
|
||||
|
||||
# Cache the entire payload and its translation
|
||||
if not config.estimateMode:
|
||||
cache_translation(subbedT, final_translations, config.language)
|
||||
|
||||
if isinstance(tItem, list):
|
||||
tList[index] = final_translations
|
||||
history = final_translations[-config.maxHistory:]
|
||||
else:
|
||||
tList[index] = final_translations
|
||||
history = final_translations
|
||||
|
||||
if lock and pbar is not None:
|
||||
with lock:
|
||||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
|
||||
else: # Failure case after all retries
|
||||
if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.")
|
||||
|
||||
formatted_mismatch_output = last_raw_translation
|
||||
try:
|
||||
parsed_json = json.loads(last_raw_translation)
|
||||
formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Failed after retries: {filename}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Final Output:\n{formatted_mismatch_output}\n")
|
||||
|
||||
if filename and mismatchList is not None and filename not in mismatchList:
|
||||
mismatchList.append(filename)
|
||||
|
||||
tList[index] = tItem
|
||||
history = text[-config.maxHistory:] if isinstance(text, list) else text
|
||||
|
||||
# Combine if multilist
|
||||
if tList and isinstance(tList[0], list):
|
||||
|
|
|
|||
Loading…
Reference in a new issue