Some fixes
This commit is contained in:
parent
977dd1bb5c
commit
77efab8dc2
3 changed files with 244 additions and 77 deletions
|
|
@ -30,6 +30,12 @@ from PyQt5.QtGui import QFont
|
|||
from gui.log_viewer import LogViewer
|
||||
|
||||
|
||||
def _strip_ansi(text):
|
||||
if not isinstance(text, str) or not text:
|
||||
return text
|
||||
return re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text)
|
||||
|
||||
|
||||
def create_section_header(title):
|
||||
"""Create a clean section header without boxes."""
|
||||
label = QLabel(title)
|
||||
|
|
@ -1697,7 +1703,26 @@ class TranslationTab(QWidget):
|
|||
item['label'].setText(f"{current}/{total}")
|
||||
item['label'].setStyleSheet("color: #007acc; font-weight: bold;")
|
||||
|
||||
def mark_file_complete(self, filename, success=True, error_message=None):
|
||||
def _apply_success_status_icon(self, item, completion_kind="normal"):
|
||||
"""Green checkmark for every successful row; tooltips explain skip / idle when relevant."""
|
||||
try:
|
||||
item["status_label"].setText("✓")
|
||||
item["status_label"].setStyleSheet(
|
||||
"color: #4ec9b0; font-weight: bold; font-size: 11px;"
|
||||
)
|
||||
if completion_kind == "skip":
|
||||
reason = (item.get("_skip_reason") or "").strip()
|
||||
tip = f"Skipped: {reason}" if reason else "Whole file skipped (paths/fonts only)."
|
||||
elif completion_kind == "idle":
|
||||
tip = "No translatable lines (non-dialogue content only)."
|
||||
else:
|
||||
tip = ""
|
||||
item["status_label"].setToolTip(tip)
|
||||
item["status_label"].setVisible(True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def mark_file_complete(self, filename, success=True, error_message=None, completion_kind="normal"):
|
||||
"""Mark a file as complete or failed."""
|
||||
if filename in self.file_progress_items:
|
||||
item = self.file_progress_items[filename]
|
||||
|
|
@ -1709,13 +1734,7 @@ class TranslationTab(QWidget):
|
|||
item['tokens_label'].setVisible(True)
|
||||
item['cost_label'].setVisible(True)
|
||||
item['time_label'].setVisible(True)
|
||||
# Show compact status in the middle column
|
||||
try:
|
||||
item['status_label'].setText("✓")
|
||||
item['status_label'].setStyleSheet("color: #4ec9b0; font-weight: bold; font-size: 11px;")
|
||||
item['status_label'].setVisible(True)
|
||||
except Exception:
|
||||
pass
|
||||
self._apply_success_status_icon(item, completion_kind)
|
||||
try:
|
||||
item['progress_bar'].setVisible(False)
|
||||
except Exception:
|
||||
|
|
@ -1728,13 +1747,11 @@ class TranslationTab(QWidget):
|
|||
pass
|
||||
else:
|
||||
# No parsed results available - show status in status_label
|
||||
self._apply_success_status_icon(item, completion_kind)
|
||||
try:
|
||||
item['status_label'].setText("✓")
|
||||
item['status_label'].setStyleSheet("color: #4ec9b0; font-weight: bold; font-size: 11px;")
|
||||
item['status_label'].setVisible(True)
|
||||
item['progress_bar'].setVisible(False)
|
||||
except Exception:
|
||||
pass
|
||||
item['progress_bar'].setValue(item['progress_bar'].maximum())
|
||||
try:
|
||||
item['label'].setText("")
|
||||
item['label'].setStyleSheet("color: #888888; font-size: 11px;")
|
||||
|
|
@ -2007,32 +2024,67 @@ class TranslationTab(QWidget):
|
|||
except Exception:
|
||||
pass
|
||||
try:
|
||||
pattern = r'^\W*(?P<filename>[^:]+):.*?\[Input:\s*(?P<input>\d+)\].*?\[Output:\s*(?P<output>\d+)\].*?\[Cost:\s*\$(?P<cost>[\d\.]+)\].*?\[(?P<time>[\d\.]+)s\]'
|
||||
m = re.search(pattern, message)
|
||||
stripped = _strip_ansi(message)
|
||||
pattern = (
|
||||
r'^\s*(?P<filename>[^:]+):.*?\[Input:\s*(?P<input>\d+)\].*?\[Output:\s*(?P<output>\d+)\]'
|
||||
r'.*?\[Cost:\s*\$(?P<cost>[\d,\.]+)\].*?\[(?P<time>[\d\.]+)s\]'
|
||||
)
|
||||
m = re.search(pattern, stripped)
|
||||
if not m:
|
||||
return
|
||||
filename = re.sub(r'^[^\w]+', '', m.group('filename')).strip()
|
||||
if filename.lower() == 'total':
|
||||
filename = m.group("filename").strip()
|
||||
if filename.lower() == "total":
|
||||
return
|
||||
input_tokens = int(m.group('input'))
|
||||
output_tokens = int(m.group('output'))
|
||||
cost = float(m.group('cost'))
|
||||
time_s = float(m.group('time'))
|
||||
self._apply_file_result(filename, input_tokens, output_tokens, cost, time_s)
|
||||
input_tokens = int(m.group("input"))
|
||||
output_tokens = int(m.group("output"))
|
||||
cost = float(m.group("cost").replace(",", ""))
|
||||
time_s = float(m.group("time"))
|
||||
m_skip = re.search(r"\[skipped\]\s*(.*)$", stripped)
|
||||
skip_reason = (m_skip.group(1) or "").strip() if m_skip else ""
|
||||
if skip_reason:
|
||||
completion_kind = "skip"
|
||||
elif input_tokens == 0 and output_tokens == 0:
|
||||
completion_kind = "idle"
|
||||
else:
|
||||
completion_kind = "normal"
|
||||
self._apply_file_result(
|
||||
filename,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cost,
|
||||
time_s,
|
||||
completion_kind=completion_kind,
|
||||
skip_reason=skip_reason,
|
||||
)
|
||||
except Exception:
|
||||
# Ignore parse/logging errors
|
||||
pass
|
||||
# Do not forward this message into the LogViewer (we tail the log file separately).
|
||||
return
|
||||
|
||||
def _apply_file_result(self, filename, input_tokens, output_tokens, cost, time_s):
|
||||
def _apply_file_result(
|
||||
self,
|
||||
filename,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cost,
|
||||
time_s,
|
||||
completion_kind="normal",
|
||||
skip_reason="",
|
||||
):
|
||||
"""Update a file's item UI with parsed result details and update totals."""
|
||||
# Update per-item display
|
||||
if filename in self.file_progress_items:
|
||||
item = self.file_progress_items[filename]
|
||||
try:
|
||||
# Compact token display: input/output
|
||||
item['tokens_label'].setText(f"{input_tokens}/{output_tokens}")
|
||||
item.pop("_skip_reason", None)
|
||||
if skip_reason:
|
||||
item["_skip_reason"] = skip_reason
|
||||
# Distinguish "no API usage" from real token counts in the list UI
|
||||
if completion_kind in ("skip", "idle"):
|
||||
item["tokens_label"].setText("—")
|
||||
else:
|
||||
item["tokens_label"].setText(f"{input_tokens}/{output_tokens}")
|
||||
item['cost_label'].setText(f"${cost:.4f}")
|
||||
item['time_label'].setText(f"{time_s:.1f}s")
|
||||
item['tokens_label'].setVisible(True)
|
||||
|
|
@ -2066,7 +2118,7 @@ class TranslationTab(QWidget):
|
|||
pass
|
||||
# Mark file as complete (this will ensure checkbox and label updated)
|
||||
try:
|
||||
self.mark_file_complete(filename, success=True)
|
||||
self.mark_file_complete(filename, success=True, completion_kind=completion_kind)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
@ -2080,8 +2132,12 @@ class TranslationTab(QWidget):
|
|||
self.files_completed = current_file
|
||||
self.files_translated_label.setText(f"{current_file}/{total_files}")
|
||||
|
||||
# Mark this filename as complete right away
|
||||
self.mark_file_complete(filename, success=True)
|
||||
# Row details (tokens/cost) come from parsed stdout via append_log. If that
|
||||
# did not run (older modules / parse miss), finalize so the row is not stuck.
|
||||
if filename in self.file_progress_items:
|
||||
sl = self.file_progress_items[filename].get("status_label")
|
||||
if not sl or not sl.text():
|
||||
self.mark_file_complete(filename, success=True)
|
||||
|
||||
# Clear current_translating_file if it was the same file
|
||||
if self.current_translating_file == filename:
|
||||
|
|
|
|||
185
modules/yuris.py
185
modules/yuris.py
|
|
@ -52,12 +52,113 @@ TRANSLATION_CONFIG = TranslationConfig(
|
|||
estimateMode=False,
|
||||
)
|
||||
|
||||
_ASSET_EXT = re.compile(
|
||||
r"\.(png|jpe?g|gif|webp|bmp|tga|tif|ogg|mp3|wav|mid|json|txt|xml|ttf|otf|woff2?|ks|tjs)\b",
|
||||
re.I,
|
||||
)
|
||||
# Typeface / UI font name tokens — translating breaks runtime font resolution.
|
||||
_FONTISH = re.compile(
|
||||
r"ゴシック|ゴチック|明朝|Mincho|Gothic|ＰＧ|ＭＳ|メイリオ|Meiryo|游ゴ|Yu\s?Gothic|"
|
||||
r"Tahoma|Arial|Consolas|Segoe|SimSun|宋体|黑体|DF体|ヒラギノ|Hiragino|"
|
||||
r"Noto\s|IPA|Ricty|VL\s|@",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def _yuris_nonempty_messages(data: list) -> list[str]:
|
||||
out: list[str] = []
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
msg = item.get("message")
|
||||
if isinstance(msg, str) and msg.strip():
|
||||
out.append(msg)
|
||||
return out
|
||||
|
||||
|
||||
def _yuris_font_face_table(msgs: list[str]) -> bool:
|
||||
if not msgs or len(msgs) > 120:
|
||||
return False
|
||||
if any(len(m) > 120 for m in msgs):
|
||||
return False
|
||||
ascii_label = re.compile(r"^[A-Za-z0-9 _+.,'-]{1,64}$")
|
||||
for m in msgs:
|
||||
if _FONTISH.search(m):
|
||||
continue
|
||||
if ascii_label.fullmatch(m.strip()):
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def yuris_whole_file_pass_through_reason(data) -> str | None:
|
||||
"""
|
||||
When True, the file is copied to translated/ unchanged (no API).
|
||||
Heuristics: only resource paths, or only font-face style entries.
|
||||
"""
|
||||
if not isinstance(data, list):
|
||||
return None
|
||||
msgs = _yuris_nonempty_messages(data)
|
||||
if not msgs:
|
||||
return None
|
||||
if all(_message_looks_like_asset_path(m) for m in msgs):
|
||||
return "path-only resource strings"
|
||||
if _yuris_font_face_table(msgs):
|
||||
return "font face list"
|
||||
return None
|
||||
|
||||
|
||||
def _message_looks_like_asset_path(message: str) -> bool:
|
||||
"""Skip resource-like strings: paths, URLs, and any line containing '_' (engine IDs / filenames)."""
|
||||
s = message.strip()
|
||||
if not s:
|
||||
return True
|
||||
if "://" in s:
|
||||
return True
|
||||
if re.match(r"^[A-Za-z]:[/\\]", s):
|
||||
return True
|
||||
if "\\" in s or "/" in s:
|
||||
norm = s.replace("\\", "/")
|
||||
tail = norm.rsplit("/", 1)[-1]
|
||||
if _ASSET_EXT.search(s):
|
||||
return True
|
||||
if re.fullmatch(r"[A-Za-z0-9_.-]+", tail):
|
||||
return True
|
||||
if "_" in s:
|
||||
return True
|
||||
# Common JP UI font face names (no underscores) — keep short to avoid snagging real prose.
|
||||
if len(s) <= 80 and any(t in s for t in ("ゴシック", "ゴチック", "明朝")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def yuris_message_is_translatable(message: str) -> bool:
|
||||
return isinstance(message, str) and bool(message.strip()) and not _message_looks_like_asset_path(message)
|
||||
|
||||
|
||||
def replace_yuris_problem_dashes(text: str) -> str:
|
||||
"""Replace characters the Yuris runtime mishandles (e.g. ― U+2015 HORIZONTAL BAR)."""
|
||||
return text.replace("\u2015", "-")
|
||||
|
||||
|
||||
def _yuris_pulse_progress_complete(filename: str) -> None:
|
||||
"""
|
||||
GUI/ subprocess_runner polls modules.yuris.PBAR. Skipped files and 0-line files
|
||||
never touched tqdm otherwise, so the monitor never emits PROGRESS:...:n:total and
|
||||
rows sit at 0% or odd states until completion.
|
||||
"""
|
||||
global PBAR
|
||||
with tqdm(
|
||||
total=1,
|
||||
bar_format=BAR_FORMAT,
|
||||
position=POSITION,
|
||||
leave=LEAVE,
|
||||
desc=filename,
|
||||
) as pbar:
|
||||
PBAR = pbar
|
||||
pbar.update(1)
|
||||
PBAR = None
|
||||
|
||||
|
||||
def handleYuris(filename, estimate):
|
||||
global ESTIMATE, FILENAME, TOKENS
|
||||
ESTIMATE = estimate
|
||||
|
|
@ -65,18 +166,36 @@ def handleYuris(filename, estimate):
|
|||
|
||||
try:
|
||||
start = time.time()
|
||||
translatedData = openFiles(filename)
|
||||
with open("files/" + filename, "r", encoding="UTF-8-sig") as readFile:
|
||||
data = json.load(readFile)
|
||||
|
||||
if not estimate:
|
||||
os.makedirs(os.path.dirname(os.path.join("translated", filename)) or "translated", exist_ok=True)
|
||||
with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
|
||||
json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
|
||||
pass_reason = yuris_whole_file_pass_through_reason(data)
|
||||
if pass_reason:
|
||||
if not estimate:
|
||||
os.makedirs(os.path.dirname(os.path.join("translated", filename)) or "translated", exist_ok=True)
|
||||
with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
|
||||
json.dump(data, outFile, ensure_ascii=False, indent=4)
|
||||
end = time.time()
|
||||
# Machine-readable line so the GUI can apply totals row state (same as other modules).
|
||||
c0 = calculateCost(0, 0, MODEL)
|
||||
tqdm.write(
|
||||
f"{filename}: [Input: 0][Output: 0][Cost: ${c0:.4f}]"
|
||||
f"[{round(end - start, 1)}s] [skipped] {pass_reason}"
|
||||
)
|
||||
_yuris_pulse_progress_complete(filename)
|
||||
else:
|
||||
translatedData = parseYuris(data, filename)
|
||||
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(translatedData, end - start, filename))
|
||||
with LOCK:
|
||||
TOKENS[0] += translatedData[1][0]
|
||||
TOKENS[1] += translatedData[1][1]
|
||||
if not estimate:
|
||||
os.makedirs(os.path.dirname(os.path.join("translated", filename)) or "translated", exist_ok=True)
|
||||
with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
|
||||
json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
|
||||
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(translatedData, end - start, filename))
|
||||
with LOCK:
|
||||
TOKENS[0] += translatedData[1][0]
|
||||
TOKENS[1] += translatedData[1][1]
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
return "Fail"
|
||||
|
|
@ -87,13 +206,6 @@ def handleYuris(filename, estimate):
|
|||
return totalString
|
||||
|
||||
|
||||
def openFiles(filename):
|
||||
with open("files/" + filename, "r", encoding="UTF-8-sig") as readFile:
|
||||
data = json.load(readFile)
|
||||
translatedData = parseYuris(data, filename)
|
||||
return translatedData
|
||||
|
||||
|
||||
def parseYuris(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
global PBAR
|
||||
|
|
@ -104,19 +216,25 @@ def parseYuris(data, filename):
|
|||
totalLines = sum(
|
||||
1
|
||||
for item in data
|
||||
if isinstance(item, dict) and isinstance(item.get("message"), str) and item["message"].strip()
|
||||
if isinstance(item, dict) and yuris_message_is_translatable(item.get("message", ""))
|
||||
)
|
||||
|
||||
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
|
||||
# total=0 breaks tqdm + GUI progress monitor; still pulse 1/1 when nothing to translate.
|
||||
pbar_total = max(1, totalLines)
|
||||
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=pbar_total, leave=LEAVE) as pbar:
|
||||
pbar.desc = filename
|
||||
PBAR = pbar
|
||||
try:
|
||||
if totalLines == 0:
|
||||
pbar.update(1)
|
||||
result = translateYuris(data, filename)
|
||||
totalTokens[0] += result[0]
|
||||
totalTokens[1] += result[1]
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
PBAR = None
|
||||
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
|
@ -140,10 +258,8 @@ def translateYuris(data, filename):
|
|||
item["name"] = speaker
|
||||
|
||||
message = item.get("message")
|
||||
if not isinstance(message, str) or not message.strip():
|
||||
if not yuris_message_is_translatable(message):
|
||||
continue
|
||||
# if not re.search(LANGREGEX, message):
|
||||
# continue
|
||||
|
||||
cleanMessage = message.replace("\r\n", " ").replace("\n", " ").strip()
|
||||
cleanMessage = replace_yuris_problem_dashes(cleanMessage)
|
||||
|
|
@ -172,9 +288,8 @@ def translateYuris(data, filename):
|
|||
if hasSpeaker:
|
||||
translatedText = stripSpeakerPrefix(translatedText)
|
||||
translatedText = replace_yuris_problem_dashes(translatedText)
|
||||
translatedText = translatedText.replace("\r\n", "\n")
|
||||
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
|
||||
translatedText = apply_yuris_newline_indent_after_wrap(translatedText)
|
||||
translatedText = translatedText.replace("\n", "\r\n")
|
||||
data[index]["message"] = originalMessage.replace(originalMessage, translatedText)
|
||||
save_progress_json(data, filename)
|
||||
if PBAR is not None:
|
||||
|
|
@ -199,28 +314,6 @@ def stripSpeakerPrefix(translated_text):
|
|||
return translated_text.strip()
|
||||
|
||||
|
||||
def apply_yuris_newline_indent_after_wrap(text: str, sep: str = "\r\n") -> str:
|
||||
"""Engine quirk: after each newline within wrapped text, add one more leading space (1st wrap line +1, 2nd +2, …).
|
||||
|
||||
Paragraph breaks from wrapText (blank line / \\n\\n) reset the counter per paragraph.
|
||||
Output uses CRLF between lines and double CRLF between paragraphs.
|
||||
"""
|
||||
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
paragraphs = normalized.split("\n\n")
|
||||
out_paragraphs: list[str] = []
|
||||
for para in paragraphs:
|
||||
lines = para.split("\n")
|
||||
adjusted: list[str] = []
|
||||
for i, raw in enumerate(lines):
|
||||
content = raw.strip()
|
||||
if i == 0:
|
||||
adjusted.append(content)
|
||||
else:
|
||||
adjusted.append((" " * i) + content)
|
||||
out_paragraphs.append(sep.join(adjusted))
|
||||
return (sep + sep).join(out_paragraphs)
|
||||
|
||||
|
||||
def save_progress_json(data, filename):
|
||||
try:
|
||||
if ESTIMATE:
|
||||
|
|
|
|||
|
|
@ -308,10 +308,22 @@ def validate_translation_content(original_items, translated_items, langRegex):
|
|||
|
||||
# Check 4: Runaway translation - translation is excessively long relative to original
|
||||
# Catches cases where the model repeats words endlessly (e.g. "it hurts it hurts it hurts...")
|
||||
if len(orig_str) > 10 and len(trans_str) > len(orig_str) * 10:
|
||||
ratio_limit = max(len(orig_str) * 8, 120)
|
||||
if len(orig_str) > 10 and len(trans_str) > ratio_limit:
|
||||
invalid_indices.append(i)
|
||||
reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars vs input {len(orig_str)} chars) for '{orig_str[:50]}...'")
|
||||
continue
|
||||
# Absolute cap: garbage outputs that are not caught by ratio alone
|
||||
if len(trans_str) > 4000 and len(trans_str) > len(orig_str) * 3:
|
||||
invalid_indices.append(i)
|
||||
reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars exceeds cap) for '{orig_str[:50]}...'")
|
||||
continue
|
||||
|
||||
# Check 5: Same character repeated many times (common API glitch / broken JSON tail)
|
||||
if re.search(r"(.)\1{44,}", trans_str):
|
||||
invalid_indices.append(i)
|
||||
reasons.append(f"Line{i+1}: Excessive character repetition (possible model glitch) in translation")
|
||||
continue
|
||||
|
||||
is_valid = len(invalid_indices) == 0
|
||||
return is_valid, invalid_indices, reasons
|
||||
|
|
@ -1257,7 +1269,6 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
|
|||
ant_kwargs = dict(
|
||||
model=model,
|
||||
max_tokens=16384,
|
||||
temperature=0,
|
||||
system=ant_system,
|
||||
messages=native_msgs,
|
||||
)
|
||||
|
|
@ -1268,6 +1279,11 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
|
|||
"schema": createTranslationSchema(numLines),
|
||||
}
|
||||
}
|
||||
else:
|
||||
# Plain completions still allow explicit sampling params.
|
||||
ant_kwargs["temperature"] = 0
|
||||
# Do not pass temperature with output_config: newer Claude (e.g. Opus 4.7)
|
||||
# returns errors such as "temperature is not supported" for structured outputs.
|
||||
|
||||
ant_resp = ant_client.messages.create(**ant_kwargs)
|
||||
except Exception as e:
|
||||
|
|
@ -1903,7 +1919,9 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism
|
|||
f" - Do NOT leave translations empty (\"\") or as single punctuation marks (\":\")\n"
|
||||
f"3. ALL placeholders (like __PROTECTED_0__, __PROTECTED_1__, etc.) are preserved EXACTLY as they appear in the input\n"
|
||||
f" - Do not modify, translate, or remove any __PROTECTED_N__ placeholders\n"
|
||||
f" - Keep them in the exact same position in your translation\n\n"
|
||||
f" - Keep them in the exact same position in your translation\n"
|
||||
f"4. Do NOT repeat the same letter or symbol many times in a row (e.g. uuuuuuuu... or broken tails)\n"
|
||||
f" - Keep moans/effects natural; never output long runs of one character\n\n"
|
||||
)
|
||||
current_user = retry_note + user
|
||||
if pbar:
|
||||
|
|
|
|||
Loading…
Reference in a new issue