Properly handle corrupted text
This commit is contained in:
parent
d86999732c
commit
5bd8a19de3
2 changed files with 75 additions and 17 deletions
|
|
@ -917,7 +917,7 @@ def parseNames(data, filename, context):
|
|||
(r"<拡張説明:(.+?)>", False),
|
||||
(r"<STS DESC>\n(.+?)\n<", False),
|
||||
(r"text:(.+)>", False),
|
||||
(r"<Skill\d+:([^,>]+)", False),
|
||||
(r"<Skill\d+:(?:\d+,)?([^,>\d][^,>]*)", False),
|
||||
]
|
||||
|
||||
for entry in entries:
|
||||
|
|
@ -1256,7 +1256,7 @@ def searchNames(data, pbar, context, filename):
|
|||
(r"<拡張説明:(.+?)>", False),
|
||||
(r"<STS DESC>\n(.+?)\n<", False),
|
||||
(r"text:(.+)>", False),
|
||||
(r"<Skill\d+:([^,>]+)", False),
|
||||
(r"<Skill\d+:(?:\d+,)?([^,>\d][^,>]*)", False),
|
||||
]
|
||||
# For each entry, collect all note matches
|
||||
for idx, entry in enumerate(data):
|
||||
|
|
@ -2843,17 +2843,51 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
for text in textLines:
|
||||
list355655.append(text)
|
||||
else:
|
||||
for lineIdx in textLineIndices:
|
||||
# Collect all translated lines and re-wrap them
|
||||
translatedLines = []
|
||||
for _ in textLineIndices:
|
||||
if len(list355655) > 0:
|
||||
translatedText = list355655[0]
|
||||
list355655.pop(0)
|
||||
translatedText = translatedText.replace('\\"', "'")
|
||||
translatedText = translatedText.replace('"', "'")
|
||||
tl = list355655.pop(0)
|
||||
tl = tl.replace('\\"', "'")
|
||||
tl = tl.replace('"', "'")
|
||||
translatedLines.append(tl)
|
||||
|
||||
origParam = codeList[lineIdx]["parameters"][0]
|
||||
origMatch = re.search(mtxtRegex, origParam)
|
||||
if origMatch:
|
||||
codeList[lineIdx]["parameters"][0] = origParam.replace(origMatch.group(1), translatedText)
|
||||
if translatedLines:
|
||||
# Join all lines and re-wrap to WIDTH
|
||||
combined = " ".join(translatedLines)
|
||||
wrapped = dazedwrap.wrapText(combined, width=WIDTH)
|
||||
wrappedLines = [l for l in wrapped.split("\n") if l.strip()]
|
||||
|
||||
# Distribute wrapped lines across existing 355/655 slots
|
||||
for idx, lineIdx in enumerate(textLineIndices):
|
||||
if idx < len(wrappedLines):
|
||||
origParam = codeList[lineIdx]["parameters"][0]
|
||||
origMatch = re.search(mtxtRegex, origParam)
|
||||
if origMatch:
|
||||
codeList[lineIdx]["parameters"][0] = origParam.replace(origMatch.group(1), wrappedLines[idx])
|
||||
else:
|
||||
# More slots than lines: blank out the text
|
||||
origParam = codeList[lineIdx]["parameters"][0]
|
||||
origMatch = re.search(mtxtRegex, origParam)
|
||||
if origMatch:
|
||||
codeList[lineIdx]["parameters"][0] = origParam.replace(origMatch.group(1), "")
|
||||
|
||||
# If more wrapped lines than slots, insert new 655 codes
|
||||
if len(wrappedLines) > len(textLineIndices):
|
||||
lastIdx = textLineIndices[-1]
|
||||
indent = codeList[lastIdx].get("indent", 0)
|
||||
for extra in range(len(textLineIndices), len(wrappedLines)):
|
||||
new_item = {
|
||||
"code": 655,
|
||||
"indent": indent,
|
||||
"parameters": [
|
||||
' _MTxt += "' + wrappedLines[extra] + '" + "\\n";'
|
||||
],
|
||||
}
|
||||
insertPos = lastIdx + 1 + (extra - len(textLineIndices))
|
||||
codeList.insert(insertPos, new_item)
|
||||
# Adjust j to account for inserted items
|
||||
j += len(wrappedLines) - len(textLineIndices)
|
||||
|
||||
i = j - 1
|
||||
|
||||
|
|
|
|||
|
|
@ -125,12 +125,16 @@ def validate_placeholders(original_text, translated_text, replacements):
|
|||
def validate_translation_content(original_items, translated_items, langRegex):
|
||||
"""
|
||||
Validate that translated items are not empty or nearly empty.
|
||||
Returns: (is_valid, invalid_indices, reason)
|
||||
Returns: (is_valid, invalid_indices, reasons, corrupted_indices)
|
||||
|
||||
Rules:
|
||||
0. If original contains corrupted/mojibake text (U+FFFD), skip it and mark as corrupted
|
||||
1. If original has content, translation must not be empty or just whitespace
|
||||
2. If original has Japanese text, translation must not be a single punctuation mark
|
||||
3. Translation should have meaningful content (more than 1-2 characters for substantial originals)
|
||||
|
||||
corrupted_indices: list of indices where the original text is corrupted/mojibake
|
||||
— the caller should substitute the original text for these.
|
||||
"""
|
||||
if not isinstance(original_items, list):
|
||||
original_items = [original_items]
|
||||
|
|
@ -138,6 +142,7 @@ def validate_translation_content(original_items, translated_items, langRegex):
|
|||
|
||||
invalid_indices = []
|
||||
reasons = []
|
||||
corrupted_indices = []
|
||||
|
||||
for i, (orig, trans) in enumerate(zip(original_items, translated_items)):
|
||||
orig_str = str(orig).strip()
|
||||
|
|
@ -147,6 +152,12 @@ def validate_translation_content(original_items, translated_items, langRegex):
|
|||
if not orig_str or orig_str == "Placeholder Text":
|
||||
continue
|
||||
|
||||
# Check 0: Corrupted / mojibake text (U+FFFD replacement character)
|
||||
# These cannot be translated meaningfully — keep the original as-is.
|
||||
if "\ufffd" in orig_str:
|
||||
corrupted_indices.append(i)
|
||||
continue
|
||||
|
||||
# Check if original has content that needs translation
|
||||
has_source_text = bool(re.search(langRegex, orig_str))
|
||||
|
||||
|
|
@ -178,7 +189,7 @@ def validate_translation_content(original_items, translated_items, langRegex):
|
|||
continue
|
||||
|
||||
is_valid = len(invalid_indices) == 0
|
||||
return is_valid, invalid_indices, reasons
|
||||
return is_valid, invalid_indices, reasons, corrupted_indices
|
||||
|
||||
# (from .env if present), strip accidental whitespace, and set the base URL,
|
||||
# organization, and API key. It also handles the Gemini compatibility layer.
|
||||
|
|
@ -813,7 +824,6 @@ def cleanTranslatedText(translatedText, language):
|
|||
"’": "'",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
}
|
||||
|
|
@ -1168,10 +1178,15 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
|
|||
pbar.write(f"Extra placeholders: {', '.join(extra)}")
|
||||
else:
|
||||
# Check 3: Validate that translations are not empty or nearly empty
|
||||
content_valid, invalid_indices, content_reasons = validate_translation_content(
|
||||
content_valid, invalid_indices, content_reasons, corrupted_indices = validate_translation_content(
|
||||
tItem, extracted, config.langRegex
|
||||
)
|
||||
|
||||
# Substitute original text for corrupted/mojibake lines
|
||||
if corrupted_indices:
|
||||
for ci in corrupted_indices:
|
||||
extracted[ci] = tItem[ci]
|
||||
|
||||
if not content_valid:
|
||||
is_valid = False
|
||||
if pbar:
|
||||
|
|
@ -1182,7 +1197,11 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
|
|||
pbar.write(f" ... and {len(content_reasons) - 5} more issues")
|
||||
else:
|
||||
# Set translations (line count matches, placeholders valid, and content is good)
|
||||
final_translations = extracted
|
||||
# Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input)
|
||||
final_translations = [
|
||||
line.replace("Placeholder Text", "").strip() if isinstance(line, str) else line
|
||||
for line in extracted
|
||||
]
|
||||
else:
|
||||
# Single string: validate placeholders
|
||||
placeholder_valid, missing, extra = validate_placeholders(protected_items, cleaned_text, all_replacements[0])
|
||||
|
|
@ -1197,10 +1216,15 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
|
|||
else:
|
||||
# Validate content for single string
|
||||
final_cleaned = cleaned_text.replace("Placeholder Text", "")
|
||||
content_valid, _, content_reasons = validate_translation_content(
|
||||
content_valid, _, content_reasons, corrupted_indices = validate_translation_content(
|
||||
tItem, final_cleaned, config.langRegex
|
||||
)
|
||||
|
||||
# If original is corrupted/mojibake, just keep original text
|
||||
if corrupted_indices:
|
||||
final_translations = tItem
|
||||
break
|
||||
|
||||
if not content_valid:
|
||||
is_valid = False
|
||||
if pbar:
|
||||
|
|
|
|||
Loading…
Reference in a new issue