Thread safety fixes

This commit is contained in:
dazedanon 2025-08-24 16:53:45 -05:00
parent fd9fc73ef5
commit 0c48e160b4
2 changed files with 82 additions and 6 deletions

1
.gitignore vendored
View file

@ -1,4 +1,5 @@
.env
*.tmp
*.json
*.js
*.yaml

View file

@ -10,6 +10,7 @@ import openai
import copy
# Removed concurrent.futures usage for simplicity; running synchronously
from pathlib import Path
import shutil
from colorama import Fore
from dotenv import load_dotenv
from retry import retry
@ -242,12 +243,55 @@ def saveProgress(data, filename):
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_path = os.path.join("translated", f"{filename}.tmp")
# Use a unique temp file name to avoid collisions across threads/processes
tmp_path = os.path.join(
"translated",
f"{filename}.{os.getpid()}.{threading.get_ident()}.tmp",
)
final_path = os.path.join("translated", filename)
with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
json.dump(data, outFile, ensure_ascii=False, indent=4)
# Replace atomically when possible
os.replace(tmp_path, final_path)
outFile.flush()
try:
os.fsync(outFile.fileno())
except Exception:
# fsync may not be available on some platforms; ignore best-effort
pass
# Replace atomically when possible, with retries to mitigate transient locks on Windows
attempts = 6
delay = 0.1
last_err = None
for attempt in range(attempts):
try:
os.replace(tmp_path, final_path)
last_err = None
break
except PermissionError as e:
last_err = e
# Try to relax permissions on target if it exists, then back off
try:
if os.path.exists(final_path):
os.chmod(final_path, 0o666)
except Exception:
pass
time.sleep(delay)
delay = min(1.0, delay * 2)
except Exception as e:
last_err = e
break
if last_err is not None:
# Fallback: try move via shutil (not guaranteed atomic), then raise on failure
try:
shutil.move(tmp_path, final_path)
except Exception:
# Ensure tmp is cleaned up if move failed
try:
if os.path.exists(tmp_path):
os.remove(tmp_path)
except Exception:
pass
raise last_err
except Exception:
# Best-effort; don't crash the translation if saving fails
traceback.print_exc()
@ -319,10 +363,41 @@ def update_vocab_section(category: str, pairs: list[tuple[str, str]]):
# Avoid writing if nothing changed
if updated == existing:
return
# Atomic write: write to temp and replace
tmp_path = vocab_path.with_suffix(vocab_path.suffix + ".tmp")
# Atomic write: write to unique temp and replace with retries on Windows
tmp_path = vocab_path.with_suffix(vocab_path.suffix + f".{os.getpid()}.{threading.get_ident()}.tmp")
tmp_path.write_text(updated, encoding="utf-8")
os.replace(tmp_path, vocab_path)
attempts = 6
delay = 0.1
last_err = None
for attempt in range(attempts):
try:
os.replace(tmp_path, vocab_path)
last_err = None
break
except PermissionError as e:
last_err = e
# Try relaxing permissions then retry
try:
if vocab_path.exists():
os.chmod(vocab_path, 0o666)
except Exception:
pass
time.sleep(delay)
delay = min(1.0, delay * 2)
except Exception as e:
last_err = e
break
if last_err is not None:
try:
shutil.move(str(tmp_path), str(vocab_path))
except Exception:
try:
if tmp_path.exists():
tmp_path.unlink(missing_ok=True)
except Exception:
pass
raise last_err
except Exception:
traceback.print_exc()