From a1af8fb985f3cfc685e21176456cbb220e62259b Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Tue, 21 Jul 2026 16:20:10 -0500 Subject: [PATCH] fix(csv): preserve finished targets and honor batch size env - Queue only untranslated rows from mixed skip-check batches - Let .env batchsize and frequency_penalty override model defaults - Cover skip behavior and pricing batch size override in tests --- gui/csv_tab.py | 4 ++-- modules/csv.py | 19 ++++++++------- tests/test_csv_skip_translated.py | 30 +++++++++++++++++------- tests/test_pricing_batchsize_override.py | 22 +++++++++++++++++ util/translation.py | 15 ++++++++++++ 5 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 tests/test_pricing_batchsize_override.py diff --git a/gui/csv_tab.py b/gui/csv_tab.py index 059cf23..bd4ba1b 100644 --- a/gui/csv_tab.py +++ b/gui/csv_tab.py @@ -193,8 +193,8 @@ class CSVTab(QWidget): self.skip_if_target_translated_cb = QCheckBox("Skip if Target Translated") self.skip_if_target_translated_cb.setToolTip( "Check candidate rows in batches matching the global Batch Size setting. " - "Skip a batch only if every target is already translated (non-empty, no Japanese). " - "If any row still needs work, the whole batch is translated." + "Skip a batch when every target is already translated (non-empty, no Japanese). " + "If any row still needs work, translate only the unfinished rows in that batch." ) left_column.addWidget(self.skip_if_target_translated_cb) diff --git a/modules/csv.py b/modules/csv.py index 9e9f0f5..c0a2e1d 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -42,13 +42,13 @@ FILENAME = None BRACKETNAMES = False # CSV Configuration Settings (configurable via GUI) -CSV_DELIMITER = " " # CSV delimiter character (comma, semicolon, tab) -SOURCE_COLUMN = 2 # Which column has the source text to translate (0-indexed) -TARGET_COLUMN = 3 # Which column to write translations to +CSV_DELIMITER = "," # CSV delimiter character (comma, semicolon, tab) +SOURCE_COLUMN = 0 # Which column has the source text to translate (0-indexed) +TARGET_COLUMN = 1 # Which column to write translations to SPEAKER_COLUMN = 1 # Which column has speaker names (-1 = none) -SKIP_HEADER_ROW = False # Skip the first row (header) +SKIP_HEADER_ROW = True # Skip the first row (header) USE_TARGET_IF_NOT_EMPTY = False # Use target column text if not empty (T++ style) -SKIP_IF_TARGET_TRANSLATED = False # Skip batches whose targets are already translated +SKIP_IF_TARGET_TRANSLATED = True # Skip batches whose targets are already translated WRITE_TO_NEXT_COLUMN = False # Write to column after target instead of overwriting PARSE_NAME_TAGS = False # Parse :name[] tags in text PARSE_M_MARKERS = False # Parse \M markers in text @@ -256,9 +256,10 @@ def _collect_process_indices(data): """Row indices to translate. When SKIP_IF_TARGET_TRANSLATED is on, candidates are checked in groups of - BATCHSIZE (same as the translation batch size). A group is skipped only if - every target in it is already translated; if any row still needs work, the - whole group runs. + BATCHSIZE (same as the translation batch size): + - If every target in the group is already translated, skip the group. + - If any row still needs work, keep the group but only queue the + untranslated rows so finished targets are left alone. """ candidates = [i for i in range(len(data)) if _is_candidate_row(data, i)] if not SKIP_IF_TARGET_TRANSLATED: @@ -270,7 +271,7 @@ def _collect_process_indices(data): batch = candidates[start:start + batch_size] if all(_target_is_translated(data[i]) for i in batch): continue - process.extend(batch) + process.extend(i for i in batch if not _target_is_translated(data[i])) return process diff --git a/tests/test_csv_skip_translated.py b/tests/test_csv_skip_translated.py index 6d6cf3c..5554df0 100644 --- a/tests/test_csv_skip_translated.py +++ b/tests/test_csv_skip_translated.py @@ -89,7 +89,7 @@ class SkipTranslatedBatchIndexTests(unittest.TestCase): # First batch (0,1) all translated -> skip; second batch (2,3) needs work self.assertEqual(csv_mod._collect_process_indices(data), [2, 3]) - def test_partial_batch_keeps_entire_batch(self): + def test_partial_batch_queues_only_untranslated_rows(self): csv_mod.SKIP_IF_TARGET_TRANSLATED = True csv_mod.BATCHSIZE = 3 data = [ @@ -100,9 +100,9 @@ class SkipTranslatedBatchIndexTests(unittest.TestCase): ["お", "O"], ["か", "Ka"], ] - # Batch [0,1,2] has untranslated row 1 -> keep all three + # Batch [0,1,2] has work -> keep only untranslated row 1 # Batch [3,4,5] all translated -> skip - self.assertEqual(csv_mod._collect_process_indices(data), [0, 1, 2]) + self.assertEqual(csv_mod._collect_process_indices(data), [1]) def test_uses_global_batch_size(self): csv_mod.SKIP_IF_TARGET_TRANSLATED = True @@ -112,6 +112,18 @@ class SkipTranslatedBatchIndexTests(unittest.TestCase): # First 30 fully translated -> skipped; last singleton batch needs work self.assertEqual(csv_mod._collect_process_indices(data), [30]) + def test_header_with_english_target_skipped_when_opt_in(self): + csv_mod.SKIP_IF_TARGET_TRANSLATED = True + csv_mod.SKIP_HEADER_ROW = False + csv_mod.BATCHSIZE = 10 + data = [ + ["Source", "Target"], + ["こんにちは", ""], + ["ありがとう", "Thank you."], + ] + # Header target looks translated; thank-you row preserved; only empty target queued + self.assertEqual(csv_mod._collect_process_indices(data), [1]) + class SkipIfTargetTranslatedCollectTests(unittest.TestCase): def setUp(self): @@ -176,14 +188,14 @@ class SkipIfTargetTranslatedCollectTests(unittest.TestCase): with patch.object(csv_mod, "dazedwrap") as mock_wrap: mock_wrap.wrapText.side_effect = lambda text, _width: text csv_mod.translateCSV(data, pbar, None, MagicMock(), "test.csv", None) - # First batch fully translated -> skipped; second batch kept whole + # First batch fully translated -> skipped; second batch keeps unfinished only self.assertEqual(mock_ai.call_args[0][0], ["さようなら", "ありがとう"]) self.assertEqual(data[0][1], "Hello") self.assertEqual(data[1][1], "Good morning") self.assertEqual(data[2][1], "Goodbye") self.assertEqual(data[3][1], "Thank you") - def test_partial_batch_retranslates_already_done_rows(self): + def test_partial_batch_preserves_already_done_rows(self): csv_mod.SKIP_IF_TARGET_TRANSLATED = True csv_mod.BATCHSIZE = 2 data = [ @@ -192,14 +204,14 @@ class SkipIfTargetTranslatedCollectTests(unittest.TestCase): ] pbar = MagicMock() with patch.object( - csv_mod, "translateAI", return_value=(["Hi again", "Goodbye"], [0, 0]) + csv_mod, "translateAI", return_value=(["Goodbye"], [0, 0]) ) as mock_ai: with patch.object(csv_mod, "dazedwrap") as mock_wrap: mock_wrap.wrapText.side_effect = lambda text, _width: text csv_mod.translateCSV(data, pbar, None, MagicMock(), "test.csv", None) - # Mixed batch: keep both rows, including the already-translated one - self.assertEqual(mock_ai.call_args[0][0], ["こんにちは", "さようなら"]) - self.assertEqual(data[0][1], "Hi again") + # Mixed batch: only the unfinished row is sent; existing translation kept + self.assertEqual(mock_ai.call_args[0][0], ["さようなら"]) + self.assertEqual(data[0][1], "Hello") self.assertEqual(data[1][1], "Goodbye") diff --git a/tests/test_pricing_batchsize_override.py b/tests/test_pricing_batchsize_override.py new file mode 100644 index 0000000..32a0d12 --- /dev/null +++ b/tests/test_pricing_batchsize_override.py @@ -0,0 +1,22 @@ +import os +import unittest +from unittest.mock import patch + +from util.translation import getPricingConfig + + +class PricingBatchSizeOverrideTests(unittest.TestCase): + def test_env_batchsize_overrides_model_default(self): + with patch.dict(os.environ, {"batchsize": "2"}, clear=False): + cfg = getPricingConfig("claude-sonnet-4-6") + self.assertEqual(cfg["batchSize"], 2) + + def test_missing_env_keeps_model_default(self): + env = {k: v for k, v in os.environ.items() if k != "batchsize"} + with patch.dict(os.environ, env, clear=True): + cfg = getPricingConfig("claude-sonnet-4-6") + self.assertEqual(cfg["batchSize"], 30) + + +if __name__ == "__main__": + unittest.main() diff --git a/util/translation.py b/util/translation.py index 2109737..875374b 100644 --- a/util/translation.py +++ b/util/translation.py @@ -1756,6 +1756,21 @@ def getPricingConfig(model): if _live_override: cfg.update(_live_override) + # Config / .env overrides win over model defaults so the Batch Size and + # Frequency Penalty controls actually affect translation batching. + env_batch = os.getenv("batchsize") + if env_batch not in (None, ""): + try: + cfg["batchSize"] = max(1, int(env_batch)) + except (TypeError, ValueError): + pass + env_penalty = os.getenv("frequency_penalty") + if env_penalty not in (None, ""): + try: + cfg["frequencyPenalty"] = float(env_penalty) + except (TypeError, ValueError): + pass + return cfg