From 85ad7c5914b84086f8750f8f7fa674ca7e8a95a2 Mon Sep 17 00:00:00 2001 From: dazedanon Date: Fri, 18 Jul 2025 16:38:33 -0500 Subject: [PATCH] Make log faster --- gui/log_viewer.py | 178 ++++++++++++++++++++++++++++++++++++----- gui/translation_tab.py | 111 ++++++++++++++++++------- 2 files changed, 240 insertions(+), 49 deletions(-) diff --git a/gui/log_viewer.py b/gui/log_viewer.py index 8985555..8aa9606 100644 --- a/gui/log_viewer.py +++ b/gui/log_viewer.py @@ -4,12 +4,13 @@ Log Viewer - Real-time log display and monitoring from PyQt5.QtWidgets import ( QWidget, QVBoxLayout, QHBoxLayout, QTextEdit, QPushButton, - QLabel, QCheckBox, QComboBox, QGroupBox, QSplitter + QLabel, QCheckBox, QComboBox, QGroupBox, QSplitter, QSpinBox ) from PyQt5.QtCore import Qt, QTimer, pyqtSignal from PyQt5.QtGui import QTextCursor, QFont from pathlib import Path import datetime +import os class LogViewer(QWidget): @@ -25,6 +26,13 @@ class LogViewer(QWidget): } self.auto_refresh_timer = QTimer() self.auto_refresh_timer.timeout.connect(self.refresh_logs) + + # Performance optimization tracking + self.file_mod_times = {} # Track file modification times + self.file_positions = {} # Track file read positions for tail-only updates + self.max_lines = 1000 # Maximum lines to display + self.refresh_interval = 2000 # Default refresh interval in ms + self.init_ui() def init_ui(self): @@ -47,6 +55,15 @@ class LogViewer(QWidget): self.auto_refresh_cb.toggled.connect(self.toggle_auto_refresh) control_layout.addWidget(self.auto_refresh_cb) + # Max lines control + control_layout.addWidget(QLabel("Max Lines:")) + self.max_lines_spinbox = QSpinBox() + self.max_lines_spinbox.setRange(100, 10000) + self.max_lines_spinbox.setValue(self.max_lines) + self.max_lines_spinbox.setSingleStep(100) + self.max_lines_spinbox.valueChanged.connect(self.update_max_lines) + control_layout.addWidget(self.max_lines_spinbox) + # Manual refresh button self.refresh_button = QPushButton("Refresh") self.refresh_button.clicked.connect(self.refresh_logs) @@ -87,12 +104,34 @@ class LogViewer(QWidget): def toggle_auto_refresh(self, enabled): """Toggle auto-refresh functionality.""" if enabled: - self.auto_refresh_timer.start(2000) # Refresh every 2 seconds - self.status_label.setText("Auto-refresh enabled") + # Adjust refresh interval based on file size + self.adjust_refresh_interval() + self.auto_refresh_timer.start(self.refresh_interval) + self.status_label.setText(f"Auto-refresh enabled ({self.refresh_interval/1000:.1f}s interval)") else: self.auto_refresh_timer.stop() self.status_label.setText("Auto-refresh disabled") + def adjust_refresh_interval(self): + """Adjust refresh interval based on current log file size.""" + selected_log = self.log_selector.currentText() + if selected_log in self.log_files: + log_path = Path(self.log_files[selected_log]) + if log_path.exists(): + file_size = log_path.stat().st_size + # Increase interval for larger files + if file_size > 500000: # > 500KB + self.refresh_interval = 5000 # 5 seconds + elif file_size > 100000: # > 100KB + self.refresh_interval = 3000 # 3 seconds + else: + self.refresh_interval = 2000 # 2 seconds + + def update_max_lines(self, value): + """Update maximum lines to display.""" + self.max_lines = value + self.load_selected_log() # Reload with new limit + def load_selected_log(self): """Load the currently selected log file.""" selected_log = self.log_selector.currentText() @@ -100,36 +139,123 @@ class LogViewer(QWidget): self.load_log_file(self.log_files[selected_log]) def load_log_file(self, file_path): - """Load content from a specific log file.""" + """Load content from a specific log file with optimization.""" try: log_path = Path(file_path) - if log_path.exists(): - with open(log_path, 'r', encoding='utf-8') as f: - content = f.read() - self.log_display.setPlainText(content) - - # Scroll to bottom - cursor = self.log_display.textCursor() - cursor.movePosition(QTextCursor.End) - self.log_display.setTextCursor(cursor) - - # Update status - file_size = log_path.stat().st_size - self.status_label.setText(f"Loaded {log_path.name} ({file_size} bytes)") - else: + if not log_path.exists(): self.log_display.setPlainText(f"Log file not found: {file_path}") self.status_label.setText("Log file not found") + return + + # Check if file has been modified since last read + current_mod_time = log_path.stat().st_mtime + last_mod_time = self.file_mod_times.get(file_path, 0) + + if current_mod_time <= last_mod_time and file_path in self.file_mod_times: + # File hasn't changed, no need to reload + return + + # Update modification time + self.file_mod_times[file_path] = current_mod_time + + # Read file efficiently + file_size = log_path.stat().st_size + + if file_size > 1000000: # > 1MB, read only tail + content = self.read_file_tail(log_path, self.max_lines) + else: + # For smaller files, read normally but limit lines + with open(log_path, 'r', encoding='utf-8', errors='ignore') as f: + content = f.read() + + # Limit to max lines if content is too long + lines = content.splitlines() + if len(lines) > self.max_lines: + content = '\n'.join(lines[-self.max_lines:]) + content = f"... (showing last {self.max_lines} lines) ...\n" + content + + self.log_display.setPlainText(content) + + # Scroll to bottom + cursor = self.log_display.textCursor() + cursor.movePosition(QTextCursor.End) + self.log_display.setTextCursor(cursor) + + # Update status + lines_count = len(content.splitlines()) + self.status_label.setText(f"Loaded {log_path.name} ({file_size:,} bytes, {lines_count:,} lines)") + except Exception as e: self.log_display.setPlainText(f"Error loading log file: {str(e)}") self.status_label.setText(f"Error: {str(e)}") + def read_file_tail(self, file_path, max_lines): + """Efficiently read the last N lines of a large file.""" + try: + with open(file_path, 'rb') as f: + # Start from end of file + f.seek(0, 2) # Seek to end + file_size = f.tell() + + # Read chunks from end until we have enough lines + lines = [] + chunk_size = min(8192, file_size) # 8KB chunks + pos = file_size + + while len(lines) < max_lines and pos > 0: + # Calculate chunk position + chunk_start = max(0, pos - chunk_size) + f.seek(chunk_start) + + # Read chunk + chunk = f.read(pos - chunk_start).decode('utf-8', errors='ignore') + chunk_lines = chunk.splitlines() + + # Prepend to lines list + if chunk_lines: + if lines and not chunk.endswith('\n'): + # Merge partial line at end of chunk with first existing line + chunk_lines[-1] += lines[0] + lines = chunk_lines + lines[1:] + else: + lines = chunk_lines + lines + + pos = chunk_start + + # Return last max_lines + if len(lines) > max_lines: + lines = lines[-max_lines:] + return f"... (showing last {max_lines} lines) ...\n" + '\n'.join(lines) + else: + return '\n'.join(lines) + + except Exception as e: + return f"Error reading file tail: {str(e)}" + def refresh_logs(self): - """Refresh the current log display.""" + """Refresh the current log display - only if tab is visible and file changed.""" + # Only refresh if widget is visible (optimization for when tab is not active) + if not self.isVisible(): + return + self.load_selected_log() + # Readjust refresh interval if auto-refresh is enabled + if self.auto_refresh_cb.isChecked(): + self.adjust_refresh_interval() + if self.auto_refresh_timer.interval() != self.refresh_interval: + self.auto_refresh_timer.stop() + self.auto_refresh_timer.start(self.refresh_interval) + def clear_log(self): - """Clear the log display.""" + """Clear the log display and reset tracking.""" self.log_display.clear() + # Reset modification time tracking to force reload on next refresh + selected_log = self.log_selector.currentText() + if selected_log in self.log_files: + file_path = self.log_files[selected_log] + if file_path in self.file_mod_times: + del self.file_mod_times[file_path] self.status_label.setText("Log cleared") def append_log_message(self, message): @@ -147,3 +273,15 @@ class LogViewer(QWidget): def get_current_log_content(self): """Get the current log content.""" return self.log_display.toPlainText() + + def showEvent(self, event): + """Handle widget show event - resume refresh if auto-refresh is enabled.""" + super().showEvent(event) + if self.auto_refresh_cb.isChecked() and not self.auto_refresh_timer.isActive(): + self.auto_refresh_timer.start(self.refresh_interval) + + def hideEvent(self, event): + """Handle widget hide event - pause refresh to save resources.""" + super().hideEvent(event) + # Note: Don't stop timer completely as other parts may depend on it + # The refresh_logs method now checks visibility anyway diff --git a/gui/translation_tab.py b/gui/translation_tab.py index ced10a5..333063a 100644 --- a/gui/translation_tab.py +++ b/gui/translation_tab.py @@ -20,7 +20,7 @@ from PyQt5.QtWidgets import ( QTextEdit, QMessageBox, QListWidget, QListWidgetItem, QSplitter, QFileDialog, QComboBox, QCheckBox, QProgressBar ) -from PyQt5.QtCore import Qt, QTimer, pyqtSignal, QThread +from PyQt5.QtCore import Qt, QTimer, pyqtSignal, QThread, QMutex from PyQt5.QtGui import QFont @@ -37,10 +37,23 @@ class TranslationWorker(QThread): self.module_info = module_info # [name, extensions, handler_function] self.estimate_only = estimate_only self.should_stop = False + self.mutex = QMutex() # For thread safety def stop(self): """Stop the translation process.""" - self.should_stop = True + self.mutex.lock() + try: + self.should_stop = True + finally: + self.mutex.unlock() + + def emit_log(self, message): + """Thread-safe log emission.""" + self.log_signal.emit(message) + + def emit_progress(self, current, total, filename): + """Thread-safe progress emission.""" + self.progress_signal.emit(current, total, filename) def run(self): """Run the translation process.""" @@ -54,18 +67,18 @@ class TranslationWorker(QThread): for env in required_envs: if os.getenv(env) is None or str(os.getenv(env))[:1] == "<": - self.log_signal.emit(f"❌ Environment variable {env} is not set!") + self.emit_log(f"❌ Environment variable {env} is not set!") env_missing = True if env_missing: - self.log_signal.emit("❌ Some required environment variables are not set. Check your .env file.") + self.emit_log("❌ Some required environment variables are not set. Check your .env file.") self.finished_signal.emit(False, "Environment variables missing") return # Get files to process files_dir = self.project_root / "files" if not files_dir.exists(): - self.log_signal.emit("❌ Files directory does not exist!") + self.emit_log("❌ Files directory does not exist!") self.finished_signal.emit(False, "Files directory missing") return @@ -79,17 +92,17 @@ class TranslationWorker(QThread): break if not matching_files: - self.log_signal.emit(f"❌ No files found matching extensions: {', '.join(self.module_info[1])}") + self.emit_log(f"❌ No files found matching extensions: {', '.join(self.module_info[1])}") self.finished_signal.emit(False, "No matching files") return - self.log_signal.emit(f"📁 Found {len(matching_files)} files to process:") + self.emit_log(f"📁 Found {len(matching_files)} files to process:") for filename in matching_files: - self.log_signal.emit(f" • {filename}") + self.emit_log(f" • {filename}") - self.log_signal.emit(f"🔧 Using module: {self.module_info[0]}") - self.log_signal.emit(f"📊 Estimate only: {'Yes' if self.estimate_only else 'No'}") - self.log_signal.emit("") + self.emit_log(f"🔧 Using module: {self.module_info[0]}") + self.emit_log(f"📊 Estimate only: {'Yes' if self.estimate_only else 'No'}") + self.emit_log("") # Process files threads = int(os.getenv("fileThreads", "1")) @@ -112,23 +125,24 @@ class TranslationWorker(QThread): for future in as_completed(future_to_filename): if self.should_stop: - self.log_signal.emit("🛑 Translation stopped by user") + self.emit_log("🛑 Translation stopped by user") break filename = future_to_filename[future] completed_count += 1 - # Emit progress signal - self.progress_signal.emit(completed_count, total_files, filename) + # Emit progress signal (less frequent updates) + self.emit_progress(completed_count, total_files, filename) try: result = future.result() total_cost = result - self.log_signal.emit(f"✅ Completed {filename} ({completed_count}/{total_files})") + # Only log every file completion, not internal progress + self.emit_log(f"✅ Completed {filename} ({completed_count}/{total_files})") except Exception as e: tb_line = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) error_msg = f"❌ Error processing {filename}: {str(e)} | Line: {tb_line}" - self.log_signal.emit(error_msg) + self.emit_log(error_msg) finally: os.chdir(old_cwd) @@ -140,21 +154,21 @@ class TranslationWorker(QThread): # Report results if total_cost != "Fail" and not self.should_stop: - self.log_signal.emit("") - self.log_signal.emit(f"💰 {total_cost}") + self.emit_log("") + self.emit_log(f"💰 {total_cost}") if not self.estimate_only: - self.log_signal.emit("✅ Translation completed successfully!") + self.emit_log("✅ Translation completed successfully!") else: - self.log_signal.emit("✅ Estimation completed!") + self.emit_log("✅ Estimation completed!") self.finished_signal.emit(True, str(total_cost)) else: if not self.should_stop: - self.log_signal.emit("❌ Translation failed!") + self.emit_log("❌ Translation failed!") self.finished_signal.emit(False, "Translation failed") except Exception as e: error_msg = f"❌ Unexpected error: {str(e)}" - self.log_signal.emit(error_msg) + self.emit_log(error_msg) self.finished_signal.emit(False, error_msg) @@ -165,6 +179,9 @@ class TranslationTab(QWidget): super().__init__(parent) self.parent_window = parent self.translation_process = None + self.log_buffer = [] # Buffer for batching log messages + self.log_timer = QTimer() # Timer for flushing log buffer + self.log_timer.timeout.connect(self.flush_log_buffer) # Set up directories self.project_root = Path(__file__).parent.parent @@ -178,7 +195,7 @@ class TranslationTab(QWidget): self.setup_ui() self.refresh_file_lists() - # Auto-refresh timer for file lists + # Auto-refresh timer for file lists (less frequent during translation) self.refresh_timer = QTimer() self.refresh_timer.timeout.connect(self.refresh_file_lists) self.refresh_timer.start(3000) # Refresh every 3 seconds @@ -634,19 +651,37 @@ class TranslationTab(QWidget): self.progress_bar.setValue(0) self.progress_label.setText("Starting...") + # Reduce file refresh frequency during translation + self.refresh_timer.stop() + # Start the worker self.translation_worker.start() def append_log(self, message): - """Append a message to the log display.""" - self.log_display.append(message) - # Auto-scroll to bottom - scrollbar = self.log_display.verticalScrollBar() - scrollbar.setValue(scrollbar.maximum()) + """Append a message to the log buffer for batched display.""" + self.log_buffer.append(message) + + # Start timer if not already running + if not self.log_timer.isActive(): + self.log_timer.start(100) # Flush every 100ms + + def flush_log_buffer(self): + """Flush the log buffer to the display.""" + if self.log_buffer: + # Add all buffered messages at once + for message in self.log_buffer: + self.log_display.append(message) + self.log_buffer.clear() + + # Auto-scroll to bottom + scrollbar = self.log_display.verticalScrollBar() + scrollbar.setValue(scrollbar.maximum()) + + # Stop timer if buffer is empty + self.log_timer.stop() def update_progress(self, current_file, total_files, filename): """Update the progress bar and label.""" - progress_percentage = int((current_file / total_files) * 100) self.progress_bar.setMaximum(total_files) self.progress_bar.setValue(current_file) self.progress_label.setText(f"Processing {filename} ({current_file}/{total_files})") @@ -657,6 +692,12 @@ class TranslationTab(QWidget): self.stop_button.setEnabled(False) self.progress_bar.setVisible(False) self.progress_label.setText("Ready") + + # Flush any remaining log messages + self.flush_log_buffer() + + # Resume file refresh + self.refresh_timer.start(3000) self.refresh_file_lists() if success: @@ -666,6 +707,9 @@ class TranslationTab(QWidget): self.append_log("") self.append_log(f"❌ Process failed: {message}") + # Force flush the final messages + self.flush_log_buffer() + def stop_translation(self): """Stop the translation process.""" if hasattr(self, 'translation_worker') and self.translation_worker.isRunning(): @@ -678,14 +722,23 @@ class TranslationTab(QWidget): self.progress_bar.setVisible(False) self.progress_label.setText("Ready") + # Resume file refresh + self.refresh_timer.start(3000) + + # Flush any remaining messages + self.flush_log_buffer() + def clear_log(self): """Clear the log display.""" + self.log_buffer.clear() # Clear buffer too self.log_display.clear() def closeEvent(self, event): """Handle widget close event.""" if hasattr(self, 'refresh_timer'): self.refresh_timer.stop() + if hasattr(self, 'log_timer'): + self.log_timer.stop() if hasattr(self, 'translation_worker') and self.translation_worker.isRunning(): self.translation_worker.stop() self.translation_worker.wait(3000)