Make log faster

gui/log_viewer.py

@@ -4,12 +4,13 @@ Log Viewer - Real-time log display and monitoring
 
 from PyQt5.QtWidgets import (
     QWidget, QVBoxLayout, QHBoxLayout, QTextEdit, QPushButton,
-    QLabel, QCheckBox, QComboBox, QGroupBox, QSplitter
+    QLabel, QCheckBox, QComboBox, QGroupBox, QSplitter, QSpinBox
 )
 from PyQt5.QtCore import Qt, QTimer, pyqtSignal
 from PyQt5.QtGui import QTextCursor, QFont
 from pathlib import Path
 import datetime
+import os
 
 
 class LogViewer(QWidget):
@@ -25,6 +26,13 @@ class LogViewer(QWidget):
         }
         self.auto_refresh_timer = QTimer()
         self.auto_refresh_timer.timeout.connect(self.refresh_logs)
+        
+        # Performance optimization tracking
+        self.file_mod_times = {}  # Track file modification times
+        self.file_positions = {}  # Track file read positions for tail-only updates
+        self.max_lines = 1000  # Maximum lines to display
+        self.refresh_interval = 2000  # Default refresh interval in ms
+        
         self.init_ui()
         
     def init_ui(self):
@@ -47,6 +55,15 @@ class LogViewer(QWidget):
         self.auto_refresh_cb.toggled.connect(self.toggle_auto_refresh)
         control_layout.addWidget(self.auto_refresh_cb)
         
+        # Max lines control
+        control_layout.addWidget(QLabel("Max Lines:"))
+        self.max_lines_spinbox = QSpinBox()
+        self.max_lines_spinbox.setRange(100, 10000)
+        self.max_lines_spinbox.setValue(self.max_lines)
+        self.max_lines_spinbox.setSingleStep(100)
+        self.max_lines_spinbox.valueChanged.connect(self.update_max_lines)
+        control_layout.addWidget(self.max_lines_spinbox)
+        
         # Manual refresh button
         self.refresh_button = QPushButton("Refresh")
         self.refresh_button.clicked.connect(self.refresh_logs)
@@ -87,12 +104,34 @@ class LogViewer(QWidget):
     def toggle_auto_refresh(self, enabled):
         """Toggle auto-refresh functionality."""
         if enabled:
-            self.auto_refresh_timer.start(2000)  # Refresh every 2 seconds
-            self.status_label.setText("Auto-refresh enabled")
+            # Adjust refresh interval based on file size
+            self.adjust_refresh_interval()
+            self.auto_refresh_timer.start(self.refresh_interval)
+            self.status_label.setText(f"Auto-refresh enabled ({self.refresh_interval/1000:.1f}s interval)")
         else:
             self.auto_refresh_timer.stop()
             self.status_label.setText("Auto-refresh disabled")
             
+    def adjust_refresh_interval(self):
+        """Adjust refresh interval based on current log file size."""
+        selected_log = self.log_selector.currentText()
+        if selected_log in self.log_files:
+            log_path = Path(self.log_files[selected_log])
+            if log_path.exists():
+                file_size = log_path.stat().st_size
+                # Increase interval for larger files
+                if file_size > 500000:  # > 500KB
+                    self.refresh_interval = 5000  # 5 seconds
+                elif file_size > 100000:  # > 100KB
+                    self.refresh_interval = 3000  # 3 seconds
+                else:
+                    self.refresh_interval = 2000  # 2 seconds
+                    
+    def update_max_lines(self, value):
+        """Update maximum lines to display."""
+        self.max_lines = value
+        self.load_selected_log()  # Reload with new limit
+            
     def load_selected_log(self):
         """Load the currently selected log file."""
         selected_log = self.log_selector.currentText()
@@ -100,36 +139,123 @@ class LogViewer(QWidget):
             self.load_log_file(self.log_files[selected_log])
             
     def load_log_file(self, file_path):
-        """Load content from a specific log file."""
+        """Load content from a specific log file with optimization."""
         try:
             log_path = Path(file_path)
-            if log_path.exists():
-                with open(log_path, 'r', encoding='utf-8') as f:
-                    content = f.read()
-                self.log_display.setPlainText(content)
-                
-                # Scroll to bottom
-                cursor = self.log_display.textCursor()
-                cursor.movePosition(QTextCursor.End)
-                self.log_display.setTextCursor(cursor)
-                
-                # Update status
-                file_size = log_path.stat().st_size
-                self.status_label.setText(f"Loaded {log_path.name} ({file_size} bytes)")
-            else:
+            if not log_path.exists():
                 self.log_display.setPlainText(f"Log file not found: {file_path}")
                 self.status_label.setText("Log file not found")
+                return
+                
+            # Check if file has been modified since last read
+            current_mod_time = log_path.stat().st_mtime
+            last_mod_time = self.file_mod_times.get(file_path, 0)
+            
+            if current_mod_time <= last_mod_time and file_path in self.file_mod_times:
+                # File hasn't changed, no need to reload
+                return
+                
+            # Update modification time
+            self.file_mod_times[file_path] = current_mod_time
+            
+            # Read file efficiently
+            file_size = log_path.stat().st_size
+            
+            if file_size > 1000000:  # > 1MB, read only tail
+                content = self.read_file_tail(log_path, self.max_lines)
+            else:
+                # For smaller files, read normally but limit lines
+                with open(log_path, 'r', encoding='utf-8', errors='ignore') as f:
+                    content = f.read()
+                    
+                # Limit to max lines if content is too long
+                lines = content.splitlines()
+                if len(lines) > self.max_lines:
+                    content = '\n'.join(lines[-self.max_lines:])
+                    content = f"... (showing last {self.max_lines} lines) ...\n" + content
+                    
+            self.log_display.setPlainText(content)
+            
+            # Scroll to bottom
+            cursor = self.log_display.textCursor()
+            cursor.movePosition(QTextCursor.End)
+            self.log_display.setTextCursor(cursor)
+            
+            # Update status
+            lines_count = len(content.splitlines())
+            self.status_label.setText(f"Loaded {log_path.name} ({file_size:,} bytes, {lines_count:,} lines)")
+            
         except Exception as e:
             self.log_display.setPlainText(f"Error loading log file: {str(e)}")
             self.status_label.setText(f"Error: {str(e)}")
             
+    def read_file_tail(self, file_path, max_lines):
+        """Efficiently read the last N lines of a large file."""
+        try:
+            with open(file_path, 'rb') as f:
+                # Start from end of file
+                f.seek(0, 2)  # Seek to end
+                file_size = f.tell()
+                
+                # Read chunks from end until we have enough lines
+                lines = []
+                chunk_size = min(8192, file_size)  # 8KB chunks
+                pos = file_size
+                
+                while len(lines) < max_lines and pos > 0:
+                    # Calculate chunk position
+                    chunk_start = max(0, pos - chunk_size)
+                    f.seek(chunk_start)
+                    
+                    # Read chunk
+                    chunk = f.read(pos - chunk_start).decode('utf-8', errors='ignore')
+                    chunk_lines = chunk.splitlines()
+                    
+                    # Prepend to lines list
+                    if chunk_lines:
+                        if lines and not chunk.endswith('\n'):
+                            # Merge partial line at end of chunk with first existing line
+                            chunk_lines[-1] += lines[0]
+                            lines = chunk_lines + lines[1:]
+                        else:
+                            lines = chunk_lines + lines
+                            
+                    pos = chunk_start
+                    
+                # Return last max_lines
+                if len(lines) > max_lines:
+                    lines = lines[-max_lines:]
+                    return f"... (showing last {max_lines} lines) ...\n" + '\n'.join(lines)
+                else:
+                    return '\n'.join(lines)
+                    
+        except Exception as e:
+            return f"Error reading file tail: {str(e)}"
+            
     def refresh_logs(self):
-        """Refresh the current log display."""
+        """Refresh the current log display - only if tab is visible and file changed."""
+        # Only refresh if widget is visible (optimization for when tab is not active)
+        if not self.isVisible():
+            return
+            
         self.load_selected_log()
         
+        # Readjust refresh interval if auto-refresh is enabled
+        if self.auto_refresh_cb.isChecked():
+            self.adjust_refresh_interval()
+            if self.auto_refresh_timer.interval() != self.refresh_interval:
+                self.auto_refresh_timer.stop()
+                self.auto_refresh_timer.start(self.refresh_interval)
+        
     def clear_log(self):
-        """Clear the log display."""
+        """Clear the log display and reset tracking."""
         self.log_display.clear()
+        # Reset modification time tracking to force reload on next refresh
+        selected_log = self.log_selector.currentText()
+        if selected_log in self.log_files:
+            file_path = self.log_files[selected_log]
+            if file_path in self.file_mod_times:
+                del self.file_mod_times[file_path]
         self.status_label.setText("Log cleared")
         
     def append_log_message(self, message):
@@ -147,3 +273,15 @@ class LogViewer(QWidget):
     def get_current_log_content(self):
         """Get the current log content."""
         return self.log_display.toPlainText()
+        
+    def showEvent(self, event):
+        """Handle widget show event - resume refresh if auto-refresh is enabled."""
+        super().showEvent(event)
+        if self.auto_refresh_cb.isChecked() and not self.auto_refresh_timer.isActive():
+            self.auto_refresh_timer.start(self.refresh_interval)
+            
+    def hideEvent(self, event):
+        """Handle widget hide event - pause refresh to save resources."""
+        super().hideEvent(event)
+        # Note: Don't stop timer completely as other parts may depend on it
+        # The refresh_logs method now checks visibility anyway

gui/translation_tab.py

@@ -20,7 +20,7 @@ from PyQt5.QtWidgets import (
     QTextEdit, QMessageBox, QListWidget, QListWidgetItem, 
     QSplitter, QFileDialog, QComboBox, QCheckBox, QProgressBar
 )
-from PyQt5.QtCore import Qt, QTimer, pyqtSignal, QThread
+from PyQt5.QtCore import Qt, QTimer, pyqtSignal, QThread, QMutex
 from PyQt5.QtGui import QFont
 
 
@@ -37,10 +37,23 @@ class TranslationWorker(QThread):
         self.module_info = module_info  # [name, extensions, handler_function]
         self.estimate_only = estimate_only
         self.should_stop = False
+        self.mutex = QMutex()  # For thread safety
         
     def stop(self):
         """Stop the translation process."""
-        self.should_stop = True
+        self.mutex.lock()
+        try:
+            self.should_stop = True
+        finally:
+            self.mutex.unlock()
+        
+    def emit_log(self, message):
+        """Thread-safe log emission."""
+        self.log_signal.emit(message)
+        
+    def emit_progress(self, current, total, filename):
+        """Thread-safe progress emission."""
+        self.progress_signal.emit(current, total, filename)
         
     def run(self):
         """Run the translation process."""
@@ -54,18 +67,18 @@ class TranslationWorker(QThread):
             
             for env in required_envs:
                 if os.getenv(env) is None or str(os.getenv(env))[:1] == "<":
-                    self.log_signal.emit(f"❌ Environment variable {env} is not set!")
+                    self.emit_log(f"❌ Environment variable {env} is not set!")
                     env_missing = True
                     
             if env_missing:
-                self.log_signal.emit("❌ Some required environment variables are not set. Check your .env file.")
+                self.emit_log("❌ Some required environment variables are not set. Check your .env file.")
                 self.finished_signal.emit(False, "Environment variables missing")
                 return
                 
             # Get files to process
             files_dir = self.project_root / "files"
             if not files_dir.exists():
-                self.log_signal.emit("❌ Files directory does not exist!")
+                self.emit_log("❌ Files directory does not exist!")
                 self.finished_signal.emit(False, "Files directory missing")
                 return
                 
@@ -79,17 +92,17 @@ class TranslationWorker(QThread):
                             break
                             
             if not matching_files:
-                self.log_signal.emit(f"❌ No files found matching extensions: {', '.join(self.module_info[1])}")
+                self.emit_log(f"❌ No files found matching extensions: {', '.join(self.module_info[1])}")
                 self.finished_signal.emit(False, "No matching files")
                 return
                 
-            self.log_signal.emit(f"📁 Found {len(matching_files)} files to process:")
+            self.emit_log(f"📁 Found {len(matching_files)} files to process:")
             for filename in matching_files:
-                self.log_signal.emit(f"   • {filename}")
+                self.emit_log(f"   • {filename}")
                 
-            self.log_signal.emit(f"🔧 Using module: {self.module_info[0]}")
-            self.log_signal.emit(f"📊 Estimate only: {'Yes' if self.estimate_only else 'No'}")
-            self.log_signal.emit("")
+            self.emit_log(f"🔧 Using module: {self.module_info[0]}")
+            self.emit_log(f"📊 Estimate only: {'Yes' if self.estimate_only else 'No'}")
+            self.emit_log("")
             
             # Process files
             threads = int(os.getenv("fileThreads", "1"))
@@ -112,23 +125,24 @@ class TranslationWorker(QThread):
                     
                     for future in as_completed(future_to_filename):
                         if self.should_stop:
-                            self.log_signal.emit("🛑 Translation stopped by user")
+                            self.emit_log("🛑 Translation stopped by user")
                             break
                             
                         filename = future_to_filename[future]
                         completed_count += 1
                         
-                        # Emit progress signal
-                        self.progress_signal.emit(completed_count, total_files, filename)
+                        # Emit progress signal (less frequent updates)
+                        self.emit_progress(completed_count, total_files, filename)
                         
                         try:
                             result = future.result()
                             total_cost = result
-                            self.log_signal.emit(f"✅ Completed {filename} ({completed_count}/{total_files})")
+                            # Only log every file completion, not internal progress
+                            self.emit_log(f"✅ Completed {filename} ({completed_count}/{total_files})")
                         except Exception as e:
                             tb_line = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
                             error_msg = f"❌ Error processing {filename}: {str(e)} | Line: {tb_line}"
-                            self.log_signal.emit(error_msg)
+                            self.emit_log(error_msg)
                             
             finally:
                 os.chdir(old_cwd)
@@ -140,21 +154,21 @@ class TranslationWorker(QThread):
                 
             # Report results
             if total_cost != "Fail" and not self.should_stop:
-                self.log_signal.emit("")
-                self.log_signal.emit(f"💰 {total_cost}")
+                self.emit_log("")
+                self.emit_log(f"💰 {total_cost}")
                 if not self.estimate_only:
-                    self.log_signal.emit("✅ Translation completed successfully!")
+                    self.emit_log("✅ Translation completed successfully!")
                 else:
-                    self.log_signal.emit("✅ Estimation completed!")
+                    self.emit_log("✅ Estimation completed!")
                 self.finished_signal.emit(True, str(total_cost))
             else:
                 if not self.should_stop:
-                    self.log_signal.emit("❌ Translation failed!")
+                    self.emit_log("❌ Translation failed!")
                     self.finished_signal.emit(False, "Translation failed")
                     
         except Exception as e:
             error_msg = f"❌ Unexpected error: {str(e)}"
-            self.log_signal.emit(error_msg)
+            self.emit_log(error_msg)
             self.finished_signal.emit(False, error_msg)
 
 
@@ -165,6 +179,9 @@ class TranslationTab(QWidget):
         super().__init__(parent)
         self.parent_window = parent
         self.translation_process = None
+        self.log_buffer = []  # Buffer for batching log messages
+        self.log_timer = QTimer()  # Timer for flushing log buffer
+        self.log_timer.timeout.connect(self.flush_log_buffer)
         
         # Set up directories
         self.project_root = Path(__file__).parent.parent
@@ -178,7 +195,7 @@ class TranslationTab(QWidget):
         self.setup_ui()
         self.refresh_file_lists()
         
-        # Auto-refresh timer for file lists
+        # Auto-refresh timer for file lists (less frequent during translation)
         self.refresh_timer = QTimer()
         self.refresh_timer.timeout.connect(self.refresh_file_lists)
         self.refresh_timer.start(3000)  # Refresh every 3 seconds
@@ -634,19 +651,37 @@ class TranslationTab(QWidget):
             self.progress_bar.setValue(0)
             self.progress_label.setText("Starting...")
             
+            # Reduce file refresh frequency during translation
+            self.refresh_timer.stop()
+            
             # Start the worker
             self.translation_worker.start()
             
     def append_log(self, message):
-        """Append a message to the log display."""
-        self.log_display.append(message)
-        # Auto-scroll to bottom
-        scrollbar = self.log_display.verticalScrollBar()
-        scrollbar.setValue(scrollbar.maximum())
+        """Append a message to the log buffer for batched display."""
+        self.log_buffer.append(message)
+        
+        # Start timer if not already running
+        if not self.log_timer.isActive():
+            self.log_timer.start(100)  # Flush every 100ms
+            
+    def flush_log_buffer(self):
+        """Flush the log buffer to the display."""
+        if self.log_buffer:
+            # Add all buffered messages at once
+            for message in self.log_buffer:
+                self.log_display.append(message)
+            self.log_buffer.clear()
+            
+            # Auto-scroll to bottom
+            scrollbar = self.log_display.verticalScrollBar()
+            scrollbar.setValue(scrollbar.maximum())
+            
+        # Stop timer if buffer is empty
+        self.log_timer.stop()
         
     def update_progress(self, current_file, total_files, filename):
         """Update the progress bar and label."""
-        progress_percentage = int((current_file / total_files) * 100)
         self.progress_bar.setMaximum(total_files)
         self.progress_bar.setValue(current_file)
         self.progress_label.setText(f"Processing {filename} ({current_file}/{total_files})")
@@ -657,6 +692,12 @@ class TranslationTab(QWidget):
         self.stop_button.setEnabled(False)
         self.progress_bar.setVisible(False)
         self.progress_label.setText("Ready")
+        
+        # Flush any remaining log messages
+        self.flush_log_buffer()
+        
+        # Resume file refresh
+        self.refresh_timer.start(3000)
         self.refresh_file_lists()
         
         if success:
@@ -666,6 +707,9 @@ class TranslationTab(QWidget):
             self.append_log("")
             self.append_log(f"❌ Process failed: {message}")
             
+        # Force flush the final messages
+        self.flush_log_buffer()
+            
     def stop_translation(self):
         """Stop the translation process."""
         if hasattr(self, 'translation_worker') and self.translation_worker.isRunning():
@@ -678,14 +722,23 @@ class TranslationTab(QWidget):
         self.progress_bar.setVisible(False)
         self.progress_label.setText("Ready")
         
+        # Resume file refresh
+        self.refresh_timer.start(3000)
+        
+        # Flush any remaining messages
+        self.flush_log_buffer()
+        
     def clear_log(self):
         """Clear the log display."""
+        self.log_buffer.clear()  # Clear buffer too
         self.log_display.clear()
         
     def closeEvent(self, event):
         """Handle widget close event."""
         if hasattr(self, 'refresh_timer'):
             self.refresh_timer.stop()
+        if hasattr(self, 'log_timer'):
+            self.log_timer.stop()
         if hasattr(self, 'translation_worker') and self.translation_worker.isRunning():
             self.translation_worker.stop()
             self.translation_worker.wait(3000)