

import os
import sys
import pandas as pd
import numpy as np
import tempfile
import time
import gc
import pytest

# Add src directory to path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))

from excel_filter.filter import ExcelFilter


class TestLargeFileHandling:
    """
    Test cases for handling very large Excel files
    """

    def generate_large_test_data(self, num_rows=50000):
        """
        Generate a large dataset for testing
        """
        np.random.seed(42)  # For reproducible results

        # Create diverse data
        names = [f"User_{i}" for i in range(num_rows)]
        ages = np.random.randint(18, 80, num_rows)
        salaries = np.random.uniform(30000, 200000, num_rows).round(2)
        departments = np.random.choice(['HR', 'IT', 'Finance', 'Marketing', 'Sales', 'Operations'], num_rows)
        statuses = np.random.choice(['Active', 'Inactive', 'Pending', 'Terminated'], num_rows)
        cities = np.random.choice(['New York', 'London', 'Tokyo', 'Berlin', 'Paris', 'Sydney', 'Toronto', 'Singapore'], num_rows)

        # Create various message types
        messages = []
        for i in range(num_rows):
            if i % 100 == 0:
                messages.append("ERROR: Critical system failure detected")
            elif i % 50 == 0:
                messages.append("WARNING: Low disk space on server")
            elif i % 25 == 0:
                messages.append("INFO: User login successful")
            elif i % 10 == 0:
                messages.append("DEBUG: Processing completed successfully")
            else:
                messages.append(f"Normal operation log entry {i}")

        # Create some rows with specific patterns for testing
        special_indices = np.random.choice(num_rows, size=int(num_rows * 0.1), replace=False)
        for idx in special_indices[:len(special_indices)//3]:
            messages[idx] = "CRITICAL: Database connection lost"
        for idx in special_indices[len(special_indices)//3:2*len(special_indices)//3]:
            messages[idx] = "ALERT: Security breach detected"
        for idx in special_indices[2*len(special_indices)//3:]:
            messages[idx] = "NOTICE: System maintenance scheduled"

        # Create DataFrame
        # For very large datasets, limit the date range to avoid pandas datetime limits
        if num_rows > 50000:
            # For large datasets, use a repeating date range to avoid datetime overflow
            join_dates = []
            base_dates = pd.date_range('2010-01-01', periods=min(50000, num_rows), freq='D')
            for i in range(num_rows):
                join_dates.append(base_dates[i % len(base_dates)])
        else:
            join_dates = pd.date_range('2010-01-01', periods=num_rows, freq='D')

        df = pd.DataFrame({
            'Name': names,
            'Age': ages,
            'Salary': salaries,
            'Department': departments,
            'Status': statuses,
            'City': cities,
            'Message': messages,
            'Employee_ID': range(100000, 100000 + num_rows),
            'Join_Date': join_dates,
            'Performance_Score': np.random.uniform(1.0, 5.0, num_rows).round(1)
        })

        return df

    def test_large_file_generation_and_basic_filtering(self):
        """
        Test basic filtering on a large file (10k rows)
        """
        # Generate large test data
        df = self.generate_large_test_data(10000)

        with tempfile.TemporaryDirectory() as temp_dir:
            input_file = os.path.join(temp_dir, "large_test_data.xlsx")
            output_file = os.path.join(temp_dir, "filtered_output.xlsx")

            # Save to Excel
            start_time = time.time()
            df.to_excel(input_file, index=False)
            save_time = time.time() - start_time
            print(f"Generated and saved {len(df)} rows in {save_time:.2f} seconds")

            # Test filtering for ERROR messages
            pattern = r"ERROR|CRITICAL"
            excel_filter = ExcelFilter(
                input_file=input_file,
                output_file=output_file,
                pattern=pattern,
                columns=['Message']
            )

            # Process and measure time
            start_time = time.time()
            success = excel_filter.process()
            process_time = time.time() - start_time

            assert success, "Filtering should succeed"

            # Verify results
            result_df = pd.read_excel(output_file)
            expected_count = len(df[df['Message'].str.contains(pattern, case=False, na=False)])

            print(f"Filtered {len(df)} rows to {len(result_df)} rows in {process_time:.2f} seconds")
            print(f"Expected {expected_count} matches")

            assert len(result_df) == expected_count, f"Expected {expected_count} rows, got {len(result_df)}"

            # Verify all results contain the pattern
            import re
            for msg in result_df['Message']:
                assert re.search(pattern, str(msg), re.IGNORECASE), f"Message '{msg}' should match pattern '{pattern}'"

    def test_large_file_numeric_filtering(self):
        """
        Test numeric filtering on large files
        """
        df = self.generate_large_test_data(30000)

        with tempfile.TemporaryDirectory() as temp_dir:
            input_file = os.path.join(temp_dir, "large_numeric_test.xlsx")
            output_file = os.path.join(temp_dir, "numeric_filtered.xlsx")

            df.to_excel(input_file, index=False)

            # Filter for high salaries (> 150000)
            numeric_filter = {'column': 'Salary', 'operator': '>', 'value': 150000}

            excel_filter = ExcelFilter(
                input_file=input_file,
                output_file=output_file,
                numeric_filter=numeric_filter
            )

            success = excel_filter.process()
            assert success

            result_df = pd.read_excel(output_file)
            expected_count = len(df[df['Salary'] > 150000])

            assert len(result_df) == expected_count
            assert all(result_df['Salary'] > 150000)

    def test_large_file_combined_filtering(self):
        """
        Test combined regex and numeric filtering on large files
        """
        df = self.generate_large_test_data(40000)

        with tempfile.TemporaryDirectory() as temp_dir:
            input_file = os.path.join(temp_dir, "combined_test.xlsx")
            output_file = os.path.join(temp_dir, "combined_filtered.xlsx")

            df.to_excel(input_file, index=False)

            # Filter: IT department employees with salary > 100000
            pattern = r"IT"
            numeric_filter = {'column': 'Salary', 'operator': '>', 'value': 100000}

            excel_filter = ExcelFilter(
                input_file=input_file,
                output_file=output_file,
                pattern=pattern,
                columns=['Department', 'Salary'],
                numeric_filter=numeric_filter
            )

            success = excel_filter.process()
            assert success

            result_df = pd.read_excel(output_file)
            expected_df = df[
                (df['Department'].str.contains(pattern, case=False, na=False)) &
                (df['Salary'] > 100000)
            ]

            assert len(result_df) == len(expected_df)

            # Verify all results meet both criteria
            assert all(result_df['Department'].str.contains(pattern, case=False, na=False))
            assert all(result_df['Salary'] > 100000)

    def test_memory_efficiency_large_file(self):
        """
        Test memory efficiency with very large files (100k+ rows)
        """
        # Skip this test if we're in a memory-constrained environment
        try:
            df = self.generate_large_test_data(100000)
        except MemoryError:
            pytest.skip("Not enough memory for 100k row test")

        with tempfile.TemporaryDirectory() as temp_dir:
            input_file = os.path.join(temp_dir, "memory_test.xlsx")
            output_file = os.path.join(temp_dir, "memory_filtered.xlsx")

            df.to_excel(input_file, index=False)

            # Simple filter that should match ~10% of rows
            pattern = r"ERROR|WARNING|CRITICAL|ALERT"

            excel_filter = ExcelFilter(
                input_file=input_file,
                output_file=output_file,
                pattern=pattern,
                columns=['Message']
            )

            # Monitor memory usage if possible
            try:
                import psutil
                process = psutil.Process(os.getpid())
                initial_memory = process.memory_info().rss / 1024 / 1024  # MB

                success = excel_filter.process()

                final_memory = process.memory_info().rss / 1024 / 1024  # MB
                memory_used = final_memory - initial_memory

                assert success
                print(f"Memory used: {memory_used:.1f}MB")

                # Should not use excessive memory (arbitrary threshold)
                assert memory_used < 2000, f"Used {memory_used:.1f}MB, should be less than 2000MB"
            except ImportError:
                # psutil not available, just run the filter without memory monitoring
                print("psutil not available, skipping memory monitoring")
                success = excel_filter.process()
                assert success

            result_df = pd.read_excel(output_file)
            expected_count = len(df[df['Message'].str.contains(pattern, case=False, na=False)])

            assert len(result_df) == expected_count

    def test_edge_cases_large_file(self):
        """
        Test edge cases with large files
        """
        df = self.generate_large_test_data(25000)

        # Add some edge case data
        df.loc[0, 'Message'] = ""  # Empty string
        df.loc[1, 'Message'] = None  # NaN value
        df.loc[2, 'Salary'] = np.nan  # NaN numeric
        df.loc[3, 'Message'] = "A" * 10000  # Very long string

        with tempfile.TemporaryDirectory() as temp_dir:
            input_file = os.path.join(temp_dir, "edge_case_test.xlsx")
            output_file = os.path.join(temp_dir, "edge_case_filtered.xlsx")

            df.to_excel(input_file, index=False)

            # Filter that should handle edge cases gracefully
            pattern = r"ERROR"

            excel_filter = ExcelFilter(
                input_file=input_file,
                output_file=output_file,
                pattern=pattern,
                columns=['Message']
            )

            success = excel_filter.process()
            assert success

            result_df = pd.read_excel(output_file)

            # Should find ERROR messages but handle edge cases
            error_messages = df[df['Message'].str.contains(pattern, case=False, na=False)]
            assert len(result_df) == len(error_messages)

    @pytest.mark.slow
    def test_very_large_file_stress_test(self):
        """
        Stress test with a very large file (marked as slow test)
        """
        # This test is marked as slow and may be skipped in regular runs
        try:
            df = self.generate_large_test_data(200000)  # 200k rows
        except MemoryError:
            pytest.skip("Not enough memory for 200k row stress test")

        with tempfile.TemporaryDirectory() as temp_dir:
            input_file = os.path.join(temp_dir, "stress_test.xlsx")
            output_file = os.path.join(temp_dir, "stress_filtered.xlsx")

            # Time the save operation
            start_time = time.time()
            df.to_excel(input_file, index=False)
            save_time = time.time() - start_time

            file_size = os.path.getsize(input_file) / 1024 / 1024  # MB
            print(f"Created {file_size:.1f}MB test file with {len(df)} rows in {save_time:.2f} seconds")

            # Apply complex filtering
            pattern = r"ERROR|WARNING|CRITICAL"
            numeric_filter = {'column': 'Age', 'operator': '>', 'value': 50}

            excel_filter = ExcelFilter(
                input_file=input_file,
                output_file=output_file,
                pattern=pattern,
                columns=['Message', 'Age'],
                numeric_filter=numeric_filter
            )

            start_time = time.time()
            success = excel_filter.process()
            process_time = time.time() - start_time

            assert success

            result_df = pd.read_excel(output_file)
            expected_df = df[
                (df['Message'].str.contains(pattern, case=False, na=False)) &
                (df['Age'] > 50)
            ]

            print(f"Processed {len(df)} rows in {process_time:.2f} seconds")
            print(f"Filtered to {len(result_df)} rows (expected {len(expected_df)})")

            assert len(result_df) == len(expected_df)

            # Verify results are correct
            assert all(result_df['Message'].str.contains(pattern, case=False, na=False))
            assert all(result_df['Age'] > 50)

            # Cleanup
            gc.collect()
