update

app.py

@@ -3,22 +3,9 @@ import gradio as gr
 import pandas as pd
 from datetime import datetime
 import time
-import sys
-import importlib.util
-import threading
+from azure_count_ip_data import count_files_per_annotator
 from log_reader import RemoteLogReader
-
-# Define the path for the azure_count_ip_data module
-azure_count_ip_data_path = os.path.join(os.path.dirname(__file__), 'azure_count_ip_data.py')
-
-# Import the module dynamically
-spec = importlib.util.spec_from_file_location("azure_count_ip_data", azure_count_ip_data_path)
-azure_count_ip_data = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(azure_count_ip_data)
-
-# Get the functions we need
-count_files_per_annotator = azure_count_ip_data.count_files_per_annotator
-count_deduplicated_files_per_annotator = azure_count_ip_data.count_deduplicated_files_per_annotator
+import threading
 
 # Define the path for storing the data
 DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
@@ -42,61 +29,18 @@ def load_stats():
             last_update = f.read().strip()
         return df, last_update
     except (FileNotFoundError, pd.errors.EmptyDataError):
-        return pd.DataFrame(columns=['Annotator', 'Total Count', 'Unique Count', 'Unique %']), ""
+        return pd.DataFrame(columns=['Annotator', 'Battle Count']), ""
 
 def update_stats():
-    """Get the latest battle statistics with both total and deduplicated counts"""
+    """Get the latest battle statistics"""
     try:
         # Initialize RemoteLogReader
         reader = RemoteLogReader()
-        
-        # Get total annotator counts
-        total_counts = count_files_per_annotator(reader)
-        
-        # Get deduplicated annotator counts
-        unique_counts = count_deduplicated_files_per_annotator(reader)
-        
-        # Combine the data into a single DataFrame
-        data = []
-        all_annotators = set(total_counts.keys()) | set(unique_counts.keys())
-        
-        # Track totals for summary row
-        total_sum = 0
-        unique_sum = 0
-        
-        for annotator in all_annotators:
-            total = total_counts.get(annotator, 0)
-            unique = unique_counts.get(annotator, 0)
-            # Calculate percentage of unique prompts
-            percentage = round((unique / total * 100) if total > 0 else 0, 1)
-            
-            # Add to totals
-            total_sum += total
-            unique_sum += unique
-            
-            data.append({
-                'Annotator': annotator,
-                'Total Count': total,
-                'Unique Count': unique,
-                'Unique %': f"{percentage}%"
-            })
-        
-        # Add summary row
-        overall_percentage = round((unique_sum / total_sum * 100) if total_sum > 0 else 0, 1)
-        data.append({
-            'Annotator': 'TOTAL',
-            'Total Count': total_sum,
-            'Unique Count': unique_sum,
-            'Unique %': f"{overall_percentage}%"
-        })
-        
-        # Convert to DataFrame and sort by total count, keeping TOTAL at the bottom
-        df = pd.DataFrame(data)
-        # Move TOTAL row to the end
-        df = pd.concat([
-            df[df['Annotator'] != 'TOTAL'].sort_values('Total Count', ascending=False),
-            df[df['Annotator'] == 'TOTAL']
-        ]).reset_index(drop=True)
+        # Get annotator counts using Azure storage
+        annotator_counts = count_files_per_annotator(reader)
+        # Convert to DataFrame for better display
+        df = pd.DataFrame(list(annotator_counts.items()), columns=['Annotator', 'Battle Count'])
+        df = df.sort_values('Battle Count', ascending=False)
         
         # Get current time
         current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -107,7 +51,7 @@ def update_stats():
         return df, current_time
     except Exception as e:
         print(f"Error updating stats: {e}")
-        return pd.DataFrame(columns=['Annotator', 'Total Count', 'Unique Count', 'Unique %']), ""
+        return pd.DataFrame(columns=['Annotator', 'Battle Count']), ""
 
 def auto_update(state):
     """Background task to update stats every hour"""
@@ -142,7 +86,7 @@ def create_ui():
     
     with gr.Blocks(title="Battle Count Statistics") as app:
         gr.Markdown("# Battle Count Statistics")
-        gr.Markdown("Displays the count of battles per annotator. 'Total Count' shows all valid battles, while 'Unique Count' shows deduplicated battles based on the first user prompt. Only conversations that pass the vote conditions are counted, and example prompts are excluded from the unique count.")
+        gr.Markdown("Displays the count of valid battles per annotator. Updates automatically every hour.")
         
         with gr.Row():
             last_update = gr.Textbox(
@@ -156,12 +100,10 @@ def create_ui():
                 value=get_current_stats,
                 interactive=False,
                 wrap=True,
-                column_widths=["40%", "20%", "20%", "20%"],
-                height=600
             )
         
-        refresh_btn = gr.Button("Refresh Now")
-        refresh_btn.click(fn=manual_refresh, outputs=[output, last_update])
+        # refresh_btn = gr.Button("Refresh Now")
+        # refresh_btn.click(fn=manual_refresh, outputs=[output, last_update])
     
     return app
 

azure_count_ip_data.py

@@ -8,7 +8,6 @@ import re
 import argparse
 from typing import Dict, Set, Tuple, Optional, List, Union
 from log_reader import RemoteLogReader
-import csv
 
 # List of IP addresses we care about
 WHITELIST_IPS_DICT = {
@@ -61,47 +60,42 @@ WHITELIST_IPS = [ip for ips in WHITELIST_IPS_DICT.values() for ip in ips]
 # Flatten username list for backward compatibility
 WHITELIST_USERNAMES = [username for usernames in WHITELIST_USERNAMES_DICT.values() for username in usernames]
 
+EXAMPLES = [
+            ["使用SVG绘制春节主题的动态图案，包括：1）一个红色的灯笼，带有金色的流苏 2）一个金色的福字，使用书法字体 3）背景添加一些烟花效果 4）在灯笼和福字周围添加一些祥云图案。确保图案布局美观，颜色搭配符合春节传统风格。"],
+            ["SVGを使用して日本の伝統的な和柄パターンを描画してください。1）波紋（さざなみ）模様 2）市松模様 3）麻の葉模様 4）雷文（らいもん）模様を含めてください。色は伝統的な日本の色（藍色、朱色、金色など）を使用し、レイアウトはバランスよく配置してください。"],
+            ["Write HTML with P5.js that simulates 25 particles in a vacuum space of a cylindrical container, bouncing within its boundaries. Use different colors for each ball and ensure they leave a trail showing their movement. Add a slow rotation of the container to give better view of what's going on in the scene. Make sure to create proper collision detection and physic rules to ensure particles remain in the container. Add an external spherical container. Add a slow zoom in and zoom out effect to the whole scene."],
+            ["Write a Python script to scrape NVIDIA's stock price for the past month using the yfinance library. Clean the data and create an interactive visualization using Matplotlib. Include: 1) A candlestick chart showing daily price movements 2) A line chart with 7-day and 30-day moving averages. Add hover tooltips showing exact values and date. Make the layout professional with proper titles and axis labels."],
+            ["Write a Python script that uses the Gradio library to create a functional calculator. The calculator should support basic arithmetic operations: addition, subtraction, multiplication, and division. It should have two input fields for numbers and a dropdown menu to select the operation."],
+            ["Write a Todo list app using React.js. The app should allow users to add, delete, and mark tasks as completed. Include features like filtering tasks by status (completed, active), sorting tasks by priority, and displaying the total number of tasks."],
+            ["Write a Python script using the Streamlit library to create a web application for uploading and displaying files. The app should allow users to upload files of type .csv or .txt. If a .csv file is uploaded, display its contents as a table using Streamlit's st.dataframe() method. If a .txt file is uploaded, display its content as plain text."],
+            ["Write a Python function to solve the Trapping Rain Water problem. The function should take a list of non-negative integers representing the height of bars in a histogram and return the total amount of water trapped between the bars after raining. Use an efficient algorithm with a time complexity of O(n)."],
+            ["Create a simple Pygame script for a game where the player controls a bouncing ball that changes direction when it collides with the edges of the window. Add functionality for the player to control a paddle using arrow keys, aiming to keep the ball from touching the bottom of the screen. Include basic collision detection and a scoring system that increases as the ball bounces off the paddle. You need to add clickable buttons to start the game, and reset the game."],
+            ["Create a financial management Dashboard using Vue.js, focusing on local data handling without APIs. Include features like a clean dashboard for tracking income and expenses, dynamic charts for visualizing finances, and a budget planner. Implement functionalities for adding, editing, and deleting transactions, as well as filtering by date or category. Ensure responsive design and smooth user interaction for an intuitive experience."],
+            ["Create a Mermaid diagram to visualize a flowchart of a user login process. Include the following steps: User enters login credentials; Credentials are validated; If valid, the user is directed to the dashboard; If invalid, an error message is shown, and the user can retry or reset the password."],
+            ["Write a Python function to calculate the Fibonacci sequence up to n numbers. Then write test cases to verify the function works correctly for edge cases like negative numbers, zero, and large inputs."],
+            ["Build an HTML page for a Kanban board with three columns with Vue.js: To Do, In Progress, and Done. Each column should allow adding, moving, and deleting tasks. Implement drag-and-drop functionality using Vue Draggable and persist the state using Vuex."],
+            ["Develop a Streamlit app that takes a CSV file as input and provides: 1) Basic statistics about the data 2) Interactive visualizations using Plotly 3) A data cleaning interface with options to handle missing values 4) An option to download the cleaned data."],
+            ["Write an HTML page with embedded JavaScript that creates an interactive periodic table. Each element should display its properties on hover and allow filtering by category (metals, non-metals, etc.). Include a search bar to find elements by name or symbol."],
+            ["Here's a Python function that sorts a list of dictionaries by a specified key:\n\n```python\ndef sort_dicts(data, key):\n    return sorted(data, key=lambda x: x[key])\n```\n\nWrite test cases to verify the function works correctly for edge cases like empty lists, missing keys, and different data types. If you use unittest, please use `unittest.main(argv=['first-arg-is-ignored'], exit=False)` to run the tests."],
+            ["Create a React component for a fitness tracker that shows: 1) Daily step count 2) Calories burned 3) Distance walked 4) A progress bar for daily goals."],
+            ["Build a Vue.js dashboard for monitoring server health. Include: 1) Real-time CPU and memory usage graphs 2) Disk space visualization 3) Network activity monitor 4) Alerts for critical thresholds."],
+            ["Write a C program that calculates and prints the first 100 prime numbers in a formatted table with 10 numbers per row. Include a function to check if a number is prime and use it in your solution."],
+            ["Write a C++ program that implements a simple calculator using object-oriented programming. Create a Calculator class with methods for addition, subtraction, multiplication, and division. Include error handling for division by zero."],
+            ["Write a Rust program that generates and prints a Pascal's Triangle with 10 rows. Format the output to center-align the numbers in each row."],
+            ["Write a Java program that simulates a simple bank account system. Create a BankAccount class with methods for deposit, withdrawal, and balance inquiry. Include error handling for insufficient funds and demonstrate its usage with a few transactions."],
+            ["Write a Go program that calculates and prints the Fibonacci sequence up to the 50th number. Format the output in a table with 5 numbers per row and include the index of each Fibonacci number."],
+            ["Write a C program that calculates and prints a histogram of letter frequencies from a predefined string. Use ASCII art to display the histogram vertically."],
+            ["Write a C++ program that implements a simple stack data structure with push, pop, and peek operations. Demonstrate its usage by reversing a predefined string using the stack."],
+            ["Write a Rust program that calculates and prints the first 20 happy numbers. Include a function to check if a number is happy and use it in your solution."],
+            ["Write a Java program that implements a simple binary search algorithm. Create a sorted array of integers and demonstrate searching for different values, including cases where the value is found and not found."],
+            ["Write a Go program that generates and prints a multiplication table from 1 to 12. Format the output in a neat grid with proper alignment."],
+]
+
+EXAMPLES = [e[0] for e in EXAMPLES]
+
 logging.basicConfig(level=logging.WARNING)
 log = logging.getLogger(__name__)
 
-# List of example prompts to exclude from counting
-EXAMPLE_PROMPTS = [
-    "使用SVG绘制春节主题的动态图案，包括：1）一个红色的灯笼，带有金色的流苏 2）一个金色的福字，使用书法字体 3）背景添加一些烟花效果 4）在灯笼和福字周围添加一些祥云图案。确保图案布局美观，颜色搭配符合春节传统风格。",
-    "SVGを使用して日本の伝統的な和柄パターンを描画してください。1）波紋（さざなみ）模様 2）市松模様 3）麻の葉模様 4）雷文（らいもん）模様を含めてください。色は伝統的な日本の色（藍色、朱色、金色など）を使用し、レイアウトはバランスよく配置してください。",
-    "Write HTML with P5.js that simulates 25 particles in a vacuum space of a cylindrical container, bouncing within its boundaries. Use different colors for each ball and ensure they leave a trail showing their movement. Add a slow rotation of the container to give better view of what's going on in the scene. Make sure to create proper collision detection and physic rules to ensure particles remain in the container. Add an external spherical container. Add a slow zoom in and zoom out effect to the whole scene.",
-    "Write a Python script to scrape NVIDIA's stock price for the past month using the yfinance library. Clean the data and create an interactive visualization using Matplotlib. Include: 1) A candlestick chart showing daily price movements 2) A line chart with 7-day and 30-day moving averages. Add hover tooltips showing exact values and date. Make the layout professional with proper titles and axis labels.",
-    "Write a Python script that uses the Gradio library to create a functional calculator. The calculator should support basic arithmetic operations: addition, subtraction, multiplication, and division. It should have two input fields for numbers and a dropdown menu to select the operation.",
-    "Write a Todo list app using React.js. The app should allow users to add, delete, and mark tasks as completed. Include features like filtering tasks by status (completed, active), sorting tasks by priority, and displaying the total number of tasks.",
-    "Write a Python script using the Streamlit library to create a web application for uploading and displaying files. The app should allow users to upload files of type .csv or .txt. If a .csv file is uploaded, display its contents as a table using Streamlit's st.dataframe() method. If a .txt file is uploaded, display its content as plain text.",
-    "Write a Python function to solve the Trapping Rain Water problem. The function should take a list of non-negative integers representing the height of bars in a histogram and return the total amount of water trapped between the bars after raining. Use an efficient algorithm with a time complexity of O(n).",
-    "Create a simple Pygame script for a game where the player controls a bouncing ball that changes direction when it collides with the edges of the window. Add functionality for the player to control a paddle using arrow keys, aiming to keep the ball from touching the bottom of the screen. Include basic collision detection and a scoring system that increases as the ball bounces off the paddle. You need to add clickable buttons to start the game, and reset the game.",
-    "Create a financial management Dashboard using Vue.js, focusing on local data handling without APIs. Include features like a clean dashboard for tracking income and expenses, dynamic charts for visualizing finances, and a budget planner. Implement functionalities for adding, editing, and deleting transactions, as well as filtering by date or category. Ensure responsive design and smooth user interaction for an intuitive experience.",
-    "Create a Mermaid diagram to visualize a flowchart of a user login process. Include the following steps: User enters login credentials; Credentials are validated; If valid, the user is directed to the dashboard; If invalid, an error message is shown, and the user can retry or reset the password.",
-    "Write a Python function to calculate the Fibonacci sequence up to n numbers. Then write test cases to verify the function works correctly for edge cases like negative numbers, zero, and large inputs.",
-    "Build an HTML page for a Kanban board with three columns with Vue.js: To Do, In Progress, and Done. Each column should allow adding, moving, and deleting tasks. Implement drag-and-drop functionality using Vue Draggable and persist the state using Vuex.",
-    "Develop a Streamlit app that takes a CSV file as input and provides: 1) Basic statistics about the data 2) Interactive visualizations using Plotly 3) A data cleaning interface with options to handle missing values 4) An option to download the cleaned data.",
-    "Write an HTML page with embedded JavaScript that creates an interactive periodic table. Each element should display its properties on hover and allow filtering by category (metals, non-metals, etc.). Include a search bar to find elements by name or symbol.",
-    "Here's a Python function that sorts a list of dictionaries by a specified key:\n\n```python\ndef sort_dicts(data, key):\n    return sorted(data, key=lambda x: x[key])\n```\n\nWrite test cases to verify the function works correctly for edge cases like empty lists, missing keys, and different data types. If you use unittest, please use `unittest.main(argv=['first-arg-is-ignored'], exit=False)` to run the tests.",
-    "Create a React component for a fitness tracker that shows: 1) Daily step count 2) Calories burned 3) Distance walked 4) A progress bar for daily goals.",
-    "Build a Vue.js dashboard for monitoring server health. Include: 1) Real-time CPU and memory usage graphs 2) Disk space visualization 3) Network activity monitor 4) Alerts for critical thresholds.",
-    "Write a C program that calculates and prints the first 100 prime numbers in a formatted table with 10 numbers per row. Include a function to check if a number is prime and use it in your solution.",
-    "Write a C++ program that implements a simple calculator using object-oriented programming. Create a Calculator class with methods for addition, subtraction, multiplication, and division. Include error handling for division by zero.",
-    "Write a Rust program that generates and prints a Pascal's Triangle with 10 rows. Format the output to center-align the numbers in each row.",
-    "Write a Java program that simulates a simple bank account system. Create a BankAccount class with methods for deposit, withdrawal, and balance inquiry. Include error handling for insufficient funds and demonstrate its usage with a few transactions.",
-    "Write a Go program that calculates and prints the Fibonacci sequence up to the 50th number. Format the output in a table with 5 numbers per row and include the index of each Fibonacci number.",
-    "Write a C program that calculates and prints a histogram of letter frequencies from a predefined string. Use ASCII art to display the histogram vertically.",
-    "Write a C++ program that implements a simple stack data structure with push, pop, and peek operations. Demonstrate its usage by reversing a predefined string using the stack.",
-    "Write a Rust program that calculates and prints the first 20 happy numbers. Include a function to check if a number is happy and use it in your solution.",
-    "Write a Java program that implements a simple binary search algorithm. Create a sorted array of integers and demonstrate searching for different values, including cases where the value is found and not found.",
-    "Write a Go program that generates and prints a multiplication table from 1 to 12. Format the output in a neat grid with proper alignment.",
-    "hello",
-    "Hello",
-    "hi",
-    "Hi",
-    "test",
-    "Test"
-]
-
 def get_ip_from_jsonl(content: str) -> Optional[str]:
     """Extract IP from the first line of a JSONL content"""
     try:
@@ -166,8 +160,9 @@ def get_file_data(content: str) -> Tuple[Optional[str], Optional[str], bool]:
     """Read file content and return IP, username, and vote condition status"""
     try:
         lines = [line.strip() for line in content.split('\n') if line.strip()]
+        user_prompt = lines[0]["messages"][0][1]
         if not lines:
-            return None, None, False
+            return None, None, False, user_prompt
         
         # Get IP from first line
         try:
@@ -221,12 +216,12 @@ def get_file_data(content: str) -> Tuple[Optional[str], Optional[str], bool]:
             
         # Early return if neither IP nor username is in whitelist
         if not (ip_in_whitelist or username_in_whitelist):
-            return ip, username, False
+            return ip, username, False, user_prompt
             
         return ip, username, vote_conditions_met
     except Exception as e:
         log.error(f"Error processing file content: {e}")
-        return None, None, False
+        return None, None, False, user_prompt
 
 def get_annotator_name(ip: Optional[str], username: Optional[str]) -> Optional[str]:
     """Get annotator name from IP or username"""
@@ -244,187 +239,11 @@ def get_annotator_name(ip: Optional[str], username: Optional[str]) -> Optional[s
     
     return None
 
-def get_first_user_prompt(content: str) -> Optional[str]:
-    """Extract the first user prompt from the conversation content"""
-    try:
-        lines = [line.strip() for line in content.split('\n') if line.strip()]
-        if not lines:
-            return None
-            
-        # Parse the first line to get the messages
-        first_line_data = json.loads(lines[0])
-        
-        # Try different message formats
-        
-        # Format 1: state.messages array with ["<|im_start|>user", "hello"] format
-        messages = first_line_data.get('state', {}).get('messages', [])
-        if messages and len(messages) > 0:
-            first_message = messages[0]
-            if isinstance(first_message, list) and len(first_message) > 1:
-                # Format: ["<|im_start|>user", "hello"]
-                message_content = first_message[1]
-                # Ensure message_content is a string
-                if isinstance(message_content, str):
-                    return message_content
-                elif isinstance(message_content, list):
-                    # If it's a list, try to join it or get the first element
-                    if message_content and all(isinstance(item, str) for item in message_content):
-                        return ' '.join(message_content)
-                    elif message_content and isinstance(message_content[0], str):
-                        return message_content[0]
-                return str(message_content) if message_content else None
-        
-        # Format 2: state.messages array with {"role": "user", "content": "hello"} format
-        if messages and len(messages) > 0:
-            first_message = messages[0]
-            if isinstance(first_message, dict) and 'content' in first_message:
-                message_content = first_message.get('content')
-                # Ensure message_content is a string
-                if isinstance(message_content, str):
-                    return message_content
-                elif isinstance(message_content, list):
-                    # If it's a list, try to join it or get the first element
-                    if message_content and all(isinstance(item, str) for item in message_content):
-                        return ' '.join(message_content)
-                    elif message_content and isinstance(message_content[0], str):
-                        return message_content[0]
-                return str(message_content) if message_content else None
-        
-        # Format 3: Direct messages array in the root
-        messages = first_line_data.get('messages', [])
-        if messages and len(messages) > 0:
-            first_message = messages[0]
-            if isinstance(first_message, list) and len(first_message) > 1:
-                message_content = first_message[1]
-                # Ensure message_content is a string
-                if isinstance(message_content, str):
-                    return message_content
-                elif isinstance(message_content, list):
-                    # If it's a list, try to join it or get the first element
-                    if message_content and all(isinstance(item, str) for item in message_content):
-                        return ' '.join(message_content)
-                    elif message_content and isinstance(message_content[0], str):
-                        return message_content[0]
-                return str(message_content) if message_content else None
-            elif isinstance(first_message, dict) and 'content' in first_message:
-                message_content = first_message.get('content')
-                # Ensure message_content is a string
-                if isinstance(message_content, str):
-                    return message_content
-                elif isinstance(message_content, list):
-                    # If it's a list, try to join it or get the first element
-                    if message_content and all(isinstance(item, str) for item in message_content):
-                        return ' '.join(message_content)
-                    elif message_content and isinstance(message_content[0], str):
-                        return message_content[0]
-                return str(message_content) if message_content else None
-        
-        # Format 4: Look for a specific user role key
-        for key in ['user', 'human', 'Human']:
-            if key in first_line_data:
-                message_content = first_line_data[key]
-                # Ensure message_content is a string
-                if isinstance(message_content, str):
-                    return message_content
-                elif isinstance(message_content, list):
-                    # If it's a list, try to join it or get the first element
-                    if message_content and all(isinstance(item, str) for item in message_content):
-                        return ' '.join(message_content)
-                    elif message_content and isinstance(message_content[0], str):
-                        return message_content[0]
-                return str(message_content) if message_content else None
-        
-        log.warning(f"Could not extract first user prompt from content: {content[:200]}...")
-        return None
-    except Exception as e:
-        log.error(f"Error extracting first user prompt: {e}")
-        return None
-
-def get_unique_prompts_per_annotator(reader: 'RemoteLogReader', start_date_str: str = "2025_02_18") -> Dict[str, Set[str]]:
-    """Get the set of unique prompts for each annotator from the given start date
-    
-    Only includes conversations that pass the vote conditions.
-    Excludes prompts from the predefined list of example prompts.
-    """
-    # Convert start date string to datetime
-    start_date = datetime.strptime(start_date_str, "%Y_%m_%d")
-    
-    # Dictionary to store unique prompts per annotator
-    annotator_prompts = defaultdict(set)
-    
-    try:
-        # Get current date for iteration
-        current_date = start_date
-        today = datetime.now()
-        
-        while current_date <= today:
-            date_str = current_date.strftime("%Y_%m_%d")
-            
-            try:
-                # Get conversation logs for battle_anony mode
-                conv_logs = reader.get_conv_logs(date_str)
-                battle_anony_logs = conv_logs.get('battle_anony', {})
-                
-                # Process each conversation
-                for conv_id, messages in battle_anony_logs.items():
-                    if messages:
-                        try:
-                            # Convert messages to file content format
-                            content = '\n'.join(json.dumps(msg) for msg in messages)
-                            
-                            # First check if the conversation passes the vote conditions
-                            ip, username, vote_conditions_met = get_file_data(content)
-                            
-                            # Only proceed if vote conditions are met
-                            if vote_conditions_met:
-                                # Get annotator name from either IP or username
-                                annotator_name = get_annotator_name(ip, username)
-                                if annotator_name:
-                                    # Extract first user prompt
-                                    try:
-                                        first_prompt = get_first_user_prompt(content)
-                                        if first_prompt:
-                                            # Strip whitespace and check if it's not in the example prompts list
-                                            cleaned_prompt = first_prompt.strip()
-                                            if cleaned_prompt and cleaned_prompt not in EXAMPLE_PROMPTS:
-                                                # Add to set of unique prompts for this annotator
-                                                annotator_prompts[annotator_name].add(cleaned_prompt.lower())
-                                        else:
-                                            log.warning(f"Could not extract first user prompt for conversation {conv_id}")
-                                    except Exception as e:
-                                        log.error(f"Error processing first prompt for conversation {conv_id}: {e}")
-                        except Exception as e:
-                            log.error(f"Error processing conversation {conv_id}: {e}")
-                            
-            except Exception as e:
-                log.error(f"Error processing logs for date {date_str}: {e}")
-            
-            # Move to next day
-            current_date += timedelta(days=1)
-                
-    except Exception as e:
-        log.error(f"Error accessing logs: {e}")
-    
-    return dict(annotator_prompts)
-
-def count_deduplicated_files_per_annotator(reader: 'RemoteLogReader', start_date_str: str = "2025_02_18") -> Dict[str, int]:
-    """Count deduplicated files per annotator name from the given start date, considering both IP and username
-    
-    Deduplication is based on the first user prompt for each annotator.
-    Only counts conversations that pass the vote conditions.
-    Excludes prompts from the predefined list of example prompts.
-    """
-    # Get the unique prompts for each annotator
-    annotator_prompts = get_unique_prompts_per_annotator(reader, start_date_str)
-    
-    # Convert sets to counts
-    return {name: len(prompts) for name, prompts in annotator_prompts.items()}
-
 def count_files_per_annotator(reader: 'RemoteLogReader', start_date_str: str = "2025_02_18") -> Dict[str, int]:
     """Count files per annotator name from the given start date, considering both IP and username"""
     # Convert start date string to datetime
     start_date = datetime.strptime(start_date_str, "%Y_%m_%d")
-    name_counts = defaultdict(int)
+    name_prompts = defaultdict(set)
     try:
         # Get current date for iteration
         current_date = start_date
@@ -442,13 +261,13 @@ def count_files_per_annotator(reader: 'RemoteLogReader', start_date_str: str = "
                     if messages:
                         # Convert messages to file content format
                         content = '\n'.join(json.dumps(msg) for msg in messages)
-                        ip, username, vote_conditions_met = get_file_data(content)
+                        ip, username, vote_conditions_met, user_prompt = get_file_data(content)
                         
                         if vote_conditions_met:
                             # Get annotator name from either IP or username
                             annotator_name = get_annotator_name(ip, username)
-                            if annotator_name:
-                                name_counts[annotator_name] += 1
+                            if annotator_name and user_prompt not in EXAMPLES:
+                                name_prompts[annotator_name].add(user_prompt.lower())
                             
             except Exception as e:
                 log.error(f"Error processing logs for date {date_str}: {e}")
@@ -459,7 +278,7 @@ def count_files_per_annotator(reader: 'RemoteLogReader', start_date_str: str = "
     except Exception as e:
         log.error(f"Error accessing logs: {e}")
     
-    return dict(name_counts)
+    return {name: len(prompts) for name, prompts in name_prompts.items()}
 
 def download_files_by_name(reader: 'RemoteLogReader', start_date_str: str = "2025_02_18", check_sandbox: bool = True) -> None:
     """Download files and organize them by annotator name
@@ -540,85 +359,6 @@ def download_files_by_name(reader: 'RemoteLogReader', start_date_str: str = "202
     except Exception as e:
         log.error(f"Error accessing logs: {e}")
 
-def export_unique_prompts_to_csv(reader: 'RemoteLogReader', output_file: str, start_date_str: str = "2025_02_18") -> None:
-    """Export the unique prompts for each annotator to a CSV file
-    
-    Args:
-        reader: RemoteLogReader instance
-        output_file: Path to the output CSV file
-        start_date_str: The start date in YYYY_MM_DD format
-    """
-    # Get the unique prompts for each annotator
-    annotator_prompts = get_unique_prompts_per_annotator(reader, start_date_str)
-    
-    # Prepare data for CSV
-    rows = []
-    for annotator, prompts in annotator_prompts.items():
-        for prompt in prompts:
-            rows.append({
-                'Annotator': annotator,
-                'Prompt': prompt
-            })
-    
-    # Write to CSV
-    with open(output_file, 'w', newline='', encoding='utf-8') as f:
-        writer = csv.DictWriter(f, fieldnames=['Annotator', 'Prompt'])
-        writer.writeheader()
-        writer.writerows(rows)
-    
-    print(f"Exported {len(rows)} unique prompts to {output_file}")
-
-def debug_problematic_conversations(reader: 'RemoteLogReader', date_str: str) -> None:
-    """Debug function to identify problematic conversations for a specific date
-    
-    Args:
-        reader: RemoteLogReader instance
-        date_str: The date in YYYY_MM_DD format
-    """
-    try:
-        # Get conversation logs for battle_anony mode
-        conv_logs = reader.get_conv_logs(date_str)
-        battle_anony_logs = conv_logs.get('battle_anony', {})
-        
-        print(f"Found {len(battle_anony_logs)} conversations for date {date_str}")
-        
-        # Process each conversation
-        for conv_id, messages in battle_anony_logs.items():
-            if not messages:
-                continue
-                
-            try:
-                # Convert messages to file content format
-                content = '\n'.join(json.dumps(msg) for msg in messages)
-                
-                # Check if the conversation passes the vote conditions
-                ip, username, vote_conditions_met = get_file_data(content)
-                
-                if vote_conditions_met:
-                    # Get annotator name from either IP or username
-                    annotator_name = get_annotator_name(ip, username)
-                    if annotator_name:
-                        # Try to extract first user prompt
-                        try:
-                            first_prompt = get_first_user_prompt(content)
-                            if first_prompt:
-                                print(f"Conversation {conv_id} - Annotator: {annotator_name} - First prompt: {first_prompt[:50]}...")
-                            else:
-                                print(f"WARNING: Could not extract first user prompt for conversation {conv_id} - Annotator: {annotator_name}")
-                                # Print the first line of the content for debugging
-                                first_line = content.split('\n')[0]
-                                print(f"First line: {first_line[:200]}...")
-                        except Exception as e:
-                            print(f"ERROR: Error processing first prompt for conversation {conv_id}: {e}")
-                            # Print the first line of the content for debugging
-                            first_line = content.split('\n')[0]
-                            print(f"First line: {first_line[:200]}...")
-            except Exception as e:
-                print(f"ERROR: Error processing conversation {conv_id}: {e}")
-    
-    except Exception as e:
-        print(f"ERROR: Error processing logs for date {date_str}: {e}")
-
 def main():
     # Initialize RemoteLogReader
     reader = RemoteLogReader()
@@ -627,49 +367,18 @@ def main():
     parser = argparse.ArgumentParser(description='Download and organize conversation files by annotator name')
     parser.add_argument('--sandbox-check', action='store_true', help='Check for matching sandbox logs')
     parser.add_argument('--download', action='store_true', help='Enable file download')
-    parser.add_argument('--export-prompts', action='store_true', help='Export unique prompts to CSV')
-    parser.add_argument('--output-file', default='unique_prompts.csv', help='Output file for unique prompts (default: unique_prompts.csv)')
-    parser.add_argument('--debug-date', help='Debug problematic conversations for a specific date (format: YYYY_MM_DD)')
     args = parser.parse_args()
     
-    # Debug problematic conversations if date is provided
-    if args.debug_date:
-        print(f"\nDebugging problematic conversations for date {args.debug_date}...")
-        debug_problematic_conversations(reader, args.debug_date)
-        return
-    
     # Download files if enabled
     if args.download:
         print("\nDownloading files and organizing by annotator name...")
         download_files_by_name(reader, check_sandbox=args.sandbox_check)
     
-    # Export unique prompts if enabled
-    if args.export_prompts:
-        print(f"\nExporting unique prompts to {args.output_file}...")
-        export_unique_prompts_to_csv(reader, args.output_file)
-    
     # Count and display statistics
     name_counts = count_files_per_annotator(reader)
-    unique_counts = count_deduplicated_files_per_annotator(reader)
-    
     print("\nFile counts per annotator:")
-    print(f"{'Name':<20} {'Total':<10} {'Unique':<10} {'Unique %':<10}")
-    print("-" * 50)
-    
-    # Calculate totals
-    total_sum = sum(name_counts.values())
-    unique_sum = sum(unique_counts.values())
-    
-    # Display counts for each annotator
     for name, count in sorted(name_counts.items(), key=lambda x: x[1], reverse=True):
-        unique = unique_counts.get(name, 0)
-        percentage = round((unique / count * 100) if count > 0 else 0, 1)
-        print(f"{name:<20} {count:<10} {unique:<10} {percentage:<10}%")
-    
-    # Display totals
-    overall_percentage = round((unique_sum / total_sum * 100) if total_sum > 0 else 0, 1)
-    print("-" * 50)
-    print(f"{'TOTAL':<20} {total_sum:<10} {unique_sum:<10} {overall_percentage:<10}%")
+        print(f"Name: {name:<20} Count: {count}")
 
 if __name__ == "__main__":
     main()