import json
import csv
import os
import glob
from collections import defaultdict
import datetime

def clean_value(val):
    if isinstance(val, str):
        return val.replace('*', '')
    return val

def parse_pollution_data(directory):
    # Map (date, location) -> {time: value}
    # Using a dictionary: data[date][location][time] = value
    data = defaultdict(lambda: defaultdict(dict))
    
    files = sorted(glob.glob(os.path.join(directory, "*.json")))
    
    # Track stats
    total_files = 0
    success_files = 0
    failed_files = 0
    
    # To be used in CSV generation
    time_labels = []
    
    with open('parse_pollution.log', 'a') as log_file:
        log_file.write(f"--- Execution started at {datetime.datetime.now()} ---\n")
        
        for file_path in files:
            total_files += 1
            date_str = os.path.basename(file_path).replace(".json", "")
            try:
                with open(file_path, 'r') as f:
                    content = json.load(f)
                    rows = content.get("24hour_api_apims", [])
                    if len(rows) < 2:
                        raise ValueError("Insufficient rows or invalid structure")
                    
                    header = rows[0]
                    # Indices 2 to 25 are the hourly times
                    time_labels = header[2:]
                    
                    for row in rows[1:]:
                        state = row[0]
                        location = row[1]
                        readings = row[2:]
                        
                        for i, time_label in enumerate(time_labels):
                            value = clean_value(readings[i])
                            data[date_str][location][time_label] = value
                    
                    success_files += 1
            except Exception as e:
                failed_files += 1
                log_file.write(f"Error processing {file_path}: {e}\n")
    
    # Display summary
    print(f"Summary: Total files: {total_files}, Successfully parsed: {success_files}, Failed: {failed_files}")
    
    if not data:
        print("No valid data to write to CSV.")
        return

    # Prepare for CSV
    # Columns: Location (and date/time)
    
    with open('pollution_data.csv', 'w', newline='') as f:
        # Collect all unique locations
        all_locations = set()
        for date in data:
            for loc in data[date]:
                all_locations.add(loc)
        sorted_locations = sorted(list(all_locations))
        
        writer = csv.writer(f)
        writer.writerow(['DateTime'] + sorted_locations)
        
        for date in sorted(data.keys()):
            for time in time_labels: # Using the time labels found
                row = [f"{date} {time}"]
                for loc in sorted_locations:
                    row.append(data[date][loc].get(time, ""))
                writer.writerow(row)

if __name__ == "__main__":
    parse_pollution_data('.')