.github/scripts/update_conferences_new.py

import yaml
import requests
import os
import re
from datetime import datetime
from typing import Dict, List, Any


def fetch_conference_files() -> List[Dict[str, Any]]:
    """Fetch all conference YAML files from ccfddl repository."""

    # First get the directory listing from GitHub API
    api_url = "https://api.github.com/repos/ccfddl/ccf-deadlines/contents/conference/AI"
    response = requests.get(api_url)
    files = response.json()

    conferences = []
    for file in files:
        if file["name"].endswith(".yml"):
            yaml_content = requests.get(file["download_url"]).text
            conf_data = yaml.safe_load(yaml_content)
            # The data is a list with a single item
            if isinstance(conf_data, list) and len(conf_data) > 0:
                conferences.append(conf_data[0])

    return conferences


def parse_date_range(date_str: str, year: str) -> tuple[str, str]:
    """Parse various date formats and return start and end dates."""
    # Remove the year if it appears at the end of the string
    date_str = date_str.replace(f", {year}", "")

    # Handle various date formats
    try:
        # Split into start and end dates
        if " - " in date_str:
            start, end = date_str.split(" - ")
        elif "-" in date_str:
            start, end = date_str.split("-")
        else:
            # For single date format like "May 19, 2025"
            start = end = date_str

        # Clean up month abbreviations
        month_map = {
            "Sept": "September",  # Handle Sept before Sep
            "Jan": "January",
            "Feb": "February",
            "Mar": "March",
            "Apr": "April",
            "Jun": "June",
            "Jul": "July",
            "Aug": "August",
            "Sep": "September",
            "Oct": "October",
            "Nov": "November",
            "Dec": "December",
        }

        # Create a set of all month names (full and abbreviated)
        all_months = set(month_map.keys()) | set(month_map.values())

        # Handle cases like "April 29-May 4"
        has_month = any(month in end for month in all_months)
        if not has_month:
            # End is just a day number, use start's month
            start_parts = start.split()
            if len(start_parts) >= 1:
                end = f"{start_parts[0]} {end.strip()}"

        # Replace month abbreviations
        for abbr, full in month_map.items():
            start = start.replace(abbr, full)
            end = end.replace(abbr, full)

        # Clean up any extra spaces
        start = " ".join(start.split())
        end = " ".join(end.split())

        # Parse start date
        start_date = datetime.strptime(f"{start}, {year}", "%B %d, %Y")

        # Parse end date
        end_date = datetime.strptime(f"{end}, {year}", "%B %d, %Y")

        return start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d")

    except Exception as e:
        raise ValueError(f"Could not parse date: {date_str} ({e})")


def transform_conference_data(
    conferences: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
    """Transform ccfddl format to our format."""
    transformed = []
    current_year = datetime.now().year

    for conf in conferences:
        # Get the most recent or upcoming conference instance
        recent_conf = None
        if "confs" in conf:
            for instance in conf["confs"]:
                if instance["year"] >= current_year:
                    recent_conf = instance
                    break

        if not recent_conf:
            continue

        # Transform to our format
        transformed_conf = {
            "title": conf.get("title", ""),
            "year": recent_conf["year"],
            "id": recent_conf["id"],
            "full_name": conf.get("description", ""),
            "link": recent_conf.get("link", ""),
            "deadline": recent_conf.get("timeline", [{}])[0].get("deadline", ""),
            "timezone": recent_conf.get("timezone", ""),
            "date": recent_conf.get("date", ""),
            "tags": [],  # We'll need to maintain a mapping for tags
        }

        # Handle city and country fields instead of place
        place = recent_conf.get("place", "")
        if place:
            # Try to parse the place into city and country if it contains a comma
            if "," in place:
                city, country = place.split(",", 1)
                transformed_conf["city"] = city.strip()
                transformed_conf["country"] = country.strip()
            else:
                # If we can't parse, just set the country
                transformed_conf["country"] = place.strip()

        # Add optional fields
        timeline = recent_conf.get("timeline", [{}])[0]
        if "abstract_deadline" in timeline:
            transformed_conf["abstract_deadline"] = timeline["abstract_deadline"]

        # Parse date range for start/end
        try:
            if transformed_conf["date"]:
                start_date, end_date = parse_date_range(
                    transformed_conf["date"], str(transformed_conf["year"])
                )
                transformed_conf["start"] = start_date
                transformed_conf["end"] = end_date
        except Exception as e:
            print(f"Warning: Could not parse date for {transformed_conf['title']}: {e}")

        # Add rankings as separate field
        if "rank" in conf:
            rankings = []
            for rank_type, rank_value in conf["rank"].items():
                rankings.append(f"{rank_type.upper()}: {rank_value}")
            if rankings:
                transformed_conf["rankings"] = ", ".join(rankings)

        transformed.append(transformed_conf)

    return transformed


def load_all_current_conferences() -> Dict[str, List[Dict[str, Any]]]:
    """Load all current conferences from individual files."""
    conferences_dir = "src/data/conferences"
    conference_groups = {}

    if not os.path.exists(conferences_dir):
        return {}

    for filename in os.listdir(conferences_dir):
        if filename.endswith(".yml"):
            filepath = os.path.join(conferences_dir, filename)
            with open(filepath, "r") as f:
                conferences = yaml.safe_load(f)
                if conferences:
                    # Extract conference title from the first entry
                    title = conferences[0]["title"]
                    conference_groups[title] = conferences

    return conference_groups


def create_filename_from_title(title: str) -> str:
    """Create a filename-safe version of the conference title."""
    filename = re.sub(r"[^a-zA-Z0-9\s&()-]", "", title.lower())
    filename = re.sub(r"\s+", "_", filename)
    filename = filename.replace("&", "and")
    filename = filename.strip("_")
    return filename


def update_conference_loader():
    """Update the conference loader file with all current conferences."""
    conferences_dir = "src/data/conferences"
    loader_path = "src/utils/conferenceLoader.ts"

    # Get all conference files
    conference_files = []
    if os.path.exists(conferences_dir):
        for filename in sorted(os.listdir(conferences_dir)):
            if filename.endswith(".yml"):
                conference_files.append(filename)

    # Generate import statements
    imports = []
    variable_names = []

    for filename in conference_files:
        # Create variable name from filename
        var_name = filename.replace(".yml", "").replace("-", "_") + "Data"
        variable_names.append(var_name)
        imports.append(f"import {var_name} from '@/data/conferences/{filename}';")

    # Generate the loader file content
    loader_content = f"""import {{ Conference }} from '@/types/conference';

// Import all conference YAML files
{chr(10).join(imports)}

// Combine all conference data into a single array
const allConferencesData: Conference[] = [
{chr(10).join(f"  ...{var_name}," for var_name in variable_names)}
];

export default allConferencesData;"""

    # Write the loader file
    with open(loader_path, "w") as f:
        f.write(loader_content)

    print(f"Updated conference loader with {len(conference_files)} conference files")


def main():
    try:
        # Load current conferences from individual files
        current_conference_groups = load_all_current_conferences()

        # Fetch and transform new data
        new_conferences = fetch_conference_files()
        if not new_conferences:
            print("Warning: No conferences fetched from ccfddl")
            return

        transformed_conferences = transform_conference_data(new_conferences)
        if not transformed_conferences:
            print("Warning: No conferences transformed")
            return

        # Create conferences directory if it doesn't exist
        conferences_dir = "src/data/conferences"
        os.makedirs(conferences_dir, exist_ok=True)

        # Group new conferences by title
        new_conference_groups = {}
        for conf in transformed_conferences:
            title = conf["title"]
            if title not in new_conference_groups:
                new_conference_groups[title] = []
            new_conference_groups[title].append(conf)

        # Update each conference group
        updated_count = 0
        for title, new_confs in new_conference_groups.items():
            filename = create_filename_from_title(title) + ".yml"
            filepath = os.path.join(conferences_dir, filename)

            # Get current conferences for this title
            current_confs = current_conference_groups.get(title, [])
            current_conf_dict = {conf["id"]: conf for conf in current_confs}

            # Update or add new conferences
            for new_conf in new_confs:
                if new_conf["id"] in current_conf_dict:
                    # Update existing conference while preserving fields
                    curr_conf = current_conf_dict[new_conf["id"]]

                    # Preserve existing fields
                    preserved_fields = [
                        "tags",
                        "venue",
                        "hindex",
                        "submission_deadline",
                        "timezone_submission",
                        "rebuttal_period_start",
                        "rebuttal_period_end",
                        "final_decision_date",
                        "review_release_date",
                        "commitment_deadline",
                        "start",
                        "end",
                        "note",
                        "city",
                        "country",
                        "deadlines",
                    ]
                    for field in preserved_fields:
                        if field in curr_conf:
                            new_conf[field] = curr_conf[field]

                    # Preserve existing rankings if available
                    if "rankings" in curr_conf:
                        new_conf["rankings"] = curr_conf["rankings"]

                    current_conf_dict[new_conf["id"]] = new_conf
                else:
                    # Add new conference
                    current_conf_dict[new_conf["id"]] = new_conf

            # Convert back to list and sort by year
            all_confs = list(current_conf_dict.values())
            all_confs.sort(key=lambda x: x.get("year", 9999))

            # Write to individual file
            with open(filepath, "w") as f:
                yaml.dump(
                    all_confs,
                    f,
                    default_flow_style=False,
                    sort_keys=False,
                    allow_unicode=True,
                )

            updated_count += 1
            print(f"Updated {filename} with {len(all_confs)} entries")

        # Update the conference loader
        update_conference_loader()

        print(f"Successfully updated {updated_count} conference files")

    except Exception as e:
        print(f"Error: {e}")
        raise


if __name__ == "__main__":
    main()