code:
"""
Support library for working with KisekaeLocal.

This module contains functions for importing Kisekae codes using KisekaeLocal,
and functions for processing the generated image files.

It can also serve as a utility for automatically importing sets of codes from
CSV files, plain text files, or directly from the command line.
"""

import asyncio
import argparse
import csv
import os
import os.path as osp
from pathlib import Path
import re
import sys
import time

import numpy as np
from scipy import ndimage
from PIL import Image

SETUP_STRING_33 = (
    "33***bc185.500.0.0.1_ga0*0*0*0*0*0*0*0*0#/]ua1.0.0.0_ub_uc7.0.30_ud7.0"
)
SETUP_STRING_36 = "36***bc185.500.0.0.1_ga0*0*0*0*0*0*0*0*0#/]a00_b00_c00_d00_w00_x00_y00_z00_ua1.0.0.0_ub_u0_v0_uc7.0.30_ud7.0"
SETUP_STRING_40 = "40***bc185.500.0.0.1*0*0*0*0*0*0*0*0#/]a00_b00_c00_d00_w00_x00_y00_z00_ua1.0.0.0.100_uf0.3.0.0_ue_ub_u0_v0_uc7.2.24_ud7.8"
SETUP_STRING_68 = "68***ba50_bb6.0_bc410.500.8.0.1.0_bd6_be180_ad0.0.0.0.0.0.0.0.0.0_ae0.3.3.0.0*0*0*0*0*0*0*0*0#/]a00_b00_c00_d00_w00_x00_e00_y00_z00_ua1.0.0.0.100_uf0.3.0.0_ue_ub_u0_v00_ud7.8_uc7.2.24"
SEPARATOR = "#/]"
IMG_ATTACH_SEPARATOR = "/#]"

CODE_SPLIT_REGEX = r"(\d+?)\*\*\*?(.+)"

class KisekaeComponent(object):
    def __init__(self, data=None):
        """
        Represents a subcomponent of a Kisekae character or scene.

        Attributes:
            id (str): An ID identifying this subcomponent's type.
            prefix (str): A prefix identifying this subcomponent.
            attributes (list of str): The attributes associated with this component.
        """

        if isinstance(data, KisekaeComponent):
            self.id = data.id
            self.prefix = data.prefix
            self.attributes = data.attributes.copy()
        elif isinstance(data, str):
            if data[1].isalpha():
                self.id = data[0:2]  # code is 2 letters
                self.prefix = data[0:2]
            else:
                self.id = data[0]
                self.prefix = data[0:3]  # code is 1 letter + 2 digits
                self.index = int(data[1:3])

            self.attributes = data[len(self.prefix) :].split(".")
        else:
            raise ValueError(
                "`data` must be either str or KisekaeComponent, not "
                + type(data).__name__
            )

    def __eq__(self, other):
        if isinstance(other, KisekaeComponent):
            if (
                self.id != other.id
                or self.prefix != other.prefix
                or len(self.attributes) != len(other.attributes)
            ):
                return False

            for x, y in zip(self.attributes, other.attributes):
                if x != y:
                    return False

            return True
        elif isinstance(other, str):
            return str(self) == other
        else:
            raise NotImplementedError(
                "KisekaeComponents can only be compared to other Components or strings."
            )

    def __len__(self):
        return len(self.attributes)

    def __iter__(self):
        return self.attributes.__iter__()

    def __getitem__(self, key):
        return self.attributes[key]

    def __setitem__(self, key, val):
        self.attributes[key] = str(val)

    def __delitem__(self, key):
        del self.attributes[key]

    def __contains__(self, item):
        return self.attributes.__contains__(self, item)

    def __str__(self):
        return self.prefix + ".".join(self.attributes)

class KisekaeCharacter(object):
    def __init__(self, character_data=None):
        """
        Represents a collection of subcodes.

        Attributes:
            subcodes (list of KisekaeComponent): The subcodes contained within this object.
        """

        self.subcodes = []
        self.image_attachments = []

        if isinstance(character_data, str):
            if IMG_ATTACH_SEPARATOR in character_data:
                s = character_data.split(IMG_ATTACH_SEPARATOR)
                character_data = s[0]
                self.image_attachments.extend(s[1:])

                # print("Attachments: "+str(self.image_attachments))

            for subcode in character_data.split("_"):
                self.subcodes.append(KisekaeComponent(subcode))
        elif isinstance(character_data, KisekaeCharacter):
            for subcode in character_data.subcodes:
                self.subcodes.append(KisekaeComponent(subcode))

            self.image_attachments = list(character_data.image_attachments)
        else:
            raise ValueError(
                "`character_data` must be either str or KisekaeCharacter, not "
                + type(character_data).__name__
            )

    def __str__(self):
        base = "_".join(str(sc) for sc in self.subcodes)

        if len(self.image_attachments) > 0:
            base += IMG_ATTACH_SEPARATOR + IMG_ATTACH_SEPARATOR.join(
                self.image_attachments
            )

        # print("Reconstructed: {}".format(base))

        return base

    def __len__(self):
        return len(self.subcodes)

    def __iter__(self):
        return self.subcodes.__iter__()

    def __getitem__(self, key):
        if isinstance(key, int):
            return self.subcodes[key]
        elif isinstance(key, str):
            v = self.find(key)
            if v is None:
                raise KeyError("No subcode with ID {:s} in this character".format(key))
            return v
        else:
            raise ValueError(
                "Index value must be either an int or a subcode ID string."
            )

    def __setitem__(self, key, val):
        if not isinstance(val, KisekaeComponent):
            raise ValueError("Assignment value must be a KisekaeComponent.")

        if isinstance(key, int):
            self.subcodes[key] = val
        elif isinstance(key, str):
            idx = None

            for i, sc in enumerate(self.subcodes):
                if sc.prefix == key:
                    idx = i
                    break
            else:
                raise KeyError("No subcode with ID {:s} in this character".format(key))

            self.subcodes[idx] = val

    def __delitem__(self, key):
        if isinstance(key, int):
            del self.subcodes[key]
        elif isinstance(key, str):
            idx = None

            for i, sc in enumerate(self.subcodes):
                if sc.prefix == key:
                    idx = i
                    break
            else:
                raise KeyError("No subcode with ID {:s} in this character".format(key))

            del self.subcodes[idx]

    def __contains__(self, item):
        if isinstance(item, KisekaeComponent):
            return self.subcodes.__contains__(self, item)
        elif isinstance(item, str):
            return self.find(item) is not None
        else:
            raise ValueError(
                "Item must be either a KisekaeComponent or a subcode ID string."
            )

    def find(self, subcode_prefix):
        """
        Find the first inner KisekaeComponent with the given `subcode_prefix`.
        """

        for sc in self.subcodes:
            if sc.prefix == subcode_prefix:
                return sc

    def iter(self, subcode_prefix):
        """
        Iterate over all inner KisekaeComponents with the given `subcode_prefix`
        """

        return filter(lambda sc: sc.prefix.startswith(subcode_prefix), self.subcodes)

class KisekaeCode(object):
    def __init__(self, code=None):
        """
        Represents an entire importable Kisekae code, possibly containing
        character data and scene data.

        Attributes:
            version (int): The version of Kisekae used to generate this code.
            scene (KisekaeCharacter): Container for scene data and attributes.
            characters (list of KisekaeCharacter): List of characters contained in the code.
        """

        self.version = 92
        self.characters = []
        self.scene = None

        if isinstance(code, KisekaeCode):
            self.version = code.version

            for character in code:
                self.characters.append(KisekaeCharacter(character))

            if code.scene is not None:
                self.scene = KisekaeCharacter(code.scene)

            return
        elif isinstance(code, KisekaeCharacter):
            self.characters.append(KisekaeCharacter(code))
            return
        elif isinstance(code, str):
            m = re.match(CODE_SPLIT_REGEX, code.strip())
            if m is None:
                return

            version, character_data = m.groups()
            scene_data = None

            if SEPARATOR in character_data:
                character_data, scene_data = character_data.split(SEPARATOR, 1)

            self.version = int(version)
            self.characters = []

            if scene_data is not None:
                self.scene = KisekaeCharacter(scene_data)
            else:
                self.scene = None

            for character in character_data.split("*"):
                if len(character) == 0 or character == "0":
                    continue

                self.characters.append(KisekaeCharacter(character))
        else:
            raise ValueError(
                "`code` must be either a KisekaeCode, KisekaeCharacter, or str, not "
                + type(code).__name__
            )

    def __str__(self):
        ret = str(self.version) + "**"

        if self.scene is not None:
            for i in range(9):
                if i >= len(self.characters):
                    ret += "*0"
                else:
                    ret += "*" + str(self.characters[i])

            ret += SEPARATOR + str(self.scene)
        else:
            ret += str(self.characters[0])

        return ret

    def __len__(self):
        return len(self.characters)

    def __iter__(self):
        return self.characters.__iter__()

    def __getitem__(self, key):
        return self.characters[key]

    def __setitem__(self, key, val):
        self.characters[key] = str(val)

    def __delitem__(self, key):
        del self.characters[key]

    def __contains__(self, item):
        return self.characters.__contains__(self, item)

def is_kisekae_code(code):
    if not isinstance(code, str):
        return False

    m = re.match(CODE_SPLIT_REGEX, code.strip())
    if m is None:
        return False
    return True

def _get_wine_kkl_directory():
    # look for kkl path under Wine:
    username = Path.home().stem
    wine_path = (
        Path.home()
        / ".wine"
        / "drive_c"
        / "users"
        / username
        / "Application Data"
        / "kkl"
        / "Local Store"
    )

    return wine_path

def get_kkl_directory():
    """
    Retrieve the path to the main KisekaeLocal application directory.
    """

    if sys.platform == "darwin":
        native_path = (
            Path.home() / "Library" / "Application Support" / "kkl" / "Local Store"
        )
        if native_path.is_dir():
            return native_path

        p = _get_wine_kkl_directory()
        if p.is_dir():
            return p
        else:
            raise FileNotFoundError("Could not find path to KKL.")
    elif sys.platform.startswith("linux"):
        p = _get_wine_kkl_directory()
        if p.is_dir():
            return p
        else:
            raise FileNotFoundError("Could not find path to KKL.")
    else:  # We're on windows, presumably
        return Path(os.getenv("APPDATA")) / "kkl" / "Local Store"

async def wait_for_predicate(p, max_wait=60.0):
    loop = asyncio.get_event_loop()
    end_time = loop.time() + max_wait

    while True:
        if p():
            return True

        if (loop.time() + 0.1) >= end_time:
            return False

        await asyncio.sleep(0.1)

async def process_kkl_code(code, scene_name, alpha_values=None):
    """
    Import an image into KisekaeLocal.
    Returns a Path object to the output image once it has been generated.

    Args:
        code (str or KisekaeCode): The Kisekae code to import.
        scene_name (str): A pose name to use when importing.
    """

    input_path = get_kkl_directory().joinpath(scene_name + ".txt")
    output_path = get_kkl_directory().joinpath(scene_name + ".png")

    # remove the input and output files if they already exist
    if output_path.is_file():
        output_path.unlink()

    if input_path.is_file():
        input_path.unlink()

    sys.stdout.write("Importing: {:s}... ".format(scene_name))
    sys.stdout.flush()

    outdata = ""

    if alpha_values is not None:
        print("Using alpha_values: {}".format(alpha_values))
        for key, value in alpha_values.items():
            if value < 0.001:
                value = 0.001
            outdata += "{}={}\n".format(key, value)

    outdata += str(code)

    print("\n" + outdata.strip())
    sys.stdout.flush()

    with input_path.open("w", encoding="utf-8") as f:
        f.write(outdata.strip())

    # wait for KKL to process the file
    # print("Waiting for output file to be generated...")
    res = await wait_for_predicate(
        lambda: (not input_path.is_file()) and output_path.is_file()
    )

    if not res:
        raise FileNotFoundError("File predicate not satisfied")

    return output_path

async def open_image_file(path):
    """
    Attempt to open an image file generated by KKL.
    """

    retry_time_limit = 10  #  try for at most 10 seconds before saying there's a problem
    retry_interval = (
        0.2  #  re-check for the existance of the image every 200 milliseconds
    )
    retry_limit = int(retry_time_limit // retry_interval)  # try 50 times

    for retry in range(retry_limit):
        try:
            image_file = Image.open(path)
            return image_file
        except IOError:
            if retry >= retry_limit - 1:
                raise
            await asyncio.sleep(retry_interval)

def get_crop_box(width=600, height=1400, center_x=1000, margin_y=15):
    """
    Calculate a cropping rectangle.

    Args:
        width (int): The width of the output image.
        height (int): The height of the output image.
        center_x (int): Position along the X-axis for the center of the rectangle.
        margin_y (int): How far down along the Y-axis to position the rectangle.

    Returns:
        A cropping box as a 4-tuple, suitable to be passed to Image.crop().
    """

    left = center_x - int(width // 2)
    right = center_x + int(width // 2)
    upper = margin_y
    lower = height + margin_y

    return (left, upper, right, lower)

def auto_crop_box(image, margin_y=15):
    """
    Automatically calculate a centered cropping rectangle from an image's bounding box.

    Args:
        margin_y (int): How much empty margin space to leave at the bottom of the image, in px.

    Returns:
        A cropping box as a 4-tuple, suitable to be passed to Image.crop().
    """

    arr = np.array(image)
    intensity = np.sum(arr, axis=2)
    nonzero = np.greater(intensity, 0).T

    centroid = ndimage.measurements.center_of_mass(nonzero)

    left, top, right, bottom = image.getbbox()

    out_height = bottom - top
    if out_height < 1400:
        out_height = 1400

    com_x = centroid[0]
    center_x = int((right + left) / 2)
    shift_x = int(center_x - com_x)  # negative:right, positive:left

    out_width = (right - left) + 2
    if out_width < 600:
        out_width = 600

    crop_left = int(com_x - (out_width // 2))
    crop_right = int(com_x + (out_width // 2))

    if shift_x < 0:
        # Add extra space to the left side
        crop_left -= shift_x
    elif shift_x > 0:
        crop_right += shift_x

    if crop_left > left:
        crop_left = left - 1
        crop_right = crop_left + out_width

    if crop_right < right:
        crop_right = right + 1
        crop_left = crop_right - out_width

    crop_top = (bottom + margin_y) - out_height
    crop_bottom = bottom + margin_y

    return (crop_left, crop_top, crop_right, crop_bottom)

async def import_character(import_code, pose_name="imported", alpha_values=None):
    """
    Import a code into KisekaeLocal and open the generated image file.

    Args:
        import_code (str or KisekaeCode): The Kisekae code to import.
        pose_name (str): A pose name to use when importing.
    """

    output_path = await process_kkl_code(import_code, pose_name, alpha_values)

    img = await open_image_file(output_path)
    img.load()

    output_path.unlink()

    return img

async def import_and_unlink(import_code, scene_name="temp"):
    """
    Import the specified `import_code` but immediately delete the generated image.
    This is useful for scene resets, quickly opening characters for editing in KKL, etc.

    Args:
        import_code (str or KisekaeCode): The Kisekae code to import.
        pose_name (str): A pose name to use when importing.
    """

    output_path = await process_kkl_code(import_code, scene_name)

    retry_time_limit = 10  #  try for at most 10 seconds before saying there's a problem
    retry_interval = (
        0.2  #  re-check for the existance of the image every 200 milliseconds
    )
    retry_limit = int(retry_time_limit // retry_interval)  # try 50 times

    for retry in range(retry_limit):
        try:
            output_path.unlink()
            break
        except PermissionError:
            if retry >= retry_limit - 1:
                raise
            await asyncio.sleep(retry_interval)

    sys.stdout.write("done.\n")
    sys.stdout.flush()

async def setup_kkl_scene():
    """
    Send a scene reset code to KKL to prepare for image importing.
    This will clear away unnecessary characters, backgrounds, objects, etc.
    """

    await import_and_unlink(SETUP_STRING_68, "scene_setup")

def crop_and_save(img, crop_box, dest_filename):
    """
    Crop an image and save it.
    This is a convenience function.

    Args:
        img (PIL.Image): The image object to crop and save.
        crop_box (4-tuple): The cropping rectangle.
        dest_filename (pathlib.Path or str): The destination filename to save to.
    """

    cropped_img = img.crop(crop_box)
    cropped_img.save(dest_filename)
    cropped_img.close()
    img.close()

def read_ce_pose_file(path):
    """
    Read a pose file in the format used by the CE, discarding cropping information.
    """

    poses = {}

    with open(path, "r", encoding="utf-8") as f:
        for line in f:
            try:
                pose_name, code = line.split("=", 1)
            except ValueError:
                continue

            poses[pose_name.strip()] = code.strip()

    return poses

async def process_single(code, dest, **kwargs):
    """
    Import and process a Kisekae code.

    Args:
        code (str or KisekaeCode): The Kisekae code to import.
        dest (str or pathlib.Path): The destination to save to. If the destination exists and is a directory, the image will be saved here as `out.png`.
            Otherwise, it will be treated as a filename to save to.

    Kwargs:
        remove_motion (bool, optional): If True (default), then a code fragment will be automatically appended to the imported codes to disable unwanted breast motion.
            This may potentially override settings from the imported codes themselves, so be wary.
        auto_crop (bool, optional): If True, then cropping boxes will be automatically calculated based on the generated image bounding boxes.
            The `width`, `height`, and `center_x` kwargs will be ignored.
        width (int): The width of the generated images (when using manual cropping).
        height (int): The height of the generated images (when using manual cropping).
        center_x (int): The position along the X-axis around which to center the generated images (when using manual cropping).
        margin_y (int): When using manual cropping, this is how far down to position the cropping box.
            When using automatic cropping, this controls how much empty space to add at the bottom of the generated images.
    """

    dest = Path(dest)

    if dest.is_dir():
        dest_filename = dest.joinpath("out.png")
    else:
        dest_filename = dest

    if kwargs.get("do_setup", True):
        await setup_kkl_scene()

    kkl_output = await import_character(
        code, dest_filename.stem, kwargs.get("alpha_values", None)
    )

    # kkl_output.save(str(dest_filename.parent.joinpath(dest_filename.stem + '.debug.png')))

    if dest_filename.is_file():
        dest_filename.unlink()

    margin_y = int(kwargs.get("margin_y", 15))

    if kwargs.get("auto_crop", True):
        crop_box = auto_crop_box(kkl_output, margin_y)
    else:
        width = int(kwargs.get("width", 600))
        height = int(kwargs.get("height", 1400))
        center_x = int(kwargs.get("center_x", 1000))
        crop_box = get_crop_box(width, height, center_x, margin_y)

    crop_and_save(kkl_output, crop_box, dest_filename)

    sys.stdout.write("done.\n")
    sys.stdout.flush()

async def process_csv(infile, dest_dir, **kwargs):
    """
    Import and process a set of Kisekae codes listed in a CSV file.

    Args:
        infile (pathlib.Path): The CSV file to read.
        dest_dir (pathlib.Path): The destination folder to save to.

    Kwargs:
        stage (list of int, optional): If set, then only codes for these stages will be imported.
        pose (list of str, optional): If set, then only codes for these poses will be imported.
    """

    await setup_kkl_scene()
    with infile.open("r", encoding="utf-8", newline="") as f:
        reader = csv.DictReader(f, restval="")
        for row in reader:
            if "enabled" in row and row["enabled"].strip().lower() == "false":
                continue

            if "stage" not in row or "code" not in row or "pose" not in row:
                continue

            if len(row["stage"]) <= 0 or len(row["code"]) <= 0 or len(row["pose"]) <= 0:
                continue

            # normalize pose name:
            row["pose"] = re.sub(r"[^\w\-\_]", "", row["pose"]).lower().strip()

            stage = int(row["stage"])

            if (
                kwargs["stage"] is not None
                and len(kwargs["stage"]) > 0
                and stage not in kwargs["stage"]
            ):
                continue

            if (
                kwargs["pose"] is not None
                and len(kwargs["pose"]) > 0
                and row["pose"] not in kwargs["pose"]
            ):
                continue

            pose_name = "{:d}-{:s}".format(stage, row["pose"])
            dest_filename = dest_dir.joinpath(pose_name + ".png")

            row["remove_motion"] = (
                row.get("remove_motion", "").strip().lower() != "false"
            )
            row["close_vagina"] = row.get("close_vagina", "").strip().lower() != "false"

            await process_single(dest=dest_filename, do_setup=False, **row)

async def process_file(infile, dest_dir, **kwargs):
    """
    Import and process a set of Kisekae codes listed in a plain-text file.

    Args:
        infile (pathlib.Path): The text file to read.
        dest_dir (pathlib.Path): The destination folder to save to.

    Kwargs:
        remove_motion (bool, optional): If True (default), then a code fragment will be automatically appended to the imported codes to disable unwanted breast motion.
            This may potentially override settings from the imported codes themselves, so be wary.
        auto_crop (bool, optional): If True, then cropping boxes will be automatically calculated based on the generated image bounding boxes.
            The `width`, `height`, and `center_x` kwargs will be ignored.
        width (int): The width of the generated images (when using manual cropping).
        height (int): The height of the generated images (when using manual cropping).
        center_x (int): The position along the X-axis around which to center the generated images (when using manual cropping).
        margin_y (int): When using manual cropping, this is how far down to position the cropping box.
            When using automatic cropping, this controls how much empty space to add at the bottom of the generated images.
    """

    dest_dir = Path(dest_dir)

    await setup_kkl_scene()
    with open(infile, "r", encoding="utf-8") as f:
        for line in f:
            try:
                pose_name, code = line.split("=", 1)
            except ValueError:
                continue

            pose_name = pose_name.strip()
            code = code.strip()

            dest_filename = dest_dir.joinpath(pose_name + ".png")

            await process_single(code, dest=dest_filename, do_setup=False, **kwargs)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Imports a pose into KisekaeLocal, and saves an automatically cropped image."
    )
    parser.add_argument("--width", "-w", default=600, help="Output image width.")
    parser.add_argument("--height", "-l", default=1400, help="Output image height.")
    parser.add_argument(
        "--center-x",
        "-x",
        default=1000,
        help="X position to center the output image around.",
    )
    parser.add_argument(
        "--margin-y",
        "-y",
        default=15,
        help="Number of margin pixels from top when cropping output image.",
    )
    parser.add_argument(
        "--manual-crop",
        "--manual",
        "-m",
        action="store_false",
        dest="auto_crop",
        help="Do not automatically calculate crop with bounding box.",
    )
    parser.add_argument(
        "--stage",
        nargs="*",
        type=int,
        help="(CSV-only) Import codes for a particular stage or set of stages only.",
    )
    parser.add_argument(
        "--pose",
        nargs="*",
        help="(CSV-only) Import codes for a particular pose or set of poses only.",
    )
    parser.add_argument(
        "--no-remove-motion",
        action="store_false",
        dest="remove_motion",
        help="Do not automatically set motion parameters to 'Manual' on importing.",
    )
    parser.add_argument(
        "--no-close-vagina",
        action="store_false",
        dest="close_vagina",
        help="Do not automatically close the flaps of a female character's vagina on importing.",
    )
    parser.add_argument("--file", "-f", help="Load codes from text file.")
    parser.add_argument("--csv", "-c", help="Load codes from a CSV file.")
    parser.add_argument(
        "--code",
        "-i",
        help="Load code as command line argument. [dest_dir] is the destination image path in this case.",
    )
    parser.add_argument("dest_dir", help="Destination directory to output to.")
    args = parser.parse_args()

    kwargs = vars(args)

    dest_dir = Path(args.dest_dir).resolve()
    del kwargs["dest_dir"]

    if args.csv is not None:
        asyncio.run(process_csv(Path(args.csv), dest_dir, **kwargs))
    elif args.file is not None:
        asyncio.run(process_file(Path(args.file), dest_dir, **kwargs))
    elif args.code is not None:
        code = args["code"]
        del kwargs["code"]
        asyncio.run(process_single(code, dest_dir, **kwargs))
    else:
        raise ValueError("Must provide at least one of --csv, --file, or --code.")


code:
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*

# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

# Runtime data
pids
*.pid
*.seed
*.pid.lock

# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov

# Coverage directory used by tools like istanbul
coverage

# nyc test coverage
.nyc_output

# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt

# Bower dependency directory (https://bower.io/)
bower_components

# node-waf configuration
.lock-wscript

# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release

# Dependency directories
node_modules/
jspm_packages/

# TypeScript v1 declaration files
typings/

# Optional npm cache directory
.npm

# Optional eslint cache
.eslintcache

# Optional REPL history
.node_repl_history

# Output of 'npm pack'
*.tgz

# Yarn Integrity file
.yarn-integrity

# dotenv environment variables file
.env
.env.test

# parcel-bundler cache (https://parceljs.org/)
.cache

# next.js build output
.next

# nuxt.js build output
.nuxt

# vuepress build output
.vuepress/dist

# Serverless directories
.serverless/

# FuseBox cache
.fusebox/

# DynamoDB Local files
.dynamodb/
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.

# dependencies
/node_modules
/.pnp
.pnp.js

# testing
/coverage

# production
/build

# misc
.DS_Store
.env.local
.env.development.local
.env.test.local
.env.production.local

npm-debug.log*
yarn-debug.log*
yarn-error.log*

repo_name: spnati/serve

license: mit


code:
This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).

## Available Scripts

In the project directory, you can run:

### `npm start`

Runs the app in the development mode.

Open [http://localhost:3000](http://localhost:3000) to view it in the browser.

The page will reload if you make edits.

You will also see any lint errors in the console.

### `npm test`

Launches the test runner in the interactive watch mode.

See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.

### `npm run build`

Builds the app for production to the `build` folder.

It correctly bundles React in production mode and optimizes the build for the best performance.

The build is minified and the filenames include the hashes.

Your app is ready to be deployed!

See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.

### `npm run eject`

**Note: this is a one-way operation. Once you `eject`, you can’t go back!**

If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project.

Instead, it will copy all the configuration files and the transitive dependencies (Webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own.

You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it.

## Learn More

You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).

To learn React, check out the [React documentation](https://reactjs.org/).

### Code Splitting

This section has moved here: https://facebook.github.io/create-react-app/docs/code-splitting

### Analyzing the Bundle Size

This section has moved here: https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size

### Making a Progressive Web App

This section has moved here: https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app

### Advanced Configuration

This section has moved here: https://facebook.github.io/create-react-app/docs/advanced-configuration

### Deployment

This section has moved here: https://facebook.github.io/create-react-app/docs/deployment

### `npm run build` fails to minify

This section has moved here: https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify


bug-report-viewer

start: react-scripts start

build: react-scripts build

test: react-scripts test

eject: react-scripts eject

extends: react-app

browserslist:
>0.2%
not dead
not ie <= 11
not op_mini all


code:
<!DOCTYPE html>

    <meta charset="utf-8" />
    <link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico" />
    <meta
      name="viewport"
      content="width=device-width, initial-scale=1, shrink-to-fit=no"
    />
    <meta name="theme-color" content="#000000" />

    <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />

    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
    <title>SPNATI Bug Report Browser</title>

    You need to enable JavaScript to run this app.

SPNATI Bug Report Viewer


SPNATI Bug Report Viewer

short_name: SPNATI Bugs

type: image/x-icon

display: standalone

background_color: #ffffff


code:
.report-header-text {
    white-space: nowrap;
    overflow: hidden;
}

.report-header-text button {
    text-overflow: ellipsis;
}

.table-dialogue {
    font-size: 0.85rem;
}

.table-image {
    font-size: 0.85rem;
}


code:
import React, { Component } from 'react';
import {report_types, title_case, fix_dialogue_formatting, get_stripping_character, get_situation, get_file_at_commit, get_commit_data, makeCancelable, SPNATI_PROJECT_ID} from './utils.js';
import './App.css';

function EmptyTableSlot(props) {
    return (

Empty

    )
}

function TableSlot(props) {
    if (!props.table_slot) {
        return EmptyTableSlot(props);
    }

    const cur_stripping = get_stripping_character(props.report);
    var header_class = "card-header";

    if (props.report.character === props.table_slot.id) {
        header_class += ' alert alert-danger';
    } else if (!!cur_stripping && props.report.table[cur_stripping-1] && props.report.table[cur_stripping-1].id == props.table_slot.id) {
        header_class += ' alert alert-info';
    }

    var img_path = props.table_slot.currentImage;
    if (img_path) {
        if (!img_path.startsWith('opponents/')) {
            img_path = 'opponents/'+props.table_slot.id+'/'+img_path
        }

        var ref = 'master';
        if (props.report.commit && props.report.commit !== '__CI_COMMIT_SHA') {
            ref = props.report.commit;
        }

        var link_to = `https://gitgud.io/api/v4/projects/${SPNATI_PROJECT_ID}/repository/files/${encodeURIComponent(img_path)}/raw?ref=${ref}`;

        var img_link = ({props.table_slot.currentImage})
    } else {
        var img_link = (Unknown);
    }

    return (

{title_case(props.table_slot.id)} (Stage {props.table_slot.stage})

Image:

{img_link}

Dialogue:

{fix_dialogue_formatting(props.table_slot.currentLine)}

    )
}

function ReportHeader(props) {
    return (

                <button className="btn btn-link collapsed" type="button" data-toggle="collapse" data-target={"#body-"+props.report.id} aria-expanded="false" aria-controls={"body-"+props.report.id}>
                    {props.report.id}: {props.report.description}
                </button>

    );
}

class CommitData extends Component {
    constructor (props) {
        super(props);
        this.state = {commitData: null};
    }

    componentDidMount() {
        if (this.props.commit && this.props.commit !== '__CI_COMMIT_SHA') {
            this.fetchPromise = makeCancelable(get_commit_data(this.props.commit));
            this.fetchPromise.promise
                .then((data) => this.setState({commitData: data}))
                .catch(function (reason) {
                    if (!reason.isCanceled) {
                        console.error(reason);
                    }
                });
        }
    }

    componentWillUnmount() {
        if (this.fetchPromise) {
            this.fetchPromise.cancel();
        }
    }

    render() {
        if (!this.props.commit || this.props.commit === '__CI_COMMIT_SHA') {
            return (Unknown);
        } else if (!this.state.commitData) {
            return ({this.props.commit.substring(0, 10)});
        } else {
            var commitDate = new Date(this.state.commitData.committed_date);
            return ({this.props.commit.substring(0, 10)} - {this.state.commitData.title} ({commitDate.toLocaleString()}));
        }
    }

}

function BugReportType(props) {
    if (props.report.type !== 'character') {
        return ({report_types[props.report.type]});
    } else {
        return ({report_types[props.report.type]} ({title_case(props.report.character)}));
    }
}

function ReportBody(props) {
    var table_elems = props.report.table.map((slot, idx) => (<TableSlot key={idx} table_slot={slot} report={props.report} />));
    const report_date = new Date(props.report.date);

    return (

Bug Report {props.report.id} - {report_date.toLocaleString()}

Report Type: <BugReportType report={props.report} />

Commit: <CommitData commit={props.report.commit}/>

Situation: {get_situation(props.report)}

Description:

{props.report.description}

                    {table_elems}

    );
}

function BugReport(props) {
    return (

            <ReportHeader report={props.report} />
            <ReportBody report={props.report} />

    )
}

class CharacterIDDataList extends Component {
    constructor(props) {
        super(props);
        this.state = {collection: []};
    }

    componentDidMount() {
        get_file_at_commit('opponents/listing.xml', 'master')
            .then(resp => resp.text())
            .then((data) => {
                var ids = [];
                var parser = new DOMParser();
                var doc = parser.parseFromString(data, 'application/xml');

                doc.querySelectorAll('individuals>opponent').forEach(function (elem) {
                    var oppStatus = elem.getAttribute('status');
                    var id = elem.innerHTML;

                    if (!oppStatus || oppStatus === 'testing') {
                        ids.push(id);
                    }
                });

                this.setState({collection: ids});
            });
    }

    render() {
        var opts = this.state.collection.map((id) => (<option key={id} value={id} />));
        return (
            <datalist id={this.props.id}>
                {opts}
            </datalist>
        )
    }
}

class BugReportFilter extends Component {
    constructor(props) {
        super(props);
        this.state = {
            'status': 'open',
            'type': '',
            'character': '',
            'limit': 20,
            'page': 1,
        }

        this.handleChange = this.handleChange.bind(this);
        this.nextPage = this.nextPage.bind(this);
        this.prevPage = this.prevPage.bind(this);
        this.props.updateFilters(this.state);
    }

    nextPage(event) {
        this.setState({page: this.state.page+1});
    }

    prevPage(event) {
        if (this.state.page > 1) this.setState({page: this.state.page-1});
    }

    handleChange(event) {
        const target = event.target;
        const name = target.name;
        var value = target.value;

        if (name === 'page' || name === 'limit') {
            value = parseInt(value, 10);

            if (value <= 0) value = 1;
            if (name === 'limit' && value > 100) value = 100;
        }

        var stateUpdate = {
            [name]: value
        };

        if (name === 'character' && !!value) {
            stateUpdate.type = 'character';
        } else if (name === 'type' && !!value) {
            stateUpdate.character = '';
        }

        if (name !== 'page') {
            stateUpdate.page = 1;
        }

        this.setState(stateUpdate);
    }

    componentDidUpdate(prevProps, prevState) {
        if (Object.keys(prevState).some((k) => prevState[k] !== this.state[k])) {
            this.props.updateFilters(this.state);
        }
    }

    render() {
        return (

                <CharacterIDDataList id="character-ids"/>
                <form>

                            Filter Report Character ID:
                            <input className="form-control" type="text" value={this.state.character} placeholder="Character ID" name="character" list="character-ids" onChange={this.handleChange} />

                            Filter Report Type:
                            <select className="form-control" value={this.state.type} name="type" onChange={this.handleChange}>
                                <option value="">None</option>
                                <option value="freeze">Game Freeze</option>
                                <option value="display">Graphical Issue</option>
                                <option value="other">Other Game Issue</option>
                                <option value="character">Character Defect</option>
                                <option value="auto">Auto-Generated Report</option>
                            </select>

                            Filter Report Status:
                            <select className="form-control" value={this.state.status} name="status" onChange={this.handleChange}>
                                <option value="">None</option>
                                <option value="open">Open</option>
                                <option value="resolved">Resolved</option>
                                <option value="wontfix">Not a Bug</option>
                                <option value="invalid">Invalid / Incomplete Report</option>
                            </select>

                            Results per Page
                            <input className="form-control" type="number" value={this.state.limit} name="limit" min="1" max="100" onChange={this.handleChange} />

                            Page
                            <input className="form-control" type="number" value={this.state.page} name="page" onChange={this.handleChange} />

                            <button type="button" className="btn btn-primary mt-auto" onClick={this.prevPage}>Prev Page</button>

                            <button type="button" className="btn btn-primary mt-auto" onClick={this.nextPage}>Next Page</button>

                </form>

        )
    }
}

class BugReportCollection extends Component {
    constructor(props) {
        super(props);
        this.state = {
            collection: []
        };

        this.handleChange = this.handleChange.bind(this);
        this.fetchData = this.fetchData.bind(this);
    }

    handleChange (filterState) {
        this.fetchData(filterState);
    }

    fetchData(filters) {
        var url = `https://spnati.faraway-vision.io/data/bug_reports?limit=${filters.limit}&page=${filters.page-1}&status=${filters.status}`;

        if (filters.character) {
            url += `&character=${filters.character}&type=character`;
        } else if (filters.type) {
            url += `&type=${filters.type}`
        }

        if (this.fetchPromise) this.fetchPromise.cancel();

        this.fetchPromise = makeCancelable(fetch(url));
        this.fetchPromise.promise.then(resp => {
            return resp.json();
        }).then(resp => {
            this.setState({collection: resp});
        }).catch(function (reason) {
            if (!reason.isCanceled) console.error(reason)
        });
    }

    render() {
        const collectionItems = this.state.collection.map(report => (<BugReport key={report.id} report={report} />));
        return (

                <BugReportFilter updateFilters={this.handleChange}/>

                    {collectionItems}

        )
    }
}

class App extends Component {
  render() {
    return (

        <BugReportCollection />

    );
  }
}

export default App;


code:
import React from 'react';
import ReactDOM from 'react-dom';
import App from './App';

it('renders without crashing', () => {
  const div = document.createElement('div');
  ReactDOM.render(<App />, div);
  ReactDOM.unmountComponentAtNode(div);
});


code:
body {
  margin: 0;
  padding: 0;
  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen",
    "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue",
    sans-serif;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
}

code {
  font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New",
    monospace;
}


code:
import React from 'react';
import ReactDOM from 'react-dom';
import './index.css';
import App from './App';

ReactDOM.render(<App />, document.getElementById('root'));


code:
export const SPNATI_PROJECT_ID = 9879;

export const report_types = {
    'freeze': 'Game Freeze',
    'display': 'Graphical Display Issue',
    'other': 'Other',
    'character': 'Character Defect',
    'auto': 'Auto-Generated'
}

export function title_case(t) {
    return (t[0].toUpperCase())+t.substring(1).toLowerCase();
}

const html_strip_re = /\<[\/\\]?(?:span|div|button|a)[^\>]*\>/gm;
const script_strip_re = /\<script\>.+?\<[\\\/]script\>/gm;
const bracket_replace_re = /&(gt|lt);/gm;

export function fix_dialogue_formatting(dialogue) {
    if (!dialogue) return '';

    return dialogue
        .replace(html_strip_re, ' ')
        .replace(script_strip_re, ' ')
        .replace(bracket_replace_re, (m, g) => (g == 'gt' ? '>' : '<'));
}

export function get_stripping_character(report) {
    const cur_phase = report.circumstances.gamePhase.toLowerCase().trim();
    const cur_round = report.circumstances.currentRound;

    if (!!cur_phase) {
        if (!(cur_phase === 'deal' && cur_round >= 0) && cur_phase !== 'strip' && cur_phase !== 'continue' && cur_phase !== 'masturbate') {
            return;
        }
    }

    var last_stripped = undefined;
    if (typeof(report.circumstances.recentLoser) === 'number' && report.circumstances.recentLoser >= 0) {
        last_stripped = report.circumstances.recentLoser;
    }

    return last_stripped;
}

export function get_situation(report) {
    const visible_screen = report.circumstances.visibleScreens[0];
    const cur_phase = report.circumstances.gamePhase.toLowerCase().trim();
    const cur_round = report.circumstances.currentRound;
    const last_stripped = get_stripping_character(report);

    var last_stripped_label = 'Unknown Player';
    if (last_stripped !== undefined) {
        if (last_stripped == 0) {
            last_stripped_label = 'Human';
        } else if (report.table[last_stripped-1]) {
            last_stripped_label = title_case(report.table[last_stripped-1].id);
        }
    }

    var situation = 'Unknown'
    if (visible_screen === 'main-select-screen') {
        situation = 'Character Selection'
    } else if (cur_phase === '' || report.circumstances.gameOver) {
        situation = 'Game Over';
    } else if (cur_phase === 'deal' && cur_round < 0) {
        situation = 'Game Start';
    } else if (cur_phase === 'deal' && cur_round >= 0) {
        situation = 'After Stripping ('+last_stripped_label+')';
    } else if (cur_phase === 'exchange' || cur_phase === 'reveal') {
        situation = 'Cards Dealt';
    } else if (cur_phase === 'strip') {
        situation = 'Stripping ('+last_stripped_label+')';
    } else if (cur_phase === 'continue') {
        situation = 'Must Strip ('+last_stripped_label+')';
    } else if (cur_phase === 'masturbate') {
        situation = 'Must Forfeit ('+last_stripped_label+')';
    } else if (cur_phase.startsWith('continue')) {
        situation = 'Finishing';
    }

    if (report.circumstances.rollback) {
        situation += ' (Rolled-Back)';
    }

    return situation;
}

var cached_commit_data = {};
export function get_commit_data(ref) {
    if (!ref) {
        ref = 'master';
    }

    const url = `https://gitgud.io/api/v4/projects/${SPNATI_PROJECT_ID}/repository/commits/${ref}`;

    if (cached_commit_data[ref]) {
        return Promise.resolve(cached_commit_data[ref]);
    }

    return fetch(url).then((resp) => resp.json()).then(function (data) {
        cached_commit_data[ref] = data;
        return data;
    });
}

export function get_file_at_commit(path, ref) {
    if (!ref) {
        ref = 'master';
    }
    const url = `https://gitgud.io/api/v4/projects/${SPNATI_PROJECT_ID}/repository/files/${encodeURIComponent(path)}/raw?ref=${ref}`;

    return fetch(url);
}

export const makeCancelable = (promise) => {
  let hasCanceled_ = false;

  const wrappedPromise = new Promise((resolve, reject) => {
    promise.then(
      val => hasCanceled_ ? reject({isCanceled: true}) : resolve(val),
      error => hasCanceled_ ? reject({isCanceled: true}) : reject(error)
    );
  });

  return {
    promise: wrappedPromise,
    cancel() {
      hasCanceled_ = true;
    },
  };
};


code:
import asyncio
import json
import os
import os.path as osp
from pathlib import PurePath
import sys
import traceback
from typing import Set

MIRRORING_REPO = "/mnt/disks/data/spnati-lfs"
MIRROR_CONFIG = "/mnt/disks/data/spnati_utils_data/mirror-config.json"
SSH_KEY_BASEDIR = osp.expanduser("~/.ssh/deploy_keys/")
SSH_MIRROR_KEY = osp.expanduser("~/.ssh/mirror_key_rsa")

EMPTY_HASH = 40 * b"0"
SMUDGE_EXTS = {
    "xcf",
    "png",
    "jpg",
    "jpeg",
    "gif",
    "image",
    "psd",
    "exe",
    "zip",
}

class CommandError(Exception):
    pass

async def create_ssh_key(name):
    print('$ ssh-keygen -t rsa -N "" -f ' + SSH_KEY_BASEDIR + name)
    proc = await asyncio.create_subprocess_exec(
        "ssh-keygen",
        "-t",
        "rsa",
        "-N",
        "",
        "-C",
        "deploy-" + name + "@spnati.faraway-vision.io",
        "-f",
        SSH_KEY_BASEDIR + name,
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
        cwd=MIRRORING_REPO,
    )

    stdout, stderr = await proc.communicate()
    stderr = stderr.decode("utf-8")
    stdout = stdout.decode("utf-8")

    if proc.returncode != 0:
        raise CommandError(
            "ssh-keygen command exited abnormally with return code {}".format(
                proc.returncode
            ),
            stdout,
            stderr,
        )

    if len(stdout) > 0:
        print(stdout)

    return proc.returncode, stdout

async def run_git_command(
    args, raise_on_nonzero=True, stdin=None, decode=True, display_stdout=True
):
    print("$ git " + " ".join(args))
    proc = await asyncio.create_subprocess_exec(
        "git",
        *args,
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
        cwd=MIRRORING_REPO
    )

    stdout, stderr = await proc.communicate(stdin)
    stderr = stderr.decode("utf-8")
    if decode:
        stdout = stdout.decode("utf-8")

    if raise_on_nonzero and proc.returncode != 0:
        raise CommandError(
            "git command `{}` exited abnormally with return code {}".format(
                " ".join(args), proc.returncode
            ),
            stdout,
            stderr,
        )

    if len(stdout) > 0 and display_stdout:
        print(stdout)

    return proc.returncode, stdout

async def reset_ops_repo():
    for _ in range(5):
        try:
            await run_git_command(["fetch", "origin"])
            break
        except Exception:
            traceback.print_exc()
            print("Retrying...")

    await run_git_command(["checkout", "-fB", "master", "origin/master"])
    await run_git_command(["lfs", "pull", "origin"])

async def remote_exists(username):
    status, _ = await run_git_command(
        ["ls-remote", "--exit-code", username, "master"], raise_on_nonzero=False
    )
    return status == 0

async def ensure_remote_exists(username):
    exists = await remote_exists(username)
    if not exists:
        await run_git_command(
            [
                "remote",
                "add",
                username,
                "git@ssh.gitgud.io:" + username + "/spnati.git",
            ],
            raise_on_nonzero=False,
        )

        await run_git_command(
            [
                "fetch",
                username,
            ],
            raise_on_nonzero=False,
        )

async def init_remote_sync(username):
    print("Initializing repository sync for: " + username)

    if not osp.isfile(SSH_KEY_BASEDIR + username):
        print("Creating SSH deployment key for " + username)
        await create_ssh_key(username)

    os.environ["GIT_SSH_COMMAND"] = "ssh -i " + SSH_KEY_BASEDIR + username

    try:
        await ensure_remote_exists(username)
        return True
    except CommandError as e:
        print("Sync initialization for " + username + " failed")
        sys.stderr.write(e.args[2] + "\n")
        sys.stderr.flush()

        return False

async def sync_with_remote(username):
    print("Synchronizing repository for: " + username)

    os.environ["GIT_SSH_COMMAND"] = "ssh -i " + SSH_KEY_BASEDIR + username

    try:
        await run_git_command(["push", username, "master:master"])
        await run_git_command(["lfs", "push", username, "master"])

        return True
    except CommandError as e:
        print("Sync for " + username + " failed")
        sys.stderr.write(e.args[2] + "\n")
        sys.stderr.flush()

        return False

async def get_changed_blobs(oid: str) -> Set[str]:
    cmd_args = ["diff-tree", "-r", "-z", oid + "^", oid]
    blobs = set()

    _, data = await run_git_command(cmd_args, decode=False, display_stdout=False)
    for record in data.split(b":"):
        if len(record) == 0:
            continue

        parts = record.split(b"\x00")[:-1]
        main_line = parts[0].split(b" ")
        src_path = PurePath(parts[1].decode("utf-8"))

        if len(parts) == 3:
            dst_path = PurePath(parts[2].decode("utf-8"))
        else:
            dst_path = None

        if main_line[3] != EMPTY_HASH and (
            dst_path is None or dst_path.suffix[1:].lower() in SMUDGE_EXTS
        ):
            blobs.add(main_line[3].decode("utf-8"))

        if main_line[2] != EMPTY_HASH and src_path.suffix[1:].lower() in SMUDGE_EXTS:
            blobs.add(main_line[2].decode("utf-8"))

    return blobs

async def smudge_worker(worker_name: str, queue: asyncio.Queue):
    while True:
        oid = await queue.get()
        try:
            _, blob_data = await run_git_command(
                ["cat-file", "blob", oid], decode=False, display_stdout=False
            )

            if blob_data is not None:
                await run_git_command(["lfs", "smudge"], stdin=blob_data, decode=False)
        except (AttributeError, Exception):
            print("[{:s}] smudge error for {:s}".format(worker_name, oid))

        queue.task_done()

async def lfs_check(start_rev: str):
    print("Scanning for updated LFS files to pull...")

    _, data = await run_git_command(["rev-list", start_rev + "..master"])
    commits = data.splitlines()

    print("Scanning {:d} commits...".format(len(commits)))

    all_blobs = set()
    for commit in filter(lambda x: len(x) > 0, map(str.strip, commits)):
        try:
            blobs = await get_changed_blobs(commit)
            all_blobs.update(blobs)
        except Exception:
            print("failed to scan " + commit)

    print("Smudging {:d} blobs...".format(len(all_blobs)))

    blob_queue = asyncio.Queue()
    for blob in all_blobs:
        blob_queue.put_nowait(blob)

    workers = []
    for i in range(20):
        workers.append(
            asyncio.create_task(smudge_worker("worker-" + str(i), blob_queue))
        )

    await blob_queue.join()
    for worker in workers:
        worker.cancel()

    await asyncio.gather(*workers, return_exceptions=True)

async def get_pointer_oid(pointer_oid) -> str:
    _, blob_data = await run_git_command(
        ["cat-file", "blob", pointer_oid], display_stdout=False, decode=False
    )

    if blob_data is None or not blob_data.startswith(b"version https://"):
        return

    if isinstance(blob_data, bytes):
        blob_data = blob_data.decode("utf-8")

    for line in blob_data.splitlines():
        k, v = line.split(" ", 1)
        if k == "oid":
            _, oid = v.split(":", 1)
            return oid

async def lfs_push_worker(worker_name: str, username: str, queue: asyncio.Queue):
    while True:
        oid_group = await queue.get()

        try:
            args = ["lfs", "push", "--object-id", username]
            args.extend(oid_group)
            await run_git_command(args, decode=False)
        except Exception:
            print(
                "[{:s}] lfs push error for {:s}, oids {:s}".format(
                    worker_name, username, ", ".join(oid_group)
                )
            )

        queue.task_done()

async def lfs_blob_scan_worker(worker_name: str, in_queue: asyncio.Queue, out_set: set):
    while True:
        blob_oid = await in_queue.get()

        try:
            lfs_oid = await get_pointer_oid(blob_oid)
            if lfs_oid is not None:
                out_set.add(lfs_oid)
        except Exception:
            print("[{:s}] blob scan error for {:s}".format(worker_name, blob_oid))

        in_queue.task_done()

async def lfs_push_to_fork(username: str):
    print("Scanning for LFS objects to push to {:s}...".format(username))

    os.environ["GIT_SSH_COMMAND"] = "ssh -i " + SSH_KEY_BASEDIR + username

    _, data = await run_git_command(["rev-list", username + "/master..master"])
    commits = data.splitlines()

    print("Scanning {:d} commits...".format(len(commits)))

    all_blobs = set()
    all_lfs_oids = set()

    blob_oid_queue = asyncio.Queue()
    scan_workers = []
    push_workers = []

    for i in range(20):
        scan_workers.append(
            asyncio.create_task(
                lfs_blob_scan_worker(
                    "scan-worker-" + str(i), blob_oid_queue, all_lfs_oids
                )
            )
        )

    for commit in filter(lambda x: len(x) > 0, map(str.strip, commits)):
        try:
            blobs = await get_changed_blobs(commit)
            for blob_oid in blobs:
                if blob_oid in all_blobs:
                    continue
                all_blobs.add(blob_oid)
                blob_oid_queue.put_nowait(blob_oid)
        except Exception:
            print("failed to scan commit " + commit)

    try:
        await asyncio.wait_for(blob_oid_queue.join(), timeout=300)
    except asyncio.TimeoutError:
        pass

    for worker in scan_workers:
        worker.cancel()

    lfs_oid_queue = asyncio.Queue()
    cur_lfs_oid_group = []
    for lfs_oid in all_lfs_oids:
        cur_lfs_oid_group.append(lfs_oid)
        if len(cur_lfs_oid_group) > 20:
            lfs_oid_queue.put_nowait(cur_lfs_oid_group)
            cur_lfs_oid_group = []

    if len(cur_lfs_oid_group) > 0:
        lfs_oid_queue.put_nowait(cur_lfs_oid_group)

    for i in range(5):
        push_workers.append(
            asyncio.create_task(
                lfs_blob_scan_worker("push-worker-" + str(i), username, lfs_oid_queue)
            )
        )

    try:
        await asyncio.wait_for(lfs_oid_queue.join(), timeout=300)
    except asyncio.TimeoutError:
        pass

    for worker in push_workers:
        worker.cancel()

    print("Pushed {:d} LFS objects to {:s}...".format(len(all_lfs_oids), username))

async def main():
    print("Getting current master commit ID...")
    _, master_id = await run_git_command(["rev-parse", "master"])
    master_id = master_id.strip()

    print("Synchronizing mirroring repository with upstream...")
    try:
        await reset_ops_repo()
    except CommandError as e:
        print("Repository reset failed")
        sys.stderr.write(e.args[2] + "\n")
        sys.stderr.flush()
        sys.exit(1)

    await lfs_check(master_id)
    status = True

    if len(sys.argv) > 1:
        for username in sys.argv[1:]:
            if username == "origin":
                continue

            try:
                init_status = await init_remote_sync(username)
                if not init_status:
                    status = False
                    continue

                await lfs_push_to_fork(username)
                status = status and (await sync_with_remote(username))
            except CommandError as e:
                print("Sync for {} failed".format(username))
                sys.stderr.write(e.args[2] + "\n")
                sys.stderr.flush()
    else:
        # sync Gitea:
        try:
            os.environ["GIT_SSH_COMMAND"] = "ssh -i " + SSH_MIRROR_KEY
            await run_git_command(["push", "backup-site", "master:master"])
            await run_git_command(["lfs", "push", "backup-site", "master"])
        except CommandError as e:
            print("Sync to Gitea failed")
            sys.stderr.write(e.args[2] + "\n")
            sys.stderr.flush()
            status = False

        with open(MIRROR_CONFIG, "r", encoding="utf-8") as f:
            config = json.load(f)

        for username in config:
            if username == "origin":
                continue

            try:
                init_status = await init_remote_sync(username)
                if not init_status:
                    status = False
                    continue

                await lfs_push_to_fork(username)
                user_status = await sync_with_remote(username)
                status = status and user_status
            except CommandError as e:
                print("Sync for {} failed".format(username))
                sys.stderr.write(e.args[2] + "\n")
                sys.stderr.flush()

    if status:
        print("All repositories synchronized.")
        sys.exit(0)
    else:
        sys.exit(1)

if __name__ == "__main__":
    asyncio.run(main())


code:
import asyncio
import aiohttp
import aioredis
import sys
import os
import os.path as osp
from yaswfp import swfparser
import hashlib
from pathlib import Path
import json
import concurrent.futures
from datetime import datetime, timezone
import shutil
import tempfile
import subprocess as sp
import re

kisekae_save_directory = "/mnt/disks/data/kisekae_versions"
if len(sys.argv) > 1:
    kisekae_save_directory = sys.argv[1]

url_template = "http://pochi.lix.jp/k_kisekae2_swf/{:s}.swf"
main_swf_url = "http://pochi.lix.jp/k_kisekae2.swf"

async def save_to_file(session, url, filename):
    for retries in range(5):
        try:
            async with session.get(url) as resp:
                if not resp.status < 300:
                    print(
                        "Error: request for {:s} errored with code {:d}".format(
                            url, resp.status
                        )
                    )
                    return False
                else:
                    print("Retrieving: {} ==> {}".format(url, filename))
                    with open(filename, "wb") as fd:
                        while True:
                            chunk = await resp.content.read(128)
                            if not chunk:
                                return True
                            fd.write(chunk)
                    return
        except aiohttp.client_exceptions.ClientPayloadError:
            continue

def split_array_items(abc_data, start_index, end_index):
    array_data = abc_data[start_index:end_index]

    items = []
    cur_bytes = bytearray()

    for b in array_data:
        if b < 0x20 or b > 0x7E:
            items.append(cur_bytes.decode())
            cur_bytes = bytearray()
        else:
            cur_bytes.append(b)

    items.append(cur_bytes.decode())

    return items

def get_loadnames(swf_file):
    with tempfile.TemporaryDirectory() as td:
        sp.run(
            ["ffdec", "-selectclass", "Main", "-export", "script", str(td), swf_file],
            check=True,
        )

        with Path(td).joinpath("scripts", "Main.as").open("r", encoding="utf-8") as f:
            data = f.read()
            m = re.search(r"loadName\:Array\s*=\s*new Array\(([^)]+)\)", data)

            if m is None:
                return

            loadNames = list(w[1] for w in re.finditer(r"\"(\w+)\"", m[1]))

            m = re.search(r"loadItemName\:Array\s*=\s*new Array\(([^)]+)\)", data)
            if m is None:
                return

            loadItemNames = list(w[1] for w in re.finditer(r"\"(\w+)\"", m[1]))

            m = re.search(r"menuName\:String\s*=\s*\"([^\"]+)\"", data)
            if m is None:
                return
            loadNames.append(m.group(1))

            m = re.search(r"loadName\:String\s*=\s*\"([^\"]+)\"", data)
            if m is None:
                return
            loadNames.append(m.group(1))

            return loadNames, loadItemNames

async def fetch_kisekae(out_dir):
    os.makedirs(osp.join(out_dir, "k_kisekae2_swf"))
    async with aiohttp.ClientSession() as session:
        main_swf_path = osp.join(out_dir, "k_kisekae2.swf")
        await save_to_file(session, main_swf_url, main_swf_path)

        shutil.copyfile(main_swf_path, Path(kisekae_save_directory) / "k_kisekae2.swf")

        loadNames, loadItemNames = get_loadnames(main_swf_path)

        loadNames = list(filter(lambda n: len(n.strip()) > 0, loadNames))
        loadItemNames = list(filter(lambda n: len(n.strip()) > 0, loadItemNames))

        fnames = map(
            lambda name: (
                url_template.format(name.strip()),
                osp.join(out_dir, "k_kisekae2_swf", name.strip() + ".swf"),
            ),
            loadNames + loadItemNames,
        )
        aws = list(map(lambda x: save_to_file(session, *x), fnames))

        return await asyncio.gather(*aws)

def hash_file(fname):
    h = hashlib.sha1()

    with open(fname, "rb") as f:
        h.update(f.read())

    return (h.digest(), h.hexdigest())

def construct_manifest(kk_dir):
    d = Path(kk_dir)
    swf_dir = d / "k_kisekae2_swf"

    manifest = dict()

    main_digest, main_hexdigest = hash_file(d / "k_kisekae2.swf")

    m = hashlib.sha1()
    m.update(main_digest)

    manifest["k_kisekae2.swf"] = main_hexdigest
    manifest["k_kisekae2_swf"] = dict()

    subdigests = {}

    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = {
            executor.submit(hash_file, fname.resolve()): fname
            for fname in swf_dir.iterdir()
        }

        for future in concurrent.futures.as_completed(futures):
            fname = futures[future]
            digest, hexdigest = future.result()

            subdigests[fname.name] = (digest, hexdigest)

            manifest["k_kisekae2_swf"][fname.name] = hexdigest

    for k in sorted(subdigests.keys()):
        digest, hexdigest = subdigests[k]
        m.update(digest)

        print("k_kisekae2_swf/{}: {}".format(k, hexdigest))

    metadigest = m.hexdigest()

    print("Overall file metadigest: {}".format(metadigest))

    manifest["meta"] = metadigest
    manifest["date"] = datetime.now(timezone.utc).isoformat()

    return manifest

async def check_for_update():
    with tempfile.TemporaryDirectory() as td:
        print("Downloading latest KK to {}".format(td))
        await fetch_kisekae(td)

        manifest = construct_manifest(td)

        save_path = Path(kisekae_save_directory) / manifest["meta"]
        if save_path.is_dir():
            print("No updates detected.")
            return
        else:
            print("Update detected (metahash: {})".format(manifest["meta"]))
            shutil.copytree(td, save_path)

            with open(save_path / "manifest.json", "w", encoding="utf-8") as mfile:
                json.dump(manifest, mfile)

            print("New KK version copied to {}".format(str(save_path)))

            if len(sys.argv) == 1:
                redis_pub = await aioredis.create_redis("redis://localhost")
                await redis_pub.publish_json(
                    "utilities:notifications",
                    {
                        "type": "notify",
                        "level": "log",
                        "file": str(save_path / "manifest.json"),
                        "message": "<@!135165531908079616>\nNew online Kisekae version detected! (meta-hash: `{}`)".format(
                            manifest["meta"]
                        ),
                    },
                )

if __name__ == "__main__":
    asyncio.run(check_for_update())


code:
#!/usr/bin/env python3

from aiohttp import web
from aiohttp.hdrs import ACCEPT

from aioprometheus import REGISTRY, Counter, Summary, timer
from aioprometheus.renderer import render

app = web.Application()
app.opp_requests_counter = Counter(
    "spnati_opponent_behavior_requests_total", "Number of behaviour.xml requests."
)
app.costume_requests_counter = Counter(
    "spnati_costume_requests_total", "Number of costume.xml requests."
)
app.request_time_collector = Summary(
    "spnati_exporter_request_seconds", "Time spent processing exporter requests"
)

@timer(app.request_time_collector)
async def handle_opp_request(request):
    app.opp_requests_counter.inc({"id": request.match_info["id"]})
    return web.Response(text="", status=204)

@timer(app.request_time_collector)
async def handle_costume_request(request):
    app.costume_requests_counter.inc({"id": request.match_info["id"]})
    return web.Response(text="", status=204)

@timer(app.request_time_collector)
async def force_empty(request):
    return web.Response(text="", status=204)

async def handle_metrics(request):
    content, http_headers = render(REGISTRY, request.headers.getall(ACCEPT, []))
    return web.Response(body=content, headers=http_headers)

app.add_routes(
    [
        web.get("/opponents/{id}/behaviour.xml", handle_opp_request),
        web.get("/opponents/{id}/behaviour.xml.gz", handle_opp_request),
        web.get("/opponents/reskins/{id}/costume.xml", handle_costume_request),
        web.get("/opponents/reskins/{id}/costume.xml.gz", handle_costume_request),
        web.get("/metrics", handle_metrics),
        web.get("/{tail:.*}", force_empty),
    ]
)

web.run_app(app, port=9155)


code:
import asyncio
from spnati_utilities_bot.main import main

if __name__ == "__main__":
    main()


code:
# Database configuration-- this config assumes MongoDB and Redis are running
# on the host (using default Docker settings)
REDIS_URL = "redis://172.17.0.1:6379/0"
REDIS_PUBSUB_URL = "redis://172.17.0.1:6379"
MONGO_HOST = "172.17.0.1"

# URL for kisekae server (this works for the given docker-compose setup)
KISEKAE_BASE_URL = "http://kisekae_server:8080"

# Directory where docker images will be placed
# (this works for the given docker-compose setup)
KKL_IMAGE_ATTACH_DIR = "/var/kkl_images"

# API key for requests to Gitlab instances
GIT_API_KEY = ""

# Sentry error reporting DSN
SENTRY_DSN = ""

# Discord webhook URLs for repo event notifications
PUSH_WEBHOOK_URLS = []
NOTE_WEBHOOK_URLS = []
ISSUE_WEBHOOK_URLS = []
MR_WEBHOOK_URLS = []

# Secret used to authenticate incoming webhook requests from Gitlab instances
# (the X-Gitlab-Token header for incoming repo event webhooks needs to match
#  this value for the event to be accepted)
WEBHOOK_TOKEN = ""

# Secret used to sign authentication cookies
COOKIE_SIGNER_KEY = ""

# OAuth2 authentication flow secrets and IDs (given by Discord)
DISCORD_CLIENT_ID = ""
DISCORD_CLIENT_SECRET = ""

# Target URI for completing OAuth2 flow
OAUTH2_REDIRECT_URI = ""

# User is redirected to this URL after login is complete
# (after redirection to OAUTH2_REDIRECT_URI)
LOGIN_REDIRECT_TARGET = ""

MODERATORS = []


code:
import csv
import math
import sys

import numpy as np
from matplotlib import pyplot as plt

import classifier

def compute_roc_point(data, threshold):
    true_positive = 0
    false_positive = 0
    condition_positive = 0
    condition_negative = 0

    for is_spam, score in data:
        if is_spam:
            condition_positive += 1
        else:
            condition_negative += 1

        if score < threshold:
            # Positive detection:
            if is_spam:
                true_positive += 1
            else:
                false_positive += 1

    tpr = true_positive / condition_positive
    fpr = false_positive / condition_negative

    return (tpr, fpr)

def analyze_corpus(corpus_file, scores):
    data = []

    print("Running classification...")
    with open(corpus_file, "r", encoding="utf-8") as f:
        for row in csv.DictReader(f):
            is_spam = row["spam"] == "True"
            score = classifier.classify(row["text"], scores)
            if score is None:
                continue

            data.append((is_spam, score))

    print("Plotting ROC...")
    thresholds = list(np.linspace(0.000001, 1.0, 20, endpoint=False))

    x = [0]
    y = [0]
    for p in thresholds:
        t = math.log(p) - math.log(1 - p)
        tpr, fpr = compute_roc_point(data, t)

        print("p={:.3f}, t={:6.3f} - tpr={:.3f} / fpr={:.3f}".format(p, t, tpr, fpr))

        x.append(fpr)
        y.append(tpr)

    x.append(1)
    y.append(1)
    thresholds.append(1)

    plt.plot(x, y, "r-")
    plt.plot(x, y, "b.")

    plt.plot(thresholds, thresholds)
    plt.show()

if __name__ == "__main__":
    scores = classifier.load_scores(sys.argv[1])
    analyze_corpus(sys.argv[2], scores)


code:
import csv
import re
from typing import Dict, List
import unicodedata

BOUNTY_ID_REGEX = r"\!\!(\d{6})"
URL_REGEX = r"https?://\S+"
CORRECTION_ARROW = r"\S+\s*[\-\=]*\>\s*\S+"

def load_scores(file) -> Dict[str, float]:
    scores = {}

    with open(file, "r", encoding="utf-8") as f:
        reader = csv.DictReader(f)

        for row in reader:
            scores[row["feature"]] = float(row["score"])

    return scores

def clean(text) -> str:
    # Remove bounty IDs and (most) URLs
    text = re.sub(BOUNTY_ID_REGEX, "", text)
    text = re.sub(URL_REGEX, "", text)

    # Perform unicode normalization
    text = unicodedata.normalize("NFKD", text)

    # Remove "{}" and contents
    l = text.find("{")
    r = text.rfind("}")
    if (l > 0) and (r > 0) and (l < r):
        text = text[:l] + text[r + 1 :]

    # Strip everything except token contents
    text = re.sub(r"[^a-zA-Z\'\s\,\.\?\!\*\"\']", "", text).strip()

    return text

def split(text) -> List[str]:
    for m in re.finditer(
        r"(?:[a-zA-Z](?:[a-zA-Z\']*[a-zA-Z])?)|,+|\.+|\?+|!+|\*+|[\"\']+", text
    ):
        word = m[0]

        if len(word) == 0:
            continue

        yield word

STRANGE_CHARS = r"[]}{|:\\/<"

def extract_features(text):
    feats = set()
    if re.search(BOUNTY_ID_REGEX, text) is not None:
        feats.add("bounty_id")

    if re.search(CORRECTION_ARROW, text) is not None:
        feats.add("correction_arrow")

    for c in STRANGE_CHARS:
        if c in text:
            feats.add("strange_char")

    words = []
    for word in split(clean(text)):
        init = word[0]
        if init.isupper():
            feats.add("capitalization")

        words.append(word.casefold())

    if len(words) == 0:
        return

    feats.add("bigram:[START]_" + words[0])
    feats.add("bigram:" + words[-1] + "_[END]")
    for i in range(len(words) - 1):
        feats.add("bigram:" + words[i] + "_" + words[i + 1])

    for word in words:
        feats.add("word:" + word)

    return feats

def classify(text, scores) -> float:
    score = 0
    features = extract_features(text)

    if features is None:
        return

    for feature in filter(lambda f: f in scores, features):
        score += scores[feature]

    return score