"""
Imagelab is the core class in CleanVision for finding all types of issues in an image dataset.
The methods in this module should suffice for most use-cases,
but advanced users can get extra flexibility via the code in other CleanVision modules.
"""
from __future__ import annotations

import random
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, TypeVar

import numpy as np
import pandas as pd
from PIL import Image

import cleanvision
from cleanvision.dataset.torch_dataset import TorchDataset
from cleanvision.dataset.utils import build_dataset
from cleanvision.issue_managers import (
    ISSUE_MANAGER_REGISTRY,
    IssueManagerFactory,
    IssueType,
)
from cleanvision.utils.base_issue_manager import IssueManager
from cleanvision.utils.constants import (
    DEFAULT_ISSUE_TYPES,
    DUPLICATE,
    DUPLICATE_ISSUE_TYPES_LIST,
    IMAGE_PROPERTY,
    IMAGE_PROPERTY_ISSUE_TYPES_LIST,
    SETS,
)
from cleanvision.utils.serialize import Serializer
from cleanvision.utils.utils import (
    deep_update_dict,
    get_is_issue_colname,
    get_max_n_jobs,
    get_score_colname,
    update_df,
)
from cleanvision.utils.viz_manager import VizManager

if TYPE_CHECKING:  # pragma: no cover
    import datasets
    from torchvision.datasets.vision import VisionDataset

__all__ = ["Imagelab"]
TImagelab = TypeVar("TImagelab", bound="Imagelab")

class Imagelab:
    """A single class to find all types of issues in image datasets.
    Imagelab detects issues in any image dataset and thus can be useful in most computer vision tasks including
    supervised and unsupervised training.
    Imagelab supports various formats for datasets: local folder containing images, a list of image
    filepaths, HuggingFace dataset and Torchvision dataset.
    Specify only one of these arguments: `data_path`, `filepaths`, (`hf_dataset`, `image_key`), `torchvision_dataset`

    Parameters
    ----------
    data_path : str
        Path to image files.
        Imagelab will recursively retrieve all image files from the specified path

    filepaths: List[str], optional
        Issue checks will be run on this list of image paths specified in `filepaths`.

    hf_dataset: datasets.Dataset
        Hugging Face dataset with images in PIL format accessible via some key in ``hf_dataset.features``.

    image_key: str
        Key used to access images within the Hugging Face `dataset.features` object. For many datasets, this key is just called "image".
        This argument must be specified if you provide a Hugging Face dataset; for other types of dataset this argument has no effect.

    torchvision_dataset: torchvision.datasets.vision.VisionDataset
        torchvision dataset where each individual  example is a tuple containing exactly one image in PIL format.

    Attributes
    ----------
    issues : pd.DataFrame
        Dataframe where each row corresponds to an image and columns specify which issues were detected in this image.
        It has two types of columns for each issue type:

        1. <issue_type>_score - This column contains a quality-score for each image for a particular type of issue.
        Scores are between 0 and 1, lower values indicate images exhibiting more severe instances of this issue.

        2. is_<issue_type>_issue - This column indicates whether or not the issue_type is detected in each image (a binary decision rather than numeric score).

    issue_summary : pd.DataFrame
        Dataframe where each row corresponds to a type of issue and columns summarize the overall prevalence of this issue in the dataset.
        Specifically, it shows the number of images detected with the issue.

    info : Dict
        Nested dictionary that contains statistics and other useful information about the dataset.
        Also contains additional information saved while checking for issues in the dataset.

    Raises
    ------
    ValueError
        If no images are found in the specified paths.
        If both `data_path` and `filepaths` are given or none of them are specified.

    Examples
    --------

    Basic usage of Imagelab class

    .. code-block:: python

        from cleanvision import Imagelab
        imagelab = Imagelab(data_path="FOLDER_WITH_IMAGES/")
        imagelab.find_issues()
        imagelab.report()

    """

    def __init__(
        self,
        data_path: Optional[str] = None,
        filepaths: Optional[List[str]] = None,
        hf_dataset: Optional["datasets.Dataset"] = None,
        image_key: Optional[str] = None,
        torchvision_dataset: Optional["VisionDataset"] = None,
        storage_opts: Dict[str, Any] = {},
    ) -> None:
        self._dataset = build_dataset(
            data_path,
            filepaths,
            hf_dataset,
            image_key,
            torchvision_dataset,
            storage_opts=storage_opts,
        )
        if len(self._dataset) == 0:
            raise ValueError("No images found in the dataset specified")
        self.info: Dict[str, Any] = {"statistics": {}}
        self.issue_summary: pd.DataFrame = pd.DataFrame(
            columns=["issue_type", "num_images"]
        ).astype({"issue_type": str, "num_images": np.int64})

        self.issues: pd.DataFrame = pd.DataFrame(index=self._dataset.index)
        self._issue_types: List[str] = []
        self._issue_managers: Dict[str, IssueManager] = {}

        # TODO: can be loaded from a file later
        self._config: Dict[str, Any] = self._set_default_config()
        self.cleanvision_version: str = cleanvision.__version__

    def _set_default_config(self) -> Dict[str, Any]:
        """Sets default values for various config variables used in Imagelab class
        The naming convention for methods is {method_name}_{config_variable_name}

        Returns
        -------
        Dict[str, Any]
            Returns a dict with keys as config variables and their values as dict values
        """
        return {
            "visualize_num_images_per_row": 4,
            "report_num_images": 4,
            "report_max_prevalence": 0.5,
            "report_cell_size": (2, 2),
        }

    @staticmethod
    def list_default_issue_types() -> List[str]:
        """Returns a list of the issue types that are run by default in :py:meth:`Imagelab.find_issues`"""
        return DEFAULT_ISSUE_TYPES

    @staticmethod
    def list_possible_issue_types() -> List[str]:
        """Returns a list of all the possible issue types that can be run in :py:meth:`Imagelab.find_issues`
        This list will also include custom issue types if properly added.
        """
        issue_types = Imagelab.list_default_issue_types()
        for key in ISSUE_MANAGER_REGISTRY:
            if key not in [IMAGE_PROPERTY, DUPLICATE]:
                issue_types.append(key)
        return list(set(issue_types))

    def _get_issues_to_compute(
        self, issue_types_with_params: Optional[Dict[str, Any]]
    ) -> Dict[str, Any]:
        if not issue_types_with_params:
            to_compute_issues_with_params: Dict[str, Any] = {
                issue_type: {} for issue_type in self.list_default_issue_types()
            }
        else:
            to_compute_issues_with_params = {
                issue_type_str: params
                for issue_type_str, params in issue_types_with_params.items()
            }
        return to_compute_issues_with_params

    def find_issues(
        self,
        issue_types: Optional[Dict[str, Any]] = None,
        n_jobs: Optional[int] = None,
        verbose: bool = True,
    ) -> None:
        """Finds issues in the dataset.
        If `issue_types` is not provided, dataset is checked for a default set of issue types.
        To see default set: :py:meth:`Imagelab.list_default_issue_types`

        Parameters
        ----------
        issue_types : Dict[str, Any], optional
            Dict with issue types to check as keys.
            The value of this dict is a dict containing hyperparameters for each issue type.
        n_jobs :  int, default=None
            Number of processing threads used by multiprocessing.
            Default None sets to the number of cores on your CPU (physical cores if you have psutil package installed, otherwise logical cores).
            Set this to 1 to disable parallel processing (if its causing issues). Windows users may see a speed-up with n_jobs=1.
        verbose : bool, default=True
            If True, prints helpful information while checking for issues.

        Examples
        --------
        To check for all default issue types use

        .. code-block:: python

            imagelab.find_issues()

        To check for specific issue types with default settings

        .. code-block:: python

            issue_types = {
                "dark": {},
                "blurry": {}
            }
            imagelab.find_issues(issue_types)

        To check for issue types with different hyperparameters. Different issue types can have different hyperparameters.

        .. code-block:: python

            issue_types = {
                "dark": {"threshold": 0.1},
                "blurry": {}
            }
            imagelab.find_issues(issue_types)

        """
        to_compute_issues_with_params = self._get_issues_to_compute(issue_types)
        if verbose:
            print(
                f"Checking for {', '.join([issue_type for issue_type in to_compute_issues_with_params.keys()])} images ..."
            )

        # update issue_types
        self._issue_types = list(
            set(self._issue_types).union(set(to_compute_issues_with_params.keys()))
        )

        # set issue managers
        issue_type_groups = self._get_issue_type_groups(to_compute_issues_with_params)
        self._set_issue_managers(issue_type_groups)

        # set number of jobs for parallelizing computation
        if n_jobs is None:
            n_jobs = get_max_n_jobs()
            if isinstance(self._dataset, TorchDataset):
                n_jobs = 1

        # find issues
        for issue_type_group, params in issue_type_groups.items():
            issue_manager = self._issue_managers[issue_type_group]
            issue_manager.find_issues(
                params=params,
                dataset=self._dataset,
                imagelab_info=self.info,
                n_jobs=n_jobs,
            )

            # update issues, issue_summary and info
            self._update_issues(issue_manager.issues)
            self._update_issue_summary(issue_manager.summary)
            self._update_info(issue_manager.info)

        self.issue_summary = self.issue_summary.sort_values(
            by=["num_images"], ascending=False
        )
        self.issue_summary = self.issue_summary.reset_index(drop=True)
        if verbose:
            print(
                f"Issue checks completed. {self.issue_summary['num_images'].sum()} issues found in the dataset. To see a detailed report of issues found, use imagelab.report()."
            )
        return

    def _update_info(self, issue_manager_info: Dict[str, Any]) -> None:
        deep_update_dict(self.info, issue_manager_info)

    def _update_issue_summary(self, issue_manager_summary: pd.DataFrame) -> None:
        # Remove results for issue types computed again
        self.issue_summary = self.issue_summary[
            ~self.issue_summary["issue_type"].isin(issue_manager_summary["issue_type"])
        ]
        # concat new results
        self.issue_summary = pd.concat(
            [self.issue_summary, issue_manager_summary], axis=0, ignore_index=True
        )

    def _update_issues(self, issue_manager_issues: pd.DataFrame) -> None:
        self.issues = update_df(self.issues, issue_manager_issues)

    def _get_issue_type_groups(
        self, issue_types_with_params: Dict[str, Any]
    ) -> Dict[str, Any]:
        issue_type_groups = {}

        for issue_type, params in issue_types_with_params.items():
            group_name = None
            if issue_type in IMAGE_PROPERTY_ISSUE_TYPES_LIST:
                group_name = IMAGE_PROPERTY
            elif issue_type in DUPLICATE_ISSUE_TYPES_LIST:
                group_name = DUPLICATE
            else:
                issue_type_groups[issue_type] = params

            if group_name:
                if issue_type_groups.get(group_name):
                    issue_type_groups[group_name][issue_type] = params
                else:
                    issue_type_groups[group_name] = {issue_type: params}
        return issue_type_groups

    def _set_issue_managers(self, issue_type_groups: Dict[str, Any]) -> None:
        for issue_type_group, params in issue_type_groups.items():
            self._issue_managers[issue_type_group] = IssueManagerFactory.from_str(
                issue_type_group
            )()

    def _filter_report(
        self, issue_types: List[str], max_prevalence: float
    ) -> List[str]:
        """Filters issues by max_prevalence in the dataset."""
        issue_summary = self.issue_summary[
            self.issue_summary["issue_type"].isin(issue_types)
        ]
        issue_to_report = []
        for row in issue_summary.itertuples(index=False):
            if getattr(row, "num_images") / len(self._dataset) < max_prevalence:
                issue_to_report.append(getattr(row, "issue_type"))
            else:
                print(
                    f"Removing {getattr(row, 'issue_type')} from potential issues in the dataset as it exceeds "
                    f"max_prevalence={max_prevalence} "
                )
        return issue_to_report

    def _get_report_args(self, user_supplied_args: Dict[str, Any]) -> Dict[str, Any]:
        report_args = {
            "max_prevalence": self._config["report_max_prevalence"],
            "num_images": self._config["report_num_images"],
            "cell_size": self._config["report_cell_size"],
        }

        non_none_args = {
            arg: value for arg, value in user_supplied_args.items() if value is not None
        }

        return {**report_args, **non_none_args}

    def report(
        self,
        issue_types: Optional[List[str]] = None,
        max_prevalence: Optional[float] = None,
        num_images: Optional[int] = None,
        verbosity: int = 1,
        print_summary: bool = True,
        show_id: bool = False,
    ) -> None:
        """Prints summary of the issues found in your dataset.
        By default, this method depicts the images representing top-most severe instances of each issue type.

        Parameters
        ----------
        issue_types : List[str], optional
            List of issue types to consider in report.
            This must be subset of the issue types specified in :py:meth:`Imagelab.find_issues``.

        max_prevalence : float, default=0.5
            Value between 0 and 1
            Issue types that are detected in more than `max_prevalence` fraction of the images in dataset will be omitted from the report.
            You are presumably already aware of these in your dataset.

        num_images : int, default=4
            Maximum number of images to show for issue type reported. These are examples of the top-most severe instances of the issue in your dataset.

        verbosity : int, {1, 2, 3, 4}
            Increasing verbosity increases the detail of the report. Set this to 1 to report less information, or to 4 to report the most information.

        print_summary : bool, default=True
            If True, prints the summary of issues found in the dataset.

        show_id: bool, default=False
            If True, prints the dataset ID of each image shown in the report.

        Examples
        --------
        Default usage

        .. code-block:: python

            imagelab.report()

        Report specific issue types

        .. code-block:: python

            issue_types = ["dark", "near_duplicates"]
            imagelab.report(issue_types=issue_types)

        """
        assert isinstance(verbosity, int) and 0 <= verbosity < 5

        user_supplied_args = locals()
        report_args = self._get_report_args(user_supplied_args)

        issue_types_to_report = (
            issue_types if issue_types else self.issue_summary["issue_type"].tolist()
        )

        # filter issues based on max_prevalence in the dataset
        filtered_issue_types = self._filter_report(
            issue_types_to_report, report_args["max_prevalence"]
        )

        issue_summary = self.issue_summary[
            self.issue_summary["issue_type"].isin(filtered_issue_types)
        ]
        if len(issue_summary) > 0:
            if verbosity:
                print("Issues found in images in order of severity in the dataset\n")
            if print_summary:
                self._pprint_issue_summary(issue_summary)
            for issue_type in filtered_issue_types:
                if (
                    self.issue_summary.query(f"issue_type == '{issue_type}'")[
                        "num_images"
                    ].values[0]
                    == 0
                ):
                    continue
                print(f"{' ' + issue_type + ' images ':-^60}\n")
                print(
                    f"Number of examples with this issue: {self.issues[get_is_issue_colname(issue_type)].sum()}\n"
                    f"Examples representing most severe instances of this issue:\n"
                )
                self._visualize(
                    issue_type,
                    report_args["num_images"],
                    report_args["cell_size"],
                    show_id,
                )
        else:
            print(
                "Please specify some issue_types to check for in imagelab.find_issues()."
            )

    def _pprint_issue_summary(self, issue_summary: pd.DataFrame) -> None:
        issue_summary_copy = issue_summary.copy()
        issue_summary_copy.dropna(axis=1, how="all", inplace=True)
        issue_summary_copy.fillna("N/A", inplace=True)
        print(issue_summary_copy.to_markdown(), "\n")

    def _get_issue_manager(self, issue_type_str: str) -> IssueManager:
        if issue_type_str in IMAGE_PROPERTY_ISSUE_TYPES_LIST:
            return self._issue_managers[IMAGE_PROPERTY]
        elif issue_type_str in DUPLICATE_ISSUE_TYPES_LIST:
            return self._issue_managers[DUPLICATE]
        else:
            return self._issue_managers[issue_type_str]

    def _visualize(
        self,
        issue_type: str,
        num_images: int,
        cell_size: Tuple[int, int],
        show_id: bool,
    ) -> None:
        # todo: remove dependency on issue manager
        issue_manager = self._get_issue_manager(issue_type)
        viz_name = issue_manager.visualization

        if viz_name == "individual_images":
            sorted_df = self.issues.sort_values(by=get_score_colname(issue_type))
            sorted_df = sorted_df[sorted_df[get_is_issue_colname(issue_type)] == 1]

            scores = sorted_df.head(num_images)[get_score_colname(issue_type)]
            indices = scores.index.tolist()
            images = [self._dataset[i] for i in indices]

            # construct title info
            title_info = {"scores": [f"score : {x:.4f}" for x in scores]}
            if show_id:
                title_info["ids"] = [f"id : {i}" for i in indices]
            if issue_type == IssueType.ODD_SIZE.value:
                title_info["size"] = [
                    f"original size: {image.size}" for image in images
                ]

            if images:
                VizManager.individual_images(
                    images=images,
                    title_info=title_info,
                    ncols=self._config["visualize_num_images_per_row"],
                    cell_size=cell_size,
                )

        elif viz_name == "image_sets":
            image_sets_indices = sorted(
                self.info[issue_type][SETS], key=len, reverse=True
            )
            image_sets_indices = image_sets_indices[:num_images]
            image_sets = []
            for indices in image_sets_indices:
                image_sets.append([self._dataset[index] for index in indices])

            title_info_sets = []
            for s in image_sets_indices:
                title_info = {"name": [self._dataset.get_name(index) for index in s]}
                title_info_sets.append(title_info)

            if image_sets:
                VizManager.image_sets(
                    image_sets,
                    title_info_sets,
                    ncols=self._config["visualize_num_images_per_row"],
                    cell_size=cell_size,
                )

    # todo: compress this code
    def visualize(
        self,
        image_files: Optional[List[str]] = None,
        indices: Optional[List[str | int]] = None,
        issue_types: Optional[List[str]] = None,
        num_images: int = 4,
        cell_size: Tuple[int, int] = (2, 2),
        show_id: bool = False,
    ) -> None:
        """Show specific images.

        Can be used for visualizing either:
        1. Particular images with paths given in `image_files`.
        2. Images representing top-most severe instances of given `issue_types` detected the dataset.
        3. If no `image_files` or `issue_types` are given, random images will be shown from the dataset.

        If `image_files` is given, this overrides the argument `issue_types`.

        Parameters
        ----------

        image_files : List[str], optional
            List of filepaths for images to visualize.

        indices: List[str|int], optional
            List of indices of images in the dataset to visualize.
            If the dataset is a local data_path, the indices are filepaths, which is also the index in `imagelab.issues` dataframe.
            If the dataset is a huggingface or torchvision dataset, indices are of type int and corresponding to the indices in the dataset object.

        issue_types: List[str], optional
            List of issue types to visualize. For each type of issue, will show a few images representing the top-most severe instances of this issue in the dataset.

        num_images : int, optional
            Number of images to visualize from the dataset.
            These images are randomly selected if `issue_types` is ``None``.
            If `issue_types` is given, then this is the number of images for each issue type to visualize
            (images representing top-most severe instances of this issue will be shown).
            If `image_files` is given, this argument is ignored.

        cell_size : Tuple[int, int], optional
            Dimensions controlling the size of each image in the depicted image grid.

        Examples
        --------

        To visualize random images from the dataset

        .. code-block:: python

            imagelab.visualize()

        .. code-block:: python

            imagelab.visualize(num_images=8)

        To visualize specific images from the dataset

        .. code-block:: python

            image_files = ["./dataset/cat.png", "./dataset/dog.png", "./dataset/mouse.png"]
            imagelab.visualize(image_files=image_files)

        To visualize top examples of specific issue types from the dataset

        .. code-block:: python

            issue_types = ["dark", "odd_aspect_ratio"]
            imagelab.visualize(issue_types=issue_types)

        """
        if issue_types is not None:
            if len(issue_types) == 0:
                raise ValueError("issue_types list is empty")
            for issue_type in issue_types:
                self._visualize(issue_type, num_images, cell_size, show_id)
        elif image_files is not None:
            if len(image_files) == 0:
                raise ValueError("image_files list is empty.")
            images = [Image.open(path) for path in image_files]
            title_info = {"path": [path.split("/")[-1] for path in image_files]}
            VizManager.individual_images(
                images,
                title_info,
                ncols=self._config["visualize_num_images_per_row"],
                cell_size=cell_size,
            )
        elif indices:
            images = [self._dataset[i] for i in indices]
            title_info = {"name": [self._dataset.get_name(i) for i in indices]}
            VizManager.individual_images(
                images,
                title_info,
                ncols=self._config["visualize_num_images_per_row"],
                cell_size=cell_size,
            )
        else:
            print("Sample images from the dataset")

            if image_files is None:
                image_indices = random.sample(
                    self._dataset.index, min(num_images, len(self._dataset))
                )
                images = [self._dataset[i] for i in image_indices]
                title_info = {
                    "name": [self._dataset.get_name(i) for i in image_indices]
                }
                VizManager.individual_images(
                    images,
                    title_info,
                    ncols=self._config["visualize_num_images_per_row"],
                    cell_size=cell_size,
                )

    # Todo: Improve mypy dict typechecking so this does not return any
    def get_stats(self) -> Any:
        """Returns dict of statistics computed from images when auditing the data such as: brightness, color space, aspect ratio, etc.
        If statistics have not been computed yet, then returns ``None``.
        """
        return self.info["statistics"]

    def save(self, path: str, force: bool = False) -> None:
        """Saves this Imagelab instance, :py:attr:`issues` and :py:attr:`issue_summary` into a folder at the given path.
        Your saved Imagelab should be loaded from the same version of the CleanVision package to avoid inconsistencies.
        This method does not save your image files.

        Parameters
        ----------
        path : str
            Path to folder where this Imagelab instance will be saved on disk.

        force: bool, default=False
            If set to True, any existing files at `path` will be overwritten.
        """
        Serializer.serialize(path=path, imagelab=self, force=force)

    @classmethod
    def load(
        cls: Type[TImagelab], path: str, data_path: Optional[str] = None
    ) -> Imagelab:
        """Loads Imagelab from given path.

        Parameters
        ----------
        path : str
            Path to the saved Imagelab folder previously specified in :py:meth:`Imagelab.save` (not the individual pickle file).
        data_path : str
            Path to image dataset previously used in Imagelab, if your data exists locally as images in a folder.
            If the `data_path` is changed, the code will break as Imagelab functionalities are dependent on it.
            You should be using the same version of the CleanVision package previously used when saving Imagelab.

        Returns
        -------
        Imagelab
            Returns a saved instance of Imagelab
        """
        imagelab = Serializer.deserialize(path)
        return imagelab


import json
from os import PathLike
from pathlib import Path

from typing import Any, Dict, List, MutableMapping, Union

import yaml

STRUCTURE = {
    "Dataset": [
        "dataset_name",
        "dataset_config",
        "dataset_revision",
        "labelcolumn",
        "textcolumn",
    ],
    "Training": [
        "optim",
        "learning_rate",
        "per_device_train_batch_size",
        "per_device_eval_batch_size",
        "gradient_accumulation_steps",
        "max_steps",
        "save_steps",
        "metric_for_best_model",
    ],
    "Best checkedpoint based on validation": [
        "best_metric",
        "best_model_checkpoint",
    ]
}

def rec_merge(d1, d2):
    '''
    https://stackoverflow.com/a/24088493/1150683
    Update two dicts of dicts recursively,
    if either mapping has leaves that are non-dicts,
    the second's leaf overwrites the first's.
    '''
    for k, v in d1.items():
        if k in d2:
            # this next check is the only difference!
            if all(isinstance(e, MutableMapping) for e in (v, d2[k])):
                d2[k] = rec_merge(v, d2[k])
            # we could further check types and merge as appropriate here.
    d3 = d1.copy()
    d3.update(d2)
    return d3

def format_dict(d: Dict[str, Any]):
    return "\n".join(f"- {k}: {v}" for k, v in d.items())

def get_modelcard(modelcard_base: Union[str, PathLike]):
    pmodelcard = Path(modelcard_base)
    modelcard = yaml.safe_load(pmodelcard.read_bytes())
    return modelcard

def generate_readme(din: Union[str, PathLike], modelcard_base: Union[str, PathLike], overwrite_readme: bool = False):
    pdin = Path(din).resolve()
    preadme = pdin / "README.md"
    mode = "w" if overwrite_readme else "a"

    test_results: Dict[str, float] = json.loads(pdin.joinpath("test_results.json").read_bytes())
    env: Dict[str, Any] = json.loads(pdin.joinpath("env.json").read_bytes())
    trainer_state: Dict[str, Any] = json.loads(pdin.joinpath("trainer_state.json").read_bytes())
    args: Dict[str, Any] = json.loads(pdin.joinpath("args.json").read_bytes())
    args = {**args, **trainer_state}

    with preadme.open(mode=mode, encoding="utf-8") as fhout:
        modelcard = get_modelcard(modelcard_base)
        modelcard["datasets"] = [args["dataset_name"]]
        modelcard["metrics"] = list(test_results.keys())
        model_index = {
            "name": f"{args['model_name_or_path'].split('/')[-1]}-{args['dataset_name'].split('/')[-1]}",
            "results": [{
                "task": {
                    "type": modelcard["model-index"][0]["results"][0]["task"]["type"],
                    "name": modelcard["model-index"][0]["results"][0]["task"]["name"],
                },
                "dataset": {
                    "type": args["dataset_name"],
                    "name": f"{args['dataset_name']} - {args['dataset_config']} - {args['dataset_revision']}",
                    "config": args['dataset_config'],
                    "split": args["testsplit_name"],
                    "revision": args['dataset_revision']
                },
                "metrics": [{
                    "type": m,
                    "value": v,
                    "name": f"{args['testsplit_name'].capitalize()} {m}"
                } for m, v in test_results.items()]
            }]
        }
        modelcard["model-index"] = [rec_merge(modelcard["model-index"][0], model_index)]

        fhout.write(f"---\n{yaml.dump(modelcard)}---\n\n")

        fhout.write(f"# {args['model_name_or_path'].split('/')[-1]}-{args['dataset_name'].split('/')[-1]}\n\n")
        for title, keys in STRUCTURE.items():
            temp_d = {k: v for k, v in args.items() if k in keys}
            temp_d = {k: v for k, v in sorted(temp_d.items(), key=lambda k: keys.index(k[0]))}
            fhout.write(f"# {title}\n{format_dict(temp_d)}\n\n")

        fhout.write(f"# Test results of best checkpoint\n{format_dict(test_results)}\n\n")
        fhout.write(f"## Confusion matric\n\n")
        fhout.write("![cfm](fig/test_confusion_matrix.png)\n\n")
        fhout.write(f"## Normalized confusion matrix\n\n")
        fhout.write("![norm cfm](fig/test_confusion_matrix_norm.png)\n\n")
        fhout.write(f"# Environment\n{format_dict(env)}\n\n")

    print("Done! Verify the README before uploading!!!")

def main():
    import argparse

    cparser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    cparser.add_argument("din", help="Input directory that contains the model and all output jsons."
                                     "Will also write README.md here.")
    cparser.add_argument("modelcard_base", help="Path to model card base YAML.")
    cparser.add_argument("-f", "--overwrite_readme", help="Overwrite a README if it already exists.",
                         action="store_true")

    cargs = cparser.parse_args()
    generate_readme(**vars(cargs))

if __name__ == "__main__":
    main()


from django.shortcuts import render
from django.http import HttpResponse
from property.choice import price_choices, bedroom_choices, county_choices

from property.models import Property
from accounts.models import Owner

def index(request):
    propertys = Property.objects.order_by('-list_date')

    context = {
        'propertys': propertys,
        'county_choices': county_choices,
        'bedroom_choices': bedroom_choices,
        'price_choices': price_choices
    }

    return render(request, 'pages/index.html', context)

def about(request):
    # Get all realtors
   owners = Owner.objects.order_by('-date_joined')

    # Get MVP
   mvp_owners = Owner.objects.all().filter(is_mvp=True)

   context = {
        'owners': owners,
        'mvp_owners': mvp_owners
    }

   return render(request, 'pages/about.html', context)

# Create your views here.


import numpy as np

def k_means_init(input,num_clusters):
    centers=[]
    for i in range(num_clusters):
        i=np.random.random(np.size(input[0]))
        centers.append(i)
    return centers
def classify(input,centers,num_clusters):
    clusters=[[] for i in range(num_clusters)]
    num_list=[i for i in range(num_clusters)]
    records = [[] for i in range(num_clusters)]
    for label,i in enumerate(input):
        distants=[]
        for center in centers:
            distant=np.linalg.norm(i-center)+1/(np.sum(i*center)+0.1)
            distants.append(distant)
        min_distant = min(distants)
        for j ,distant in enumerate(distants):
            if distant == min_distant:
                clusters[j].append(i)
                records[j].append(label)
                break
    records_dict=dict(zip(num_list,records))
    return clusters,records_dict
def centers_refresh(clusters,input):
    centers = []
    for cluster in clusters:
        if len(cluster):
            center=sum(cluster)/len(cluster)
        else:
            center =np.random.random(np.size(input[0]))
        centers.append(center)
    return centers

def judge(centers,pre_centers):
    for i in range(len(centers)):
        if (centers[i] == pre_centers[i]).all():
            pass
        else:
            return False
    return  True
def k_means(input,num_clusters,iter_number):
    centers=k_means_init(input,num_clusters)
    for i in range(iter_number):
        pre_centers=centers
        clusters,records=classify(input,centers,num_clusters)
        centers=centers_refresh(clusters,input)
        if judge(centers,pre_centers):
            centers=np.array(centers)
            return  records,centers
    centers = np.array(centers)
    return records,centers


def add(numbers, x, y, z):
    numbers[z] = numbers[x] + numbers[y]

def multiply(numbers, x, y, z):
    numbers[z] = numbers[x] * numbers[y]

def opcode(sequence):
    inst_point = 0
    while True:
        op = sequence[inst_point]
        if op == 99:
            break
        pos1 = sequence[inst_point + 1]
        pos2 = sequence[inst_point + 2]
        pos3 = sequence[inst_point + 3]
        if op == 1:
            add(sequence, pos1, pos2, pos3)
        elif op == 2:
            multiply(sequence, pos1, pos2, pos3)
        inst_point += 4
    return sequence[0]

inputs = ""
f = open("day2_input.txt", "r")
for line in f:
    inputs += line
f.close()
inputs = inputs.split(',')
numbers = []
for i in inputs:
    numbers.append(int(i))

for noun in range(100):
    for verb in range(100):
        check = numbers.copy()
        check[1] = noun
        check[2] = verb
        if opcode(check) == 19690720:
            print(100 * noun + verb)
            break


## set up logging
import logging, os
logging.basicConfig(level=os.environ.get("LOGLEVEL","INFO"))
log = logging.getLogger("glam_command_line")

import argparse, glob, json, subprocess, sys
import glam_data_processing.legacy as glam
from datetime import datetime

# create temporary directory for shenanigans
TEMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),"temp/")
try:
	os.mkdir(TEMP_DIR)
except FileExistsError:
	pass

def statsOneFile(file_path,admin_level, mask_level,admin_specified=None,mask_specified=None) -> bool:
	base = os.path.basename(file_path)
	log.info(f"Processing {base}")
	try:
		img = glam.getImageType(file_path)(file_path) # create Image object
		img.uploadStats(admin_level=admin_level,crop_level=mask_level,admin_specified=admin_specified,crop_specified=mask_specified)
		return True
	except:
		log.exception(f"FAILED in processing {base}")
		return False

def download_and_statsOneFile(product,date,directory,admin_level,mask_level,admin_specified=None,mask_specified=None,save=False) -> bool:
	"""Wrapper for statsOneFile in case downloading is required"""
	paths = ()
	try:
		log.info(f"Downloading {product} {date}")
		downloader = glam.Downloader()
		paths = downloader.pullFromS3(product,date,directory)
		log.info(f"Processing {product} {date}")
		for p in paths:
			assert statsOneFile(file_path=p,admin_level=admin_level,mask_level=mask_level,admin_specified=admin_specified,mask_specified=mask_specified)
		downloader = None
		return True
	except:
		log.exception(f"{product} {date} FAILED")
		return False
	finally:
		if not save:
			for p in paths:
				os.remove(p)

def getAllTiffs(in_dir) -> list:
	"""Takes input directory path, returns list of all tiff full paths"""
	return glob.glob(os.path.join(in_dir,"*.tif"))

def getProductDateTuple(file_path) -> tuple:
	"""extracts the product and date from a file path; returns as a tuple"""
	try:
		base = os.path.basename(file_path)
		parts= base.split(".")
		product = parts[0]
		try:
			date = datetime.strptime(f"{parts[1]}.{parts[2]}","%Y.%j").strftime("%Y-%m-%d")
		except:
			date = datetime.strptime(parts[1],"%Y-%m-%d").strftime("%Y-%m-%d")
		return (product,date)
	except:
		log.exception(f"Failed to extract product-date tuple for {os.path.basename(file_path)}")

def getUniqueFilename(directory,file_name) -> str:
	"""given desired directory and file name, iterates through
	integers until it finds a unique file_name_{i}.ext"""
	base,ext=os.path.splitext(file_name)
	outPath = os.path.join(directory,file_name)
	i = 0
	while os.path.exists(outPath):
		i += 1
		outPath = os.path.join(directory,f"{base}_{i}{ext}")
	return outPath

def main():
	## parse arguments
	parser = argparse.ArgumentParser(description="Generate new statistics, in parallel")
	parser.add_argument("file_directory",
		help="Directory where imagery files are / will be stored")
	parser.add_argument("-al",
		"--admin_level",
		default="ALL",
		choices=["ALL","GAUL","BRAZIL"],
		help="Run statistics for only a subset of administrative divisions")
	parser.add_argument("-as",
		"--admin_specified",
		choices = glam.admins,
		help="Run statistics for a single administrative division")
	parser.add_argument("-ml",
		"--mask_level",
		default="ALL",
		choices=["ALL","BRAZIL","CROPMONITOR","NOMASK"],
		help="Run statistics for only a subset of crop masks")
	parser.add_argument("-ms",
		"--mask_specified",
		choices=glam.crops,
		help="Run statistics for a single crop mask")
	parser.add_argument("-c",
		"--cores",
		required=True,
		help="How many cores to use")
	parser.add_argument("-l",
		"--logfile",
		help="Where to write output log messages")
	parser.add_argument("-d",
		"--download_files",
		action='store_true',
		help="Download any files not currently on disk. Pull them from S3 before running stats. Requires product flag.")
	parser.add_argument('-p',
		"--product",
		help="Which product to pull. Required if --download_files is set.")
	parser.add_argument("-s",
		"--save_results",
		action='store_true',
		help="If set, does not delete downloaded files")
	parser.add_argument("-mo",
		"--missing_only",
		action='store_true',
		help="Generate statistics only for files NOT currently on disk in file_directory")
	parser.add_argument("-META",
		metavar='metastring',
		default = None,
		action='store',
		help="Do not set this flag. For internal use only.")

	args = parser.parse_args()

	## META stores file path or product/date as json
	if args.META:
		meta_parts = args.META.split(".")
		if args.download_files:
			product = meta_parts[0]
			date = meta_parts[1]
			#print(product)
			#print(date)
			download_and_statsOneFile(product,date,args.file_directory,args.admin_level,args.mask_level,args.admin_specified,args.mask_specified,args.save_results)
		else:
			in_file = args.META
			#print(in_file)
			statsOneFile(in_file, args.admin_level,args.mask_level,args.admin_specified,args.mask_specified)
		sys.exit()

	## confirm argument validity
	# download_files requires product
	try:
		if args.download_files:
			assert args.product
	except AssertionError:
		log.error("If download_files is set, a product must be specified.")
		sys.exit()
	# no overlap of BRAZIL crops and GAUL admins
	if args.mask_level == "BRAZIL" and args.admin_level == "GAUL":
		log.error("Can't generate Brazil crop stats for Gaul regions")
		sys.exit()
	# don't set both _level and _specified
	if args.admin_specified and (args.admin_level != "ALL"):
		log.error("Do not set both --admin_level and --admin_specified")
		sys.exit()
	if args.mask_specified and (args.mask_level != "ALL"):
		log.error("Do not set both --mask_level and --mask_specified")
		sys.exit()
	# warn the user if arguments are being ignored
	if not args.download_files:
		try:
			assert not args.save_results
		except AssertionError:
			log.warning("--download_files not set; --save_results flag ignored")
		try:
			assert not args.missing_only
		except AssertionError:
			log.warning("--download_files not set; --missing_only flag ignored")
	if not args.logfile:
		args.logfile = getUniqueFilename(TEMP_DIR,"gns_log.txt")

	lines = []
	## perform stats on existing files
	extant = getAllTiffs(args.file_directory)
	if not args.missing_only:
		for f in extant:
			line = f"python {os.path.abspath(__file__)} {args.file_directory} -al {args.admin_level} -ml {args.mask_level} -c {args.cores} -META '{f}'"
			if args.save_results:
				line += " -s"
			if args.admin_specified:
					line += f" -as {args.admin_specified}"
			if args.mask_specified:
				line += f" -ms {args.mask_specified}"
			line += "\n"
			lines += line

	## download and process new files if requested
	if args.download_files:
		downloader= glam.Downloader()
		onDisk = [getProductDateTuple(f) for f in extant]
		#available = downloader.getAllS3(product=args.product)
		#missing = [t for t in available if t not in onDisk]
		missing = downloader.listMissing(args.file_directory)
		if len(missing) == 0:
			log.info(f"No missing files; all available files on S3 are also in {args.file_directory}")
		else:
			for t in missing:
				product,date=t
				meta = f"{product}.{date}"
				line = f"python {os.path.abspath(__file__)} {args.file_directory} -al {args.admin_level} -ml {args.mask_level} -c {args.cores} -d -META {meta}"
				if args.save_results:
					line += " -s"
				if args.admin_specified:
					line += f" -as {args.admin_specified}"
				if args.mask_specified:
					line += f" -ms {args.mask_specified}"
				line += " \n"
				lines += line

			downloader=None
	command_file = getUniqueFilename(TEMP_DIR,"gns_commands.txt")

	with open(command_file,'w') as wf:
		wf.writelines(lines)
	shell_call = ["nohup","sh","-c", f'"cat {command_file} | parallel -j {args.cores}"', "&>",args.logfile,"&"]
	print(" ".join(shell_call))
	subprocess.call(shell_call)
	#os.remove(command_file)

if __name__ == "__main__":
	main()


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
df = pd.read_csv (r'Seed Yield.csv')
print (df)
sns.set_theme(style="whitegrid")
# Draw a nested barplot by cultivar and irrigation
g = sns.catplot(
    data=df, kind="bar",
    x="Cultivar", y="Seed Yield", hue="Irrigation",
    ci="sd", palette="dark", alpha=.6, height=6
)
g.despine(left=False)
g.set_axis_labels("", "Seed Yield (kg)")
g.legend.set_title("Irrigation")
plt.show()


from torchvision.datasets import CIFAR10, CIFAR100, MNIST, SVHN, FashionMNIST
from data.MVTecDataset import getMVTecDataset
from data.AdaptiveExposureDataset import getAdaptiveExposureDataset
import os
import torch
import numpy as np
import torchvision.transforms as transforms
from torch.utils.data import Dataset
from PIL import Image
from glob import glob
from utils import sparse2coarse
from data.constants import CIFAR10_PATH, CIFAR100_PATH, MNIST_PATH, FMNIST_PATH, SVHN_PATH, MVTEC_PATH, ADAPTIVE_PATH, mvtec_labels
import torchvision.transforms.functional as F
import requests
from PIL import Image
from tqdm import tqdm

dataset_paths = {
    "cifar10":CIFAR10_PATH,
    "cifar100":CIFAR100_PATH,
    "mnist":MNIST_PATH,
    "fashion":FMNIST_PATH,
    "svhn":SVHN_PATH,
    "mvtec":MVTEC_PATH,
    "adaptive":ADAPTIVE_PATH
}

tansform_224 = transforms.Compose([
                                    transforms.Resize(224),
                                    transforms.ToTensor()
                                ])

tansform_224_gray = transforms.Compose([
                                    transforms.Resize(224),
                                    transforms.Grayscale(num_output_channels=3),
                                    transforms.ToTensor()
                                ])

tansform_32 = transforms.Compose([
                                    transforms.Resize(32),
                                    transforms.ToTensor()
                                ])

tansform_32_gray = transforms.Compose([
                                    transforms.Resize(32),
                                    transforms.Grayscale(num_output_channels=3),
                                    transforms.ToTensor()
                                ])

class GeneralDataset(torch.utils.data.Dataset):
    def __init__(self, data, targets, transform=None):
        self.transform = transform
        self.data = [F.to_pil_image(x).convert('RGB') for x in data]
        self.targets = targets

    def __getitem__(self, index):
        image = self.data[index]
        target = self.targets[index]

        if self.transform is not None:
            image = self.transform(image)

        return image, target

    def __len__(self):
        return len(self.data)

class GeneralDatasetOODClf(torch.utils.data.Dataset):
    def __init__(self, normal_data, exposure_data, transform=None):
        self.transform = transform
        normal_data = [F.to_pil_image(x).convert('RGB') for x in normal_data]
        exposure_data = [F.to_pil_image(x).convert('RGB') for x in exposure_data]
        self.data = normal_data + exposure_data
        self.targets = [0] * len(normal_data) + [1] * len(exposure_data)

    def __getitem__(self, index):
        image = self.data[index]
        target = self.targets[index]

        if self.transform is not None:
            image = self.transform(image)

        return image, target

    def __len__(self):
        return len(self.data)

def get_dataloader(normal_dataset:str, normal_class_indx:int, exposure_dataset:str, batch_size):

    transform = None

    is_big: bool = True if normal_dataset in ['mvtec', 'ctscan'] else False
    is_colorful: bool = True if normal_dataset in ['cifar10', 'cifar100', 'svhn', 'mvtec'] else False

    if is_big:
        if is_colorful:
            transform = tansform_224
        else:
            transform = tansform_224_gray
    else:
        if is_colorful:
            transform = tansform_32
        else:
            transform = tansform_32_gray

    normal_data, test_data, test_targets = get_normal_class(dataset=normal_dataset, normal_class_indx=normal_class_indx, transform=transform)
    print("Length of Normal Dataset:", len(normal_data))
    print("Length of Test Dataset:", len(test_data))

    assert 1 in test_targets and 0 in test_targets

    exposure_data = get_exposure(dataset=exposure_dataset, normal_dataset=normal_dataset, normal_class_indx=normal_class_indx, count=len(normal_data))
    print("Length of Exposure Dataset:", len(exposure_data))

    trainset = GeneralDatasetOODClf(normal_data=normal_data, exposure_data=exposure_data, transform=transform)
    testset = GeneralDataset(data=test_data, targets = test_targets, transform=transform)
    del exposure_data, normal_data

    train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2)
    test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2)

    return train_loader, test_loader

####################
#  Normal Datastes #
####################

def get_normal_class(dataset='cifar10', normal_class_indx = 0,  transform=None):
    datasets_builders = {
        "cifar10":CIFAR10,
        "cifar100":CIFAR100,
        "mnist":MNIST,
        "fashion":FashionMNIST,
        "svhn":SVHN,
        "mvtec":getMVTecDataset(normal_class_indx=normal_class_indx, only_anomaly_in_test=False)
    }

    if dataset in datasets_builders.keys():
        trainset = datasets_builders[dataset](root=dataset_paths[dataset], train=True, download=True) #CHECK transform ?

        if dataset == "cifar100":
            trainset.targets = sparse2coarse(trainset.targets)
        trainset.data = trainset.data[np.array(trainset.targets) == normal_class_indx]

        testset = datasets_builders[dataset](root=dataset_paths[dataset], train=False, download=True)
        if dataset == "cifar100":
            testset.targets = sparse2coarse(testset.targets)
        testset.targets  = [int(t!=normal_class_indx) for t in testset.targets]

        return [F.to_tensor(np.array(x).astype(np.uint8)) for x  in trainset.data], [F.to_tensor(np.array(x).astype(np.uint8)) for x  in testset.data], testset.targets
    else:
        raise Exception("Dataset is not supported yet. ")

######################
#  Exposure Datastes #
######################

def get_exposure(dataset:str='cifar10', normal_dataset:str='cifar100', normal_class_indx:int = 0, count:int = 0):
    assert count > 0

    datasets_builders = {
        "cifar10":CIFAR10,
        "cifar100":CIFAR100,
        "mnist":MNIST,
        "fashion":FashionMNIST,
        "svhn":SVHN,
        "mvtec":getMVTecDataset(
            normal_class_indx = None,
            only_anomaly_in_test = False if normal_dataset!='mvtec' else True,
            remove_class_indx = None if normal_dataset!='mvtec' else mvtec_labels[normal_class_indx]
        ),
        "adaptive":getAdaptiveExposureDataset(normal_dataset, normal_class_indx)
    }

    if dataset in datasets_builders.keys():
        exposure_train = datasets_builders[dataset](root=dataset_paths[dataset], train=True, download=True) #CHECK transforms ?

        assert len(exposure_train) > 0

        #exposure_test = datasets_builders[dataset](root=dataset_paths[dataset], train=False, download=True) #CHECK transforms ?
        if dataset == "cifar100":
            exposure_train.targets = sparse2coarse(exposure_train.targets)
            #exposure_test.targets = sparse2coarse(exposure_test.targets)

        if normal_dataset.lower() == dataset:
            exposure_train.data = exposure_train.data[np.array(exposure_train.targets) != normal_class_indx]
            #exposure_test.data = exposure_test.data[np.array(exposure_test.targets) != normal_class_indx]

        if dataset == 'adaptive':
            exposure_data = torch.stack(exposure_train.data)
        else:
            exposure_data = torch.tensor(exposure_train.data)

        del exposure_train

        #if exposure_data.size(0) < count:
        #    exposure_data = torch.cat((exposure_data, torch.tensor(exposure_test.data)), 0)
        #del exposure_train

        if exposure_data.size(0) < count:
            copy_dataset(exposure_data, count)

        indices = torch.randperm(exposure_data.size(0))[:count]
        exposure_data =  exposure_data[indices]

        if dataset == 'adaptive':
            return exposure_data
        else:
            return [F.to_tensor(np.array(x).astype(np.uint8)) for x  in exposure_data]
    else:
        raise Exception("Dataset is not supported yet. ")

def copy_dataset(dataset , target_count:int):
    while target_count > len(dataset):
        dataset = torch.cat((dataset, dataset.data), 0)

    return dataset

def get_ADAPTIVE_exposure(normal_dataset:str, normal_class_indx:int,count:int):
    exposure_data = []
    try:
        exposure_path = glob(os.path.join(ADAPTIVE_PATH, normal_dataset, f'{normal_class_indx}', "*.npy"), recursive=True)
        for path in exposure_path:
            exposure_data += np.load(path).tolist()
    except:
        raise ValueError('Wrong Exposure Address!')
        exit()

    exposure_data = torch.tensor(exposure_data)

    if exposure_data.size(0) < count:
        copy_dataset(exposure_data, count)

    indices = torch.randperm(exposure_data.size(0))[:count]
    exposure_data =  exposure_data[indices]

    return exposure_data


import numpy as np

def board_to_hash_code(board):
	'''Converts board to a string for hashing.

	The hash code will read the board left to right, top to bottom and place a
	'0' for an empty slot, a '1' for a 1 ('x' plays as 1), and a '2' for a 2
	('o' plays as 2).

	Example:
		[[0, 1, 1], [2, 1, 0], [0, 0, 0]] -> '011210000'

	This is a bijective relationship (since we can invert it), so we need not
	worry about issues of mapping two board states to the same hash.

	Args:
		board: An [[int]] np array representing the current state.
	Returns:
		A string of 0, 1, or 2 representing the board.
	'''
	char_list = []
	for row in board:
		char_list.extend([str(elem) for elem in row])
	return ''.join(char_list)

def hash_code_to_board(board_hash):
	'''Converts a ternary string to a board hash.

	Args:
		board_hash: A string length nine consisting of '0', '1', and '2'.
	Returns:
		The corresponding np.array representing the state.
	'''
	board = np.array([int(char) for char in board_hash])
	return board.reshape((3, 3))

def get_board_status(board):
	'''Get the status of the board, 1 is 'x' won, 2 is 'o' won, and 0 otherwise.

	Args:
		board: A [[int]] np array representing the current state.
	Returns:
		An integer encoding the termination status of this game.
	'''
	flat_board = board.flatten()
	winning_indices = [
		[0, 1, 2], [3, 4, 5], [6, 7, 8], [0, 3, 6], [1, 4, 7], [2, 5, 8],
		[0, 4, 8], [6, 4, 2]
	]
	for indices in winning_indices:
		cur_line = flat_board[np.array(indices)]
		if np.array_equal(cur_line, [1, 1, 1]):
			return 1
		elif np.array_equal(cur_line, [2, 2, 2]):
			return 2
	return 0

def get_initial_board_value(board):
	'''Get the initial value of the board as stated by the algorithm.

	Args:
		board: A 3x3 numpy array.
	Returns:
		The numerical initial value.  1.0 for 'x' winning, 0.0 for '0' winning,
		and 0.5 otherwise.
	'''
	status = get_board_status(board)
	if status == 1:
		return 1.0
	elif status == 2:
		return 0.0
	else:
		return 0.5


from time import time

import glob
from sys import argv
import cv2
import pickle

import shutil
from sklearn.cluster import KMeans
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score, confusion_matrix
from sklearn import preprocessing, svm
import numpy as np

## Inputs

k1 = 50
relearnK = False

c1 = 20
svc_kernel = 'rbf' # 'linear'

working_dir = ".\\images"

# phase subdirs
learn_subdir       = "train"
test_subdir        = "test"
validation_subdir  = "valid"

# classes
cat1 = "chat4"
cat2 = "chien3"
cat3 = "formule1"
cat4 = "motoGrandPrix"

def getCatPath(cat_name,extension,phase_subdir):
    return ("%s\\%s\\%s"+"/*.%s") % (working_dir,phase_subdir,cat_name,extension)

baryName = ".\\data\\" # saved files dir

# verbose options

verbose_params = True

verbose_steps = True
verbose_files = False
verbose_state = False

verbose_kmeans = False
verbose_score = True
verbose_confusion = True

## Sequence variables

LEARN = 0
VALIDATE = 1
TEST = 2

stepNameDict = {LEARN: "train",   VALIDATE: "valid",      TEST: "test"}
subDirDict   = {LEARN: learn_subdir, VALIDATE: validation_subdir, TEST: test_subdir}

SEQUENCE = [LEARN, VALIDATE, TEST]

## Running sequence

if (verbose_params):
    print("k1 = %d"%(k1))
    print("c1 = %3f"%(c1))
    print("kernel = %s"%(svc_kernel))

debug = False

for STEP in SEQUENCE:

    if (verbose_steps):
        print("\n"+stepNameDict[STEP]+"...\n")

    subdir = subDirDict[STEP]

    ## Establishing groundTruth
    path_cat_1 = getCatPath(cat1,"jpg",subdir)
    path_cat_2 = getCatPath(cat2,"jpg",subdir)
    path_cat_3 = getCatPath(cat3,"jpeg",subdir)
    path_cat_4 = getCatPath(cat4,"jpeg",subdir)

    if (verbose_files):
        print("Reading" + 4*"\n\t%s" % (path_cat_1,path_cat_2,path_cat_3,path_cat_4))

    listImg = glob.glob(path_cat_1)
    tmp1 = len(listImg)
    groundTruth = [0]*tmp1

    listImg += glob.glob(path_cat_2)
    tmp2 = len(listImg) - tmp1
    groundTruth += [1]*tmp2

    listImg += glob.glob(path_cat_3)
    tmp3 = len(listImg) - ( tmp1 + tmp2 )
    groundTruth += [2]*tmp3

    listImg += glob.glob(path_cat_4)
    tmp4 = len(listImg) - ( tmp1 + tmp2 + tmp3 )
    groundTruth += [3]*tmp4

    if (verbose_files): print("Done reading" + 4*" %d" % (tmp1, tmp2, tmp3, tmp4))

    ## retrieving SIFT descriptors

    dimImg = [] # nb of sift per file
    lesSift = np.empty(shape=(0, 128), dtype=float) # array of all SIFTS from all images

    for s in listImg:
        if (verbose_files):
            print("###",s,"###")
        image = cv2.imread(s)
        gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
        sift = cv2.xfeatures2d.SIFT_create()
        kp,des = sift.detectAndCompute(gray,None)
        if (verbose_files):
            print("SIFT: ", len(kp))
        dimImg.append(len(des))
        lesSift = np.append(lesSift,des,axis=0)

    ## Computing k-means

    if (verbose_state): print("Done sifting")

    sifts = lesSift ###### preprocessing.normalize(lesSift)

    if ((STEP==LEARN) and relearnK):
        t0 = time()
        kmeans1 = KMeans(n_clusters=k1, random_state=0).fit(sifts)
        print("KMeans completed in %0.3fs" % (time() - t0))

        labels = kmeans1.labels_

        #sauvegarde de l'objet
        with open(baryName+'multiclass.bary', 'wb') as output:
            pickle.dump(kmeans1, output, pickle.HIGHEST_PROTOCOL)

    if (STEP==VALIDATE or STEP==TEST or (not relearnK)):
        #chargement de l'objet
        with open(baryName+'multiclass.bary', 'rb') as input:
            kmeans1 = pickle.load(input)

        labels = kmeans1.predict(sifts)

    if (verbose_kmeans): print("result of kmeans k=%d"%(k1), labels)

    ## Writing BOWs from k-means result

    bows = np.empty(shape=(0,k1),dtype=float) #BOW initialization

    i = 0
    for nb in dimImg: # for each sound (file)
        tmpBow = [0]*k1
        j = 0
        while j < nb: # for each SIFT of this sound (file)
            tmpBow[labels[i]] += 1
            j+=1
            i+=1
        copyBow = tmpBow.copy()
        bows = np.append(bows, [copyBow], 0)

    # ** Here was the culprit **
    BOWs = bows ###### preprocessing.normalize(bows)

    ## Classification
    # Todo : Tweak 'decision_function_shape'
    classif = svm.SVC(C=c1,kernel=svc_kernel, decision_function_shape='ovr')

    if (STEP==LEARN):
        classif.fit(BOWs, groundTruth) # apprentissage
        #sauvegarde de l'objet
        with open(baryName+'multiclassSVC.svm', 'wb') as output:
            pickle.dump(classif, output, pickle.HIGHEST_PROTOCOL)

    if (STEP==VALIDATE or STEP==TEST):
        #chargement de l'objet
        with open(baryName+'multiclassSVC.svm', 'rb') as input:
            classif = pickle.load(input)

    if (verbose_score):
        print("groundTruth: ", np.array(groundTruth))
        res = classif.predict(BOWs)
        print("prediction : ",res)

        # Todo : Tweak 'average'
        F1 = f1_score(groundTruth, res, average=None) #F1
        print("F-measure =",F1)

        score = classif.score(BOWs, groundTruth)
        print(stepNameDict[STEP],"score = ", score)

    if (verbose_confusion):
        confusion = confusion_matrix(groundTruth, res, labels=[0,1,2,3])
        print(confusion)


from typing import Optional

from fastapi import FastAPI
from pydantic import BaseModel

app = FastAPI()
persons = [{"id": 1, "ad": "Seyma", "soyad": "Sarigil", "meslek": "Gelistirici", "memleket": "Hatay"},
           {"id": 2, "ad": "Alp", "soyad": "Kara", "meslek": "Müzisyen", "memleket": "İstanbul"}]

class Person(BaseModel):
    id: Optional[int] = None
    ad: str
    soyad: str
    meslek: str
    memleket: str

@app.get("/")
def get_all_persons():
    return persons

@app.post("/add_person/")
def add_person(person: Person): #ad, soyad, meslek, memleket
    person_dict = person.dict()
    person_dict["id"] = persons[-1]["id"] + 1
    persons.append(dict(person_dict))
    return persons

@app.get("/get_person/{person_id}/")
def get_specific_person(person_id: int):
    person = [i for i in persons if i["id"] == person_id]
    return person

@app.put("/update_person/{person_id}/")
def update_name_person(person_id: int, person: Person):
    p = [i for i in persons if person.id == person_id]
    p[0]["ad"] = person.ad
    return persons

@app.delete("/delete_person/{person_id}/")
def delete_person(person_id: int):
    person = [i for i in persons if i["id"] == person_id]
    persons.remove(person[0])
    return person


from libreria_grafo import *
from grafo import Grafo
import sys

def itinerario(vertices, n_archivo):

	grafo = Grafo(True, False)

	for v in vertices:
		grafo.agregar_vertice(v)

	with open(n_archivo) as archivo:
		linea = archivo.readline()
		linea = linea.rstrip("\n")
		linea = linea.split(",")
		grafo.agregar_arista(linea[0], linea[1])

	res = orden_topologico(grafo)
	cant_res = len(res)
	peso = 0
	for i in range(1, cant_res):
		peso += grafo.ver_peso(res[i - 1], res[i])
	stdout_flechas(res, cant_res, peso)
	return res

def camino_ini_fin(grafo, desde, hasta):

	camino = camino_minimo(grafo, desde, hasta)
	res, peso = reconstruir_camino(grafo, camino, desde, hasta)
	stdout_flechas(res, len(res), peso)
	return res

def recorrido_optimo(grafo, origen):
	peso, camino = viajante(grafo,origen)
	cant_camino = len(camino)
	stdout_flechas(camino, cant_camino, peso)
	return camino

def recorrido_aproximado(grafo, origen):

	camino, peso = viajante_aproximado(grafo, origen)
	cant_camino = len(camino)
	stdout_flechas(camino, cant_camino, peso)
	return camino

def reducir_caminos(grafo, n_archivo, ciudades):

	contador = 0
	cant_ciudades = len(ciudades)
	grafo_minimo = arbol_tendido_minimo(grafo)
	with open(n_archivo, "w") as archivo:
		archivo.write("{}\n".format(cant_ciudades))
		for key, values in ciudades.items():
			archivo.write("{},{},{}\n".format(key, *values))
		archivo.write("{}\n".format(grafo_minimo.cant_aristas()))
		for v in grafo_minimo.obt_vertices():
			for w in grafo_minimo.obtener_vecinos(v):
				contador += grafo_minimo.ver_peso(v, w)
				archivo.write("{},{},{}\n".format(v, w, grafo_minimo.ver_peso(v, w)))
	return contador

def reconstruir_camino(grafo, diccio, desde, hasta):

	res = []
	aux = hasta
	pila = Pila()
	cont = 0
	peso = 0
	while(aux):
		pila.apilar(aux)
		aux = diccio[aux]
	while(not pila.esta_vacia()):
		res.append(pila.desapilar())
		cont += 1
	for i in range(1, cont):
		peso += grafo.ver_peso(res[i-1], res[i])
	return  res, peso

def stdout_flechas(lista, cant_elem, peso = None):
	for i in range(cant_elem - 1):
		sys.stdout.write("{} -> ".format(lista[i]))
	sys.stdout.write("{}\n".format(lista[cant_elem-1]))
	if peso != None:
		sys.stdout.write("Costo total: {}\n".format(peso))


from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
from rest_framework_simplejwt.views import TokenObtainPairView
from rest_framework import exceptions

class StaffTokenObtainPairSerializer(TokenObtainPairSerializer):
    def validate(self, attrs):
        data = super().validate(attrs)
        if self.user.is_employee == True:
            refresh = self.get_token(self.user)
            data['refresh'] = str(refresh)
            data['access'] = str(refresh.access_token)
            data['work_area'] = self.user.employees.work_area.name
            data['fullname'] = self.user.fullname
            return data
        else:
            raise exceptions.AuthenticationFailed(
                self.error_messages['no_active_account'],
                'no_active_account',
            )

class StaffTokenObtainPairView(TokenObtainPairView):
    serializer_class = StaffTokenObtainPairSerializer


# geocoding

from zope import interface, component

from getpaid.core.interfaces import IStoreSettings, IShippableLineItem, IOrder, IOriginRouter
from getpaid.core.payment import ContactInformation

import interfaces

class OriginRouter( object ):

    " warehouse aware origin router "

    component.adapts( IOrder )

    interface.implements( IOriginRouter )

    def __init__( self, context ):
        self.context = context

    def getOrigin( self ):
        warehouses = self.getOrderWarehouses()
        if not warehouses:
            fallback_router = component.getAdapter( self.context, IOriginRouter, name="default" )
            return fallback_router.getOrigin()
        assert len(warehouses) == 1, "only one origin warehouse allowed per order, atm"
        warehouse = warehouses.pop()
        contact = self.getStoreContact()
        contact.name = warehouse.name
        return contact, warehouse.location

    def getOrderWarehouses( self ):
        items = filter( IShippableLineItem.providedBy, self.context.shopping_cart.values())
        if not items:
            return

        warehouse_names = set()
        for i in items:
            shippable = i.resolve()
            if shippable is None:
                continue
            inventory = interfaces.IProductInventory( shippable )
            warehouse_names.add( inventory.warehouse )

        container = component.getUtility( interfaces.IWarehouseContainer )
        return filter( None, map( container.get, warehouse_names ) )

    def getStoreContact( self ):
        store_settings = component.getUtility( IStoreSettings )
        contact = ContactInformation( name = ( store_settings.contact_company or store_settings.store_name ),
                                      phone_number = store_settings.contact_phone,
                                      email = store_settings.contact_email )
        return contact


from ..binary import BinaryReader, BinaryWriter
from ..binary.types import Int8, Int16, Int32, UInt8, UInt16, UInt32

class HKHeader:
    """Represents Havok file header
    """

    # fmt:off
    magic0: UInt32 = 0x57E0E057  # 0x00  # Always 0x57E0E057
    magic1: UInt32 = 0x10C0C010  # 0x04  # Always 0x10C0C010
    user_tag: Int32 = 0x0  # 0x08  # Always 0x00 (?)
    version: Int32 = 0x0B  # 0x0C  # Always 0x0B

    pointer_size: Int8  # 0x10  # 4 on WiiU, 8 on Switch
    endian: Int8  # 0x11  # 0 (big) on WiiU, 1 (little) on Switch
    padding_option: Int8  # 0x12  # 0 on WiiU, 1 on Switch
    base_class: Int8  # 0x13  # Always 1

    section_count: Int32 = 3  # 0x14  # Always 3 (classnames, types, data)
    data_section_id: Int32 = 2  # 0x18  # Always 2, refers to __data__ section
    data_section_offset: UInt32 = 0  # 0x1C  # Always 0, beginning offset in __data__
    classnames_section_id: Int32 = 0  # 0x20  # Always 0, refers to __classnames__ section
    classnames_section_offset: UInt32 = 0x4B  # 0x24  # Always 75, offset to hkRootLevelContainer?
    hk_version: str = "hk_2014.2.0-r1"  # 0x28  # Always same, terminated with 0xFF
    flags: Int32 = 0  # 0x38  # Always 0x00 (?)
    max_predicate: Int16 = 21  # 0x3C  # Always 0x15 (?)
    predicate_size_plus_padding: UInt16 = 0  # 0x3E  # 0 or 16; additional data (?)

    unk40: Int16 = 20  # 0x40  # Always 20
    unk42: Int16 = 0  # 0x42  # Always 0 (padding?)
    unk44: Int32 = 0  # 0x44  # Always 0 (padding?)
    unk48: Int32 = 0  # 0x48  # Always 0 (padding?)
    unk4C: Int32 = 0  # 0x4C  # Always 0 (padding?)

    # fmt:on

    def to_switch(self):
        self.pointer_size = 8
        self.endian = 1
        self.padding_option = 1
        self.base_class = 1

    def to_wiiu(self):
        self.pointer_size = 4
        self.endian = 0
        self.padding_option = 0
        self.base_class = 1

    def read(self, br: BinaryReader):
        # Read the header
        self.magic0 = br.assert_uint32(0x57E0E057)
        self.magic1 = br.assert_uint32(0x10C0C010)
        self.user_tag = br.assert_int32(0x0)
        self.version = br.assert_int32(0x0B)

        self.pointer_size = br.assert_int8(4, 8)
        self.endian = br.assert_int8(0, 1)
        self.padding_option = br.assert_int8(0, 1)
        self.base_class = br.assert_int8(1)

        self.section_count = br.assert_int32(3)
        self.data_section_id = br.assert_int32(2)
        self.data_section_offset = br.assert_uint32(0x0)

        self.classnames_section_id = br.assert_int32(0)
        self.classnames_section_offset = br.assert_uint32(0x4B)
        self.hk_version = br.assert_string("hk_2014.2.0-r1", size=15)
        br.assert_uint8(0xFF)

        self.flags = br.assert_int32(0x0)
        self.max_predicate = br.assert_int16(0x15)
        self.predicate_size_plus_padding = br.assert_uint16(0, 16)

        if self.predicate_size_plus_padding == 0:
            pass
        elif self.predicate_size_plus_padding == 16:
            self.unk40 = br.assert_int16(0x14)
            self.unk42 = br.assert_int16(0x0)
            self.unk44 = br.assert_int32(0x0)
            self.unk48 = br.assert_int32(0x0)
            self.unk4C = br.assert_int32(0x0)
        else:
            raise NotImplementedError("Invalid predicate size? Idk really")

    def write(self, bw: BinaryWriter):
        bw.write_uint32(self.magic0)
        bw.write_uint32(self.magic1)
        bw.write_int32(self.user_tag)
        bw.write_int32(self.version)

        bw.write_int8(self.pointer_size)
        bw.write_int8(self.endian)
        bw.write_int8(self.padding_option)
        bw.write_int8(self.base_class)
        bw.write_int32(self.section_count)
        bw.write_int32(self.data_section_id)
        bw.write_uint32(self.data_section_offset)

        bw.write_int32(self.classnames_section_id)
        bw.write_uint32(self.classnames_section_offset)
        bw.write_string(self.hk_version, size=15)
        bw.write_uint8(UInt8(0xFF))

        bw.write_int32(self.flags)
        bw.write_int16(self.max_predicate)
        bw.write_uint16(self.predicate_size_plus_padding)

        if self.predicate_size_plus_padding == 16:
            bw.write_int16(self.unk40)
            bw.write_int16(self.unk42)
            bw.write_int32(self.unk44)
            bw.write_int32(self.unk48)
            bw.write_int32(self.unk4C)

    def as_dict(self):
        if self.predicate_size_plus_padding == 16:
            return {
                "unk40": self.unk40,
                "unk42": self.unk42,
                "unk44": self.unk44,
                "unk48": self.unk48,
                "unk4C": self.unk4C,
            }
        elif self.predicate_size_plus_padding == 0:
            return None
        else:
            raise NotImplementedError()

    @classmethod
    def from_dict(cls, d: dict):
        inst = cls()
        if d:
            inst.predicate_size_plus_padding = UInt16(16)
            inst.unk40, inst.unk42, inst.unk44, inst.unk48, inst.unk4C = d.values()

        return inst

    def __repr__(self):
        return f"{self.__class__.__name__}({self.pointer_size}, {self.endian}, {self.padding_option}, {self.base_class}, {self.predicate_size_plus_padding})"


class Home:
    def __init__(self):
        self.food = 50
        self.cat_food = 30
        self.money = 0
        self.people = []
        self.dirt = 0

    def add_peoplе(self, human):
        self.people.append(human)
        human.house = self


# -*- coding: utf-8 -*-
"""HTTP Endpoint for Netflix session management"""
from __future__ import absolute_import, division, unicode_literals

import json
import BaseHTTPServer

from SocketServer import TCPServer
import resources.lib.common as common

from .nfsession import NetflixSession

class NetflixHttpRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    """Handles & translates requests from Inputstream to Netflix"""
    # pylint: disable=invalid-name, broad-except
    def do_HEAD(self):
        """Answers head requests with a success code"""
        self.send_response(200)

    def do_POST(self):
        """Loads the licence for the requested resource"""
        common.debug('Handling HTTP IPC call to {}'.format(self.path[1:]))
        func = getattr(NetflixSession, self.path[1:])
        length = int(self.headers.get('content-length', 0))
        data = json.loads(self.rfile.read(length)) or None
        result = func(self.server.netflix_session, data)
        self.send_response(200
                           if (not isinstance(result, dict) or
                               'error' not in result)
                           else 500)
        self.end_headers()
        self.wfile.write(json.dumps(result))
        self.finish()

    def log_message(self, *args):
        # pylint: disable=arguments-differ
        """Disable the BaseHTTPServer Log"""
        pass

class NetflixTCPServer(TCPServer):
    """Override TCPServer to allow usage of shared members"""
    def __init__(self, server_address):
        """Initialization of MSLTCPServer"""
        common.log('Constructing NetflixTCPServer')
        self.netflix_session = NetflixSession()
        TCPServer.__init__(self, server_address, NetflixHttpRequestHandler)


# -*- coding: utf-8 -*-

"""Main module."""
import pandas as pd
import csv
from genedb import cleanerfunc

# def removespecchar(test):
#     import re
#     if type(test) == str:
#         test2=re.sub('\t','',test)
#         test=re.sub('\"','',test2)
#     return(test)
"""String Cleaning Function."""

"""Function to create bed file.
   Changes excel file into seperate bed files"""
def createbed(dfgenes, disease, sheetindex):
    df=pd.read_excel("data/MyeloidGeneKB01.xlsx", sheetname=sheetindex, index_col="GeneSymbol", parse_cols = [0,1,2,3,4])
    df.columns= disease +"_"+ df.columns
    dfgenes2 = dfgenes.join(df,  how="outer")
    dfgenes2.reset_index(inplace=True)
    dfgenes2=dfgenes2.fillna("")
    dfgenes2=dfgenes2.applymap(cleanerfunc.removespecchar)
    collist=list(dfgenes2.columns)
    newcollist=collist[6:9] + collist[0:6] + collist[9:]
    dfgenes2.to_csv("data\\" + disease+".bed", sep='\t', columns=newcollist, index=False)

dfgenes=pd.read_excel("data/MyeloidGeneKB01.xlsx", sheetname=0,index_col="GeneSymbol", dtypes=object)
myeloidDiseaseList = ["Unknown", "AML", "MDS", "MPN", ]
myeloidSheetList = [1,2,3,4]
for disease, sheetindex in zip (myeloidDiseaseList, myeloidSheetList):
    createbed(dfgenes,disease,sheetindex)


import matplotlib.pyplot as plt
import plotly.graph_objs as go
from plotly.offline import plot
from plotly.subplots import make_subplots

def plot_clandlestics(df):

    fig = go.Figure(data=[go.Candlestick(x=df['date'],
                                     open=df['open'],
                                     high=df['high'],
                                     low=df['low'],
                                     close=df['close']),
                      go.Scatter(x=df['date'], y=df['MA_close_21'], line=dict(color='orange', width=2)),
                      go.Scatter(x=df['date'], y=df['MA_close_100'], line=dict(color='green', width=2))])

    fig.update_layout(xaxis_rangeslider_visible=False)

    plot(fig, filename='go_candle1.html')

def plot_cross_ma(df):
    plt.figure(figsize=(15,5))
    plt.plot(df['close'], label='Close')
    plt.plot(df['MA9'], label='MA9')
    plt.plot(df['MA21'], label='MA21')

    plt.plot(df[df['PositionCrossMA'] == 1].index,
            df['MA20'][df['PositionCrossMA'] == 1],
            '^', markersize = 15, color = 'g', label = 'buy')

    plt.plot(df[df['PositionCrossMA'] == -1].index,
            df['MA20'][df['PositionCrossMA'] == -1],
            'v', markersize = 15, color = 'r', label = 'sell')
    plt.title('Price chart ')
    plt.show()
    return None

def plot_price(df, attr):
    # plot price
    plt.figure(figsize=(15,5))
    plt.plot(df[attr])
    plt.title('Price chart ' + attr)
    plt.show()
    return None

def plot_RSI(df, upper=80, down=20):
    # plot correspondingRSI values and significant levels
    plt.figure(figsize=(15,5))
    plt.title('RSI chart')
    plt.plot(df['RSI'])

    plt.axhline(0, linestyle='--', alpha=0.1)
    plt.axhline(down, linestyle='--', alpha=0.5)
    plt.axhline(30, linestyle='--')

    plt.axhline(70, linestyle='--')
    plt.axhline(upper, linestyle='--', alpha=0.5)
    plt.axhline(100, linestyle='--', alpha=0.1)
    plt.show()
    return None

def plot_stoch_RSI(df, upper=80, down=20):
    # plot corresponding Stoch RSI values and significant levels
    plt.figure(figsize=(15,5))
    plt.title('stochRSI chart')
    plt.plot(df['K'])
    plt.plot(df['D'])

    plt.axhline(0, linestyle='--', alpha=0.1)
    plt.axhline(down, linestyle='--', alpha=0.5)
    #plt.axhline(30, linestyle='--')

    #plt.axhline(70, linestyle='--')
    plt.axhline(80, linestyle='--', alpha=0.5)
    plt.axhline(upper, linestyle='--', alpha=0.1)
    plt.show()
    return None


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"Interval detection: finding flat sections in the signal"
import numpy as np

from  utils        import initdefaults
from  signalfilter import PrecisionAlg, CppPrecisionAlg
from .splitting    import SplitDetector, PyMultiGradeSplitDetector
from .merging      import PyMultiMerger, EventMerger, PyEventSelector
from ._core        import EventDetector as _EventDetector # pylint: disable=import-error

class PyEventDetector(PrecisionAlg):
    """
    Detect, merge and select flat intervals in `PHASE.measure`

    # Attributes

    * `split`: splits the data into too many intervals. This is based on a grade
    computed for each frame indicating the likelihood that an event is finished.
    See `eventdetection.splitting` for the available grades.

    * `merge`: merges the previous intervals when the difference between their
    population is not statistically relevant.

    * `select`: possibly clips events and discards those too small.
    """
    split: SplitDetector = PyMultiGradeSplitDetector()
    merge: EventMerger   = PyMultiMerger()
    select               = PyEventSelector()
    @initdefaults(frozenset(locals()))
    def __init__(self, **kwa):
        super().__init__(**kwa)

    def compute(self, data:np.ndarray, precision: float) -> np.ndarray:
        "computes the intervals"
        return self.select(data, self.merge(data, self.split(data, precision), precision))

    def __call__(self, data:np.ndarray, precision: float = None)-> np.ndarray:
        precision = self.getprecision(precision, data)
        return self.select(data, self.merge(data, self.split(data, precision), precision))

    @classmethod
    def run(cls, *args, **kwa):
        "instantiates and calls class"
        return cls(**kwa)(*args)

class EventDetector(CppPrecisionAlg, _EventDetector, zattributes = ('precision', 'merge')):
    """
    Detect, merge and select flat intervals in `PHASE.measure`

    # Attributes

    * `split`: splits the data into too many intervals. This is based on a grade
    computed for each frame indicating the likelihood that an event is finished.
    See `eventdetection.splitting` for the available grades.

    * `merge`: merges the previous intervals when the difference between their
    population is not statistically relevant.

    * `select`: possibly clips events and discards those too small.
    """
    split: SplitDetector
    merge: EventMerger
    def __init__(self, **kwa):
        CppPrecisionAlg.__init__(self, **kwa)
        _EventDetector.__init__(self, **kwa)

    def __call__(self, data:np.ndarray, precision: float = None):
        return self.compute(data, self.getprecision(precision, data))

    @classmethod
    def run(cls, *args, **kwa):
        "instantiates and calls class"
        return cls(**kwa)(*args)


import logging
import psutil
import signal

from devil.android import device_errors
from devil.android import device_utils

def _KillWebServers():
  for s in [signal.SIGTERM, signal.SIGINT, signal.SIGQUIT, signal.SIGKILL]:
    signalled = []
    for server in ['lighttpd', 'webpagereplay']:
      for p in psutil.process_iter():
        try:
          if not server in ' '.join(p.cmdline):
            continue
          logging.info('Killing %s %s %s', s, server, p.pid)
          p.send_signal(s)
          signalled.append(p)
        except Exception: # pylint: disable=broad-except
          logging.exception('Failed killing %s %s', server, p.pid)
    for p in signalled:
      try:
        p.wait(1)
      except Exception: # pylint: disable=broad-except
        logging.exception('Failed waiting for %s to die.', p.pid)

def CleanupLeftoverProcesses(devices):
  """Clean up the test environment, restarting fresh adb and HTTP daemons.

  Args:
    devices: The devices to clean.
  """
  _KillWebServers()
  device_utils.RestartServer()

  def cleanup_device(d):
    try:
      d.WaitUntilFullyBooted()
      d.RestartAdbd()
      d.EnableRoot()
      d.WaitUntilFullyBooted()
    except (device_errors.CommandFailedError,
            device_errors.CommandTimeoutError):
      logging.exception('Failed to clean up device. Attempting to continue.')

  device_utils.DeviceUtils.parallel(devices).pMap(cleanup_device)


import random
import time
import queue
import hashlib
from threading import *
from treelib import Node, Tree
from datetime import timedelta
import threading

lock = Lock() # Global LOCK (use to mutual exclusion)

n = 3 # Network cardinality (number of processes in the network)

growth_speed = 0.1 # regulates the growth rate of process blockchains

global_Δ = 2 # global time parameter Δ

# BROADCAST COMMUNICATION ABSTRACTION

# Channel of PROPOSAL messages
broadcast_proposal = {
		"1" : queue.Queue(),
        "2" : queue.Queue(),
        "3" : queue.Queue()
}

# Channel of VOTE messages
broadcast_vote = {
		"1" : queue.Queue(),
        "2" : queue.Queue(),
        "3" : queue.Queue()
}

# Channel of CLOCK messages
broadcast_clock = {
		"1" : queue.Queue(),
        "2" : queue.Queue(),
        "3" : queue.Queue()
}

# Given an epoch e, it randomly returns the proponent of this epoch (deterministically)
def proposer(e):
    random.seed(e)
    return str(random.randint(1, n)) # It's possible to change this implementation of random selection

# Returns an hash of a single Process identifier together with a Block object
def sign_block(process, block):
    signature = (process.identifier, block)
    return hash(signature)

# Returns an hash of a single Process identifier together with a Clock object
def sign_clock(process, clock):
    signature = (process.identifier, clock)
    return hash(signature)

def vrfy_block(process, block, signature):
    if sign_block(process,block) == signature:
        return True
    else:
        return False

# Class representing a single block
class Block:
    def __init__(self, epoch, TXs, hash=0):
        self.epoch = epoch # Block's local epoch
        self.TXs = TXs # Transactions
        self.hash = hash # Previous block hash (recursive reference to the chain)

# Class representing an entire tree of blocks
class BlockChain:
    def __init__(self):
        self.root = Tree()
        self.root.create_node(0, 0) # Genesis block

    # Adds a block to a blockchain
    def add_block(self, block):
        node = self.root.create_node(block.epoch, hash(block), block.hash)
        node.data = block

    # Print the blockchain graphically
    def print_chain(self):
        self.root.show()

    # Returns the list containing all the blocks of the entire blockchain
    def return_nodes_data(self):
        nodes = self.root.all_nodes()
        nodes_data = []
        for i in range(1,len(nodes)):
            nodes_data.append(nodes[i].data)
        if len(nodes_data) == 0:
            nodes_data.append(0)
        return nodes_data

    # Returns the list of identifiers of the leaves of the tree (blockchain)
    def leaves(self):
        leaves = self.root.leaves(nid=None)
        leaves_identifiers = []
        for i in range(len(leaves)):
            leaves_identifiers.append(leaves[i].identifier)
        return leaves_identifiers

    # returns the Block object with the major epoch in the blockchain
    def block_max_epoch(self):
        nodes = self.root.all_nodes()
        max_epoch = 0
        block_max = 0
        for i in range(len(nodes)):
            if nodes[i].tag > max_epoch:
                max_epoch = nodes[i].tag
                block_max = nodes[i].data
        return block_max

# Proposal message object
class Proposal:
    def __init__(self, sender, block, signature):
        self.sender = sender
        self.block = block
        self.signature = signature

# Vote message object
class Vote:
    def __init__(self, sender, block, signature):
        self.sender = sender
        self.block = block
        self.signature = signature

# Clock message object
class Clock:
    def __init__(self, sender, epoch, signature):
        self.sender = sender
        self.epoch = epoch
        self.signature = signature

# Class representing a single process
class Process(Thread):
    def __init__(self, identifier):
        super(Process, self).__init__()

        self.B = BlockChain() # Process blockchain, initialized to Genesis Block
        self.e = 1 # Number of local epoch
        self.has_voted = [False for i in range(200)]
        self.prop_rec = ['⊥' for i in range(200)]
        self.fchain = ['⊥' for i in range(200)] # Output: current finalized chain

        self.identifier = identifier
        self.TX = 'Transactions pool' # It simulates an entire transaction pool of a process

        self.Δ = global_Δ # Local time clock for epoch change
        self.votes = {} # Vote counter for each block
        self.clocks = {} # Clock counter for each epoch

    # Time passing simulator: every second the local Δ is decreased by 1
    def countdown(self):
        while self.Δ > 0:
            self.Δ = self.Δ - 1
            time.sleep(1)

    # Update the number of votes for each Block object that arrives
    def counter_votes(self):
        while True:
            vote = broadcast_vote[self.identifier].get() # It waits on the queue
            if vote.block not in self.votes:
                self.votes[vote.block] = 1
            else:
                self.votes[vote.block] = self.votes[vote.block] + 1 # aumenta il contatore di voti relativo a un blocco

            # TODO: FLOODING

            time.sleep(growth_speed)

    # Update the number of clocks for each Epoch object that arrives
    def counter_clocks(self):
        while True:
            clock = broadcast_clock[self.identifier].get() # It waits on the queue
            if clock.epoch not in self.clocks:
                self.clocks[clock.epoch] = 1
            else:
                self.clocks[clock.epoch] = self.clocks[clock.epoch] + 1

            # TODO: FLOODING

            time.sleep(growth_speed)

    '''
    EVENT 1: trigger when the value of 'e' changes
    '''
    def New_epoch_start(self):
        e_prec = 0
        while True:
            if self.identifier == proposer(self.e) and e_prec != self.e: # se io sono il proponente dell'epoca

                e_prec = self.e # to not repeat the function twice

                lb = self.B.block_max_epoch() # lb is the "last block"

                b = Block(self.e, self.TX, hash(lb)) # creation of a new block b

                my_proposal = Proposal(self, b, sign_block(self, b)) # create the Proposal object

                # Proposal object broadcast
                for k in broadcast_proposal:
                    # time.sleep(random.uniform(0.0, 3.0)) # It simulates different speeds in the network
                    broadcast_proposal[k].put(my_proposal)

            time.sleep(growth_speed)

    '''
    EVENT 2: acceptance of a Block proposal (first and only time!)
    '''
    def New_proposal(self):

        while True:
            proposal = broadcast_proposal[self.identifier].get() # It waits on the queue

            if vrfy_block(proposal.sender, proposal.block, proposal.signature) and proposal.sender.identifier == proposer(proposal.block.epoch) and self.prop_rec[proposal.block.epoch] == '⊥' :
                self.prop_rec[proposal.block.epoch] = proposal.block

            time.sleep(growth_speed)

    '''
    EVENT 3: counter of votes for blockchain update and change of epoch
    '''
    def Update_blockchain(self):
        while True:
            for b in self.votes:
                if self.votes[b] >= 2*n/3:

                    # TODO: aggiungere un meccanismo per controllare le firme

                    self.B.add_block(b)
                    self.votes[b] = -1

                    self.finalize()

                    self.e = max(self.e , b.epoch + 1) # update epoch
                    self.Δ = global_Δ # update Δ for waiting for the next age

            time.sleep(growth_speed)

    '''
    EVENT 4: broadcast of the clock message for possible epoch update
    '''
    def Clock_broadcast(self):
        while True:
            if self.Δ == 0:
                print('cloook')
                my_clock = Clock(self, self.e, sign_clock(self,self.e)) # Clock message object
                for k in broadcast_clock:
                    # time.sleep(random.uniform(0.0, 3.0)) # It simulates different speeds in the network
                    broadcast_clock[k].put(my_clock)
                self.Δ = -1 # To prevent it from continuing to send messages
            time.sleep(growth_speed)

    '''
    EVENT 5: clock counter for possible epoch update
    '''
    def Change_epoch(self):
        while True:
            for e in self.clocks:
                if self.clocks[e] >= 2*n/3:
                    # TODO: aggiungere un meccanismo per controllare le firme
                    self.e = max(self.e, e + 1) # Update current epoch
                    self.Δ = global_Δ # update Δ for waiting for the next age

            time.sleep(growth_speed)

    '''
    EVENT 6: Vote event
    '''
    def Vote(self):

        while True:
            b_parent = None

            for block in self.B.return_nodes_data():
                if self.prop_rec[self.e] != '⊥' and hash(block) == self.prop_rec[self.e].hash:
                    if self.B.block_max_epoch() == block:
                        b_parent = block
                        break

            if self.prop_rec[self.e] != '⊥' and self.has_voted[self.e] == False and b_parent != None:
                b = self.prop_rec[self.e]
                my_vote = Vote(self, b, sign_block(self,b))
                for k in broadcast_vote:
                    time.sleep(random.uniform(0.0, 7.0)) # It simulates different speeds in the network
                    broadcast_vote[k].put(my_vote)
                self.has_voted[self.e] = True

            time.sleep(growth_speed)

    # Updates the finalized chain
    def finalize(self):

        '''
        N = set() # N is the set of "normal blocks"
        for b in self.B.return_nodes_data(): # b is a block
            for b_1 in self.B.return_nodes_data(): # b_1 is a block
                if b.epoch + 1 == b_1.epoch and hash(b) == b_1.hash:
                    N.add(b)

        max_epoch = 0
        lnb = None # Last 'normal block'
        for b in N:
                if b.epoch >= max_epoch:
                    max_epoch = b.epoch
                    lnb = b

        self.fchain[max_epoch] = lnb

        '''

    def print_demonstration(self):
        while True:
            with lock:

                print('.............')
                print('blockchain of ' + self.identifier + ' (epoch ' + str(self.e) + '):')
                if proposer(self.e) == self.identifier:
                    print('PROPOSER')
                self.B.print_chain()
                print('.............')

            time.sleep(1)

    def run(self):

        # Sotto-thread (contatori e altro)
        p1 = Thread(target = self.countdown)
        p2 = Thread(target = self.counter_votes)
        p3 = Thread(target = self.counter_clocks)

        # Sotto-thread (EVENTI)
        e1 = Thread(target = self.New_epoch_start)
        e2 = Thread(target = self.New_proposal)
        e3 = Thread(target = self.Update_blockchain)
        e4 = Thread(target = self.Clock_broadcast)
        e5 = Thread(target = self.Change_epoch)
        e6 = Thread(target = self.Vote)

        p1.start()
        p2.start()
        p3.start()
        e1.start()
        e2.start()
        e3.start()
        e4.start()
        e5.start()
        e6.start()

        print_demonstration = Thread(target = self.print_demonstration)
        print_demonstration.start()

def main():

    process_1 = Process("1")
    process_1.start()

    process_2 = Process("2")
    process_2.start()

    process_3 = Process("3")
    process_3.start()

if __name__ == '__main__':
    main();


import os

def pdf_to_html(filename:str, pdfname:str):
    """
    filename是路径，到当前目录就可以，
    pdfname是pdf的名称，需要带后缀pdf，
    该方法需要本地安装docker desktop才可使用
    目前设置pdf和html都在同一个路径
    """
    str = "docker run -i --rm -v "
    second_str = ":/pdf bwits/pdf2htmlex pdf2htmlEX --zoom 1.3 "
    all_str = str + filename + second_str + pdfname
    os.system(all_str)

if __name__ == '__main__':
    basepath = os.path.dirname(__file__)
    pdf_to_html(filename = basepath + "/Datas/", pdfname="如何才能摆脱贫穷？穷人和富人有什么差别？.pdf")


# pygame python第三方游戏库   .pyd 动态模块(可以导入,但是看不到源码) .py 静态模块
import pygame  # 官方推荐这样导入
from pygame.locals import *
import sys
import time
import random

# 定义常量记录数据 (常量特点: 字母全大写  一旦定义,不要修改记录的值)
WINDOW_H = 768
WINDOW_W = 512

class Bullet:  # 子弹类
    def __init__(self, img_path, x, y, window):
        self.img = pygame.image.load(img_path)  # 子弹图片
        self.x = x  # 子弹坐标
        self.y = y
        self.window = window  # 子弹所在的窗口

    def display(self):
        """贴子弹图"""
        self.window.blit(self.img, (self.x, self.y))

    def move(self):
        """向上飞"""
        self.y -= 15

    # def __del__(self):
    #     print("子弹将要被删除")

class EnemyPlane:  # 敌人飞机
    def __init__(self, img_path, x, y, window):
        self.img = pygame.image.load(img_path)  # 飞机图片
        self.x = x  # 飞机坐标
        self.y = y
        self.window = window  # 飞机所在的窗口

    def display(self):
        """贴飞机图"""
        self.window.blit(self.img, (self.x, self.y))

    def move(self):
        """向下飞"""
        self.y += 10
        # 判断是否飞出边界
        if self.y >= WINDOW_H:  # 重新回到窗口顶部
            self.y = random.randint(-500, -70)
            self.x = random.randint(0, WINDOW_W - 100)
            self.img = pygame.image.load("res/img-plane_%d.png" % random.randint(1, 7))

class HeroPlane:  # 英雄飞机类
    def __init__(self, img_path, x, y, window):
        self.img = pygame.image.load(img_path)  # 飞机图片
        self.x = x  # 飞机坐标
        self.y = y
        self.window = window  # 飞机所在的窗口
        self.bullets = []  # 记录发出的所有子弹

    def display(self):
        """贴飞机图"""
        self.window.blit(self.img, (self.x, self.y))  # 290 500

    def move_left(self):
        """左移"""
        self.x -= 5

    def move_right(self):
        """右移"""
        self.x += 5

    def fire(self):
        """发射子弹"""
        # 创建子弹对象
        # 子弹x = 飞机x + 飞机宽度一半 - 子弹宽度一半
        # 子弹y =  飞机y - 子弹高度
        bullet = Bullet("res/bullet_9.png", self.x + 50, self.y - 31, self.window)
        self.bullets.append(bullet)

    def display_bullets(self):
        """贴子弹图"""
        out_window_bullets = []
        for bullet in self.bullets:
            # 判断子弹是否飞出边界
            if bullet.y >= -31:  # 在边界内,显示和移动
                bullet.display()
                bullet.move()
            else:  # 出了边界,删除子弹
                # self.bullets.remove(bullet)
                out_window_bullets.append(bullet)

        for out_window_bullet in out_window_bullets:
            self.bullets.remove(out_window_bullet)

class Map:  # 地图类
    def __init__(self, img_path, x, y, window):
        self.img = pygame.image.load(img_path)  # 地图图片
        self.x = x  # 地图坐标
        self.y = y
        self.window = window  # 地图所在的窗口

    def display(self):
        """贴地图"""
        self.window.blit(self.img, (self.x, self.y))  # 290 500

def main():  # 一般将程序的入口定义为main函数
    """主函数"""
    # 1. 创建窗口
    window = pygame.display.set_mode((WINDOW_W, WINDOW_H))
    # 加载图片
    # bg_img = pygame.image.load("res/img_bg_level_1.jpg")
    # 创建地图对象
    map = Map("res/img_bg_level_1.jpg", 0, 0, window)
    # 创建飞机对象
    hero_plane = HeroPlane("res/hero2.png", WINDOW_W / 2 - 60, WINDOW_H / 2 - 39, window)
    enemy_list = []  # 记录每架敌机
    for _ in range(5):
        enemy_plane = EnemyPlane('res/img-plane_%d.png' % random.randint(1, 7), random.randint(0, WINDOW_W-100), random.randint(-500, -70), window)
        enemy_list.append(enemy_plane)
    while True:
        # 贴背景图
        # window.blit(bg_img, (0, 0))
        map.display()
        # 贴飞机图
        hero_plane.display()
        for enemy in enemy_list:
            enemy.display()
            enemy.move()
        # 贴子弹图
        hero_plane.display_bullets()
        # 3.刷新窗口
        pygame.display.update()

        # 获取新事件
        for event in pygame.event.get():
            # 1. 鼠标点击关闭窗口事件
            if event.type == QUIT:
                print("点击关闭窗口按钮")
                sys.exit()  # 关闭程序
            # 2. 键盘按下事件
            if event.type == KEYDOWN:
                # 判断用户按键
                if event.key == K_SPACE:
                    hero_plane.fire()

        # 检测键盘长按事件
        pressed_keys = pygame.key.get_pressed()
        if pressed_keys[K_a] or pressed_keys[K_LEFT]:
            # x -= 5
            hero_plane.move_left()
        if pressed_keys[K_d] or pressed_keys[K_RIGHT]:
            # x += 5
            hero_plane.move_right()
        time.sleep(0.02)  # 让循环每执行一次,休息一段时间  让CPU可以做别的工作

if __name__ == '__main__':
    main()


import pytest
from src.kbt.helpers import divide_by_lines, numbers

def test_divide_by_lines_error():
    data = [1, 2, 3, 4, 5, 6, 7, 8]
    with pytest.raises(Exception) as e:
        divide_by_lines(data, None)
    assert "Error trying to divide the data in None" in str(e)

def test_divide_by_lines_error():
    data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
    expected = [[1, 2, 3], [5, 6, 7], [9, 10, 11]]
    length = 4
    result = divide_by_lines(data, length)
    assert result is not None
    assert result == expected, "Result has to be like expected"

@pytest.mark.parametrize("test_input,expected",
                         [(" _ | ||_|", 0), ("     |  |", 1), (" _  _||_ ", 2), (" _  _| _|", 3),
                          ("   |_|  |", 4), (" _ |_  _|", 5), (" _ |_ |_|", 6), (" _   |  |", 7),
                          (" _ |_||_|", 8), (" _ |_| _|", 9)])
def test_numbers_invalid(test_input, expected):
    result = numbers[test_input]
    assert result is not None
    assert result == expected, "Result has to be like expected"