import sys

solveorsimplify = ''

def equalvalidator(equationparts):
	#This checks where the "=" sign is in the equation, then it figures out whether there is something on both sides of the equation
	if "=" in equationparts: #if there is an equal sign in the equation
		indexequal = equationparts.index("=") #setting a variable to the index of the equal sign in the list
		afterequal = indexequal + 1 #setting a variable to the value of the index directly after the equal sign
		highestindex = len(equationparts) - 1 #finding the highest index number in the list (basically checking whether there is something after the "=" sign in the equation)
		#print('highest index: '+str(highestindex)) #printing that value

		if highestindex < afterequal: #if there is not a value after the equal sign
			sys.exit('\nThis is not a valid equation')

		else: #if there is a value after the equal sign
			solveorsimplify = 'solve'
			return solveorsimplify

	else: #if there is not an equal sign in the equation
		solveorsimplify = 'simplify'
		return solveorsimplify


import logging
import os

import yaml

from page.app import App
from page.logit import logit

class TestVideoNews:
    """
    测试Video news tab页
    """
    def setup_class(self):
        logging.info("TestVideoNews:setup_class")
        # 获取测试数据文件名
        testcase_file = os.path.dirname(__file__).split("testcase")[0] + r"data/testcase_data/video/video_news.yaml"
        self.video_news_data = yaml.safe_load(open(testcase_file, "r", encoding="utf-8"))
        self.video_news_page = App.start_logged_in().to_video_home_page().click_news_tab()

    def setup_method(self, method):
        logging.info("TestVideoNews:setup_method")
        # 记录类名和方法名
        logging.info(type(self).__name__+":"+method.__name__)

    def teardown_class(self):
        logging.info("TestVideoNews:teardown_class")
        App.quit()

    @logit()
    def test_channels_card_title(self):
        """
        测试channels 卡片标题
        :return:
        """
        data_list = self.video_news_data["test_channels_card_title"]
        title = data_list["title"]
        actual_title = self.video_news_page.get_channels_card_title()
        assert actual_title == title

    @logit()
    def test_click_channels_more(self):
        """
        测试点击channels 查看更多
        :return:
        """
        data_list = self.video_news_data["test_click_channels_more"]
        title = data_list["title"]
        more_page = self.video_news_page.click_channels_more()
        actual_title = more_page.get_tool_bar_title()
        assert actual_title == title


#!/usr/bin/python

import sys

def sam_dic(sam):
	dic = {}
	for i in range(len(sam)):
		info = sam[i].split('\t')
		read = info[0]
		tag = int(info[8])
		if read not in dic.keys():
			dic[read] = ['','']
		if tag > 0:
			dic[read][0] = sam[i]
		elif tag < 0:
			dic[read][1] = sam[i]
	return dic

if __name__ == '__main__':
	sam0 = sys.argv[1]
	outputPath = sys.argv[2]
	with open(sam0,'r') as f:
		sam = f.read()
	sam = sam.split('\n')[:-1]
	samDic = sam_dic(sam)
	# output
	out = open(outputPath + '/' + sam0.split('.')[0] + '_1.fastq','w')
	out2 = open(outputPath + '/' + sam0.split('.')[0] + '_2.fastq','w')
	readLst = samDic.keys()
	for read in readLst:
		forwd,rev = samDic[read]
		if (forwd != '') and (rev != ''):
			forwd_info = forwd.split('\t')
			rev_info = rev.split('\t')
			forwd_seq = forwd_info[9]
			rev_seq = rev_info[9]
			forwd_qual = forwd_info[10]
			rev_qual = rev_info[10]
			if (len(forwd_seq) == len(forwd_qual)) and (len(rev_seq) == len(rev_qual)):
				forwd_string = '@' + read + '\n' + forwd_seq + '\n' + '+' + '\n' + forwd_qual + '\n'
				rev_string = '@' + read + '\n' + rev_seq + '\n' + '+' + '\n' + rev_qual + '\n'
				out.write(forwd_string)
				out2.write(rev_string)
	out.close()
	out2.close()


from .common import VLABEL, ELABEL
from scipy.io import loadmat
import networkx as nx

MUTAG_MAT_FILE_NAME = './data/MUTAG.mat'

def load_mutag():
    data = loadmat(MUTAG_MAT_FILE_NAME)
    # print(data.keys())
    y = data['lmutag'].reshape((-1,))
    # print('y', y.shape)
    N, = y.shape

    for i in range(N):
        graph = nx.Graph()
        molecule = data['MUTAG'][0, i]
        node_labels = list(molecule['nl'][0,0][0].reshape((-1,)))
        for node_id, node_label in enumerate(node_labels):
            graph.add_node(node_id, **{VLABEL:int(node_label)})

        edge_list = list(molecule['el'][0,0][0])

        for v, w, elabel in edge_list:
            graph.add_edge(v-1, w-1, **{ELABEL:int(elabel)})
        # print('y[i]', i, y[i])
        yield y[i], graph


class Solution:
    def maximumGroups(self, grades: List[int]) -> int:
        grades = sorted(grades)
        count = 0
        start_idx = 0

        cur_num = 0
        total_num = len(grades)
        cur_grade = 0
        total_grade = sum(grades)

        while total_num >= 0 or total_grade >= 0:
            if total_grade - cur_grade <= 0 or total_num - cur_num <= 0:
                return count
            cur_num += 1
            cur_grade = sum(grades[start_idx:start_idx+cur_num])
            count += 1
            start_idx += cur_num
            total_grade -= cur_grade
            total_num -= cur_num
        return count


'''
Created on 20.06.2011

@author: michi
'''

from sqlalchemy.orm import object_mapper
from sqlalchemy.orm.attributes import InstrumentedAttribute
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.ext.hybrid import hybrid_property

def decorator(objOrClassOrProperty):
    '''OrmDecorator decorator(ems.model.sa.orm.base_object.OrmBaseObject)'''
    if isinstance(objOrClassOrProperty, InstrumentedAttribute):
        if hasattr(objOrClassOrProperty.class_,'__ormDecorator__'):
            return objOrClassOrProperty.class_.__ormDecorator__()
    elif hasattr(objOrClassOrProperty, '__ormDecorator__'):
        return objOrClassOrProperty.__ormDecorator__()

def friendly_name(classProperty):
    if isinstance(classProperty, InstrumentedAttribute):
        if hasattr(classProperty.class_,'__ormDecorator__'):
            deco = classProperty.class_.__ormDecorator__()
            return deco.getPropertyFriendlyName(classProperty.key)
    elif hasattr(classProperty, '__ormDecorator__'):
        return classProperty.__ormDecorator__().\
            getReprasentiveString(classProperty)

class OrmDecorator(object):
    def __init__(self, cls):
        self._class = cls
        self._shownProperties = []
        self._objectMapper = None
        self._valueFormatOptions = {}
        self._rProperties = {}
        self._colInfos = {}

    def getObjectMapper(self):
        if self._objectMapper is None:
            self._objectMapper = object_mapper(self._class())
        return self._objectMapper

    objectMapper = property(getObjectMapper)

    def getReprasentiveString(self, obj, view='default'):
        if hasattr(self._class,'__reprasentive_column__'):
            col = self._class.__reprasentive_column__
            return self.valueToFormattedString(col,
                                               obj.__getattribute__(col))
        return repr(obj)

    def getFriendlyName(self, obj=None, view='default'):
        if hasattr(self._class, "__friendlyName__"):
            return self._class.__friendlyName__
        return self._class.__name__

    def getPropertyFriendlyName(self, propertyName):
        if hasattr(self._class, "__propertyFriendlyNames__"):
            notDottedProperty = str(propertyName).split('.')[-1:][0]
            if self._class.__propertyFriendlyNames__.has_key(notDottedProperty):
                return self._class.__propertyFriendlyNames__[notDottedProperty]

        return propertyName

    def getShownProperties(self):
        if not len(self._shownProperties):
            if hasattr(self._class,'__shownProperties__'):
                self._shownProperties = self._class.__shownProperties__
            else:
                for prop in self.objectMapper.iterate_properties:
                    self._shownProperties.append(prop.key)

        return self._shownProperties

    def getDefaultOrderByProperty(self, obj):
        if hasattr(self._class,'__reprasentive_column__'):
            return self._class.__reprasentive_column__

    def getDefaultAllQuery(self, session):
        return session.query(self._class).order_by(self.getDefaultOrderByProperty(None))

    def getAll(self, session):
        return self.getDefaultAllQuery(session).all()

    def getFullTextQuery(self, session, text):
        return None

    def valueToFormattedString(self, propertyName, value):
        if hasattr(value, '__ormDecorator__'):
            dec = value.__ormDecorator__()
            return dec.getReprasentiveString(value)
        formatOption = self.getValueFormatOption(propertyName)

        newString = "%s%s%s" % (unicode(formatOption['prefix']),
                                unicode(formatOption['textFormat']).format((value)),
                                unicode(formatOption['suffix'])
                                )
        return newString

    def getValueFormatOption(self, propertyName):
        prefix = u''
        suffix = u''
        textFormat = u'{0}'

        try:
            colInfo = self.getColInfo(propertyName)
            if colInfo.has_key('unit'):
                suffix = " " + unicode(colInfo['unit'])
            if colInfo.has_key('textFormat'):
                textFormat = u"{0:" + colInfo['textFormat'] + '}'
        except AttributeError:
            pass
        except KeyError:
            pass

        return {
                'prefix':prefix,
                'suffix':suffix,
                'textFormat':textFormat
                }

    def getColInfo(self, propertyName):
        if not self._colInfos.has_key(propertyName):
            cols = self.getRProperty(propertyName).columns
            if len(cols) == 1:
                col = cols[0]
            self._colInfos[propertyName] = col.info
        return self._colInfos[propertyName]

    def getRProperty(self, propertyName):
        if not self._rProperties.has_key(propertyName):
            try:
                self._rProperties[propertyName] = self.objectMapper.\
                    get_property(propertyName)
            except InvalidRequestError:
                clsProp = self._class.__getattribute__(self._class, propertyName)
                if isinstance(clsProp, hybrid_property):
                    functionName = clsProp.fget.__name__
                    self._rProperties[propertyName] = clsProp

        return self._rProperties[propertyName]


import typing

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from scipy.interpolate import interpn

def density_scatter(ax, x, y, bins, **kwargs) -> None:
    """Create a scatter plot colored by 2d histogram density.

    Parameters
    ----------
    ax: matplotlib.axes.Axes
    x: np.ndarray
    y: np.ndarray
    bins: int
    kwargs
        any kwargs passed to 'ax.scatter'

    Returns
    -------

    References
    ----------
    Adapted from https://stackoverflow.com/a/53865762/10504481

    """
    # convert e.g. DataFrame to numpy array values
    x = np.array(x)
    y = np.array(y)

    if "cmap" not in kwargs:
        kwargs["cmap"] = "viridis"

    data, x_e, y_e = np.histogram2d(x, y, bins=bins, density=True)
    points = (0.5 * (x_e[1:] + x_e[:-1]), 0.5 * (y_e[1:] + y_e[:-1]))
    xi = np.vstack([x, y]).T
    z = interpn(points, data, xi, method="splinef2d", bounds_error=False)
    # To be sure to plot all data
    z[np.where(np.isnan(z))] = 0.0
    # Sort the points by density, so that the densest points are plotted last
    idx = z.argsort()
    x, y, z = x[idx], y[idx], z[idx]
    ax.scatter(x, y, c=z, **kwargs)

def get_figure(
    true, prediction, datalabel: str, xlabel: str, ylabel: str, figsize: tuple = (10, 7)
) -> plt.Figure:
    """Create a correlation plot for true, prediction values.

    Parameters
    ----------
    true: the true values
    prediction: the predicted values
    datalabel: str, the label for the prediction, e.g. 'MAE: 0.123 eV'
    xlabel: str, the xlabel
    ylabel: str, the xlabel
    figsize: tuple, size of the figure

    Returns
    -------
    plt.Figure

    """
    sns.set()
    fig, ax = plt.subplots(figsize=figsize)
    ax.plot(true, true, color="grey", zorder=0)  # plot the diagonal in the background
    bins = 500 if (len(true) / 10) > 500 else int(len(true) * 0.1)
    if bins < 20:
        # don't use density for very small datasets
        ax.scatter(true, prediction, marker="x", s=20.0, label=datalabel)
    else:
        density_scatter(
            ax, true, prediction, bins=bins, marker="x", s=20.0, label=datalabel
        )
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.legend()
    return fig

def get_hist(data, label, xlabel, ylabel) -> typing.Tuple[plt.Figure, plt.Axes]:
    fig, ax = plt.subplots()

    sns.histplot(
        data,
        ax=ax,
        stat="percent",
        label=label,
    )
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.legend()

    return fig, ax

def get_histogram_figure(
    bin_edges,
    counts,
    datalabel: str,
    xlabel: str,
    ylabel: str,
    x_lim: typing.Tuple[float, float] = None,
    y_lim: typing.Tuple[float, float] = None,
    logy_scale=True,
    figsize: tuple = (10, 7),
) -> plt.Figure:
    """Creates a Matplotlib figure based on precomputed bin edges and counts.

    Parameters
    ----------
    bin_edges: np.array
        Edges of the histogram bins.
    counts: np.array
        Number of occurrences in each bin.
    datalabel: str
        Labels for the figure legend.
    xlabel: str
        X-axis label.
    ylabel: str
        Y-axis label.
    x_lim: tuple
        X-axis limits.
    y_lim: tuple
        Y-axis limits.
    figsize: tuple
        Size of the Matplotlib figure
    """
    sns.set()
    fig, ax = plt.subplots(figsize=figsize)

    ax.stairs(counts, bin_edges, label=datalabel, fill=True)

    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.legend()
    if logy_scale:
        ax.set_yscale("log")
    if x_lim is not None:
        ax.set_xlim(x_lim)
    if y_lim is not None:
        ax.set_ylim(y_lim)
    return fig


"""Utility for creating and inserting user data into the database"""
import logging

from faker import Faker

from db import database_connection

fake = Faker()

def insert_user_data_to_db():
    """Insert initial user data into the database"""
    insert_query = 'INSERT INTO public."User"(first_name, last_name, email) VALUES '
    query_bind_values = ""
    for user_count in range(5):
        first_name = fake.unique.first_name()
        last_name = fake.unique.last_name()
        email = f"{(first_name + last_name).lower()}@example.com"
        if user_count < 1:
            query_bind_values = f"('{first_name}','{last_name}','{email}')"
        else:
            query_bind_values = "".join(
                [query_bind_values, "," f"('{first_name}','{last_name}','{email}')"]
            )
    insert_query = "".join([insert_query, query_bind_values])
    conn = database_connection()
    conn.set_session(autocommit=True)
    cur = conn.cursor()
    try:
        cur.execute(insert_query)
        cur.close()
    except Exception as e:
        logging.error("Error executing query", e)
        conn.close()

if __name__ == "__main__":
    insert_user_data_to_db()


from pathlib import Path
from typing import AnyStr, Set

from datascience_model_commons.model import BaseModel, ModelException

import numpy as np
import pandas as pd
import tensorflow as tf
import yaml

DEFAULT_VALUES_DICT = {tf.string: (tf.string, ""), tf.float32: (tf.float32, 0.0)}

# Enable loading of metadata without model source code
class SafeLoaderIgnoreUnknown(yaml.SafeLoader):
    def ignore_unknown(self, node):
        return None

class Model(BaseModel):
    def __init__(self, model_path):
        model_directory = Path(__file__).parent
        with open(model_directory / "config.yaml", "r") as f:
            self.config_dict = yaml.load(f)
            self.test_description_prefix = self.config_dict.get(
                "TEST_DESCRIPTION_PREFIX"
            )

        from datascience_model_commons.transactions.business_rules.dutch_tax_rules import (
            apply_tax_rules as apply_nl_tax_rules,
        )
        from datascience_model_commons.transactions.business_rules.uk_tax_rules import (
            apply_tax_rules as apply_uk_tax_rules,
        )

        self.tax_function_dict = {
            "NL": apply_nl_tax_rules,  # function reference
            "GB": apply_uk_tax_rules,  # function reference
        }
        self.load_model(model_path=model_path)

    def load_model(self, model_path):
        def find_artifact(root):
            import os

            # Traverse root to find model signature files (.pb file extension)
            for root, dirs, files in os.walk(root):
                for file in files:
                    if file.endswith(".pb"):
                        return root

            raise ModelException("No .pb model artifact found!")

        try:
            artifact_location = find_artifact(root=model_path)
            model = tf.saved_model.load(artifact_location)
        except Exception as e:
            raise ModelException(f"Unable to load model {model_path}") from e

        metadata_path = f"{model_path}/training_metadata.yaml"
        try:
            with open(metadata_path, "r") as stream:
                SafeLoaderIgnoreUnknown.add_constructor(
                    None, SafeLoaderIgnoreUnknown.ignore_unknown
                )
                root = yaml.load(stream, Loader=SafeLoaderIgnoreUnknown)
                labels = root["labels"]
        except IOError as ioerr:
            raise ModelException(
                f'Error reading yaml config file: "{metadata_path}"'
            ) from ioerr
        except yaml.YAMLError as yerr:
            raise ModelException(f'Unable to parse "{metadata_path}"') from yerr

        self.model = model
        self.labels = np.array(labels)
        self.predict_function = self.model.signatures["serving_default"]
        self.input_tensors = [
            tensor
            for tensor in self.predict_function.inputs
            if "unknown" not in tensor.name
        ]
        self.input_description = {
            tensor.name.split(":")[0]: DEFAULT_VALUES_DICT.get(tensor.dtype)
            for tensor in self.input_tensors
        }

    def df_to_tensor_dict(self, df):
        N = df.shape[0]
        return {
            key: np.atleast_2d(
                df[key].fillna(default_value).values.astype(dtype=dtype.as_numpy_dtype)
                if key in df
                else np.repeat(default_value, N)
            ).T
            for key, (dtype, default_value) in self.input_description.items()
        }

    def _create_labels_set(self, trx_scores):
        return set(self.labels[trx_scores == 1.0])

    def scores_to_labels(self, scores):
        return np.apply_along_axis(self._create_labels_set, 1, scores)

    @staticmethod
    def append_labels(
        prefix: AnyStr,
        input_df: pd.DataFrame,
        add_label_column: AnyStr,
        label_column: AnyStr = "labels",
    ) -> pd.DataFrame:
        def map_to_label(record: pd.Series) -> Set[AnyStr]:
            current_labels = record[label_column]
            add_label = record[add_label_column]

            if type(add_label) == str:
                add_label = add_label.lower()
                add_label = f"{prefix}_{add_label}"
                current_labels.add(add_label)

            return current_labels

        input_df[label_column] = input_df.apply(map_to_label, axis=1)
        return input_df

    def add_tax_labels(self, input_df: pd.DataFrame):
        # Make sure country code is available
        if "country_code" not in input_df:
            input_df = input_df.assign(country_code="")
        else:
            input_df["country_code"].fillna("", inplace=True)

        # Add labels for all country_code tax rules
        for country_code, tax_func in self.tax_function_dict.items():
            country_code_match = input_df["country_code"] == country_code
            country_code_missing = input_df["country_code"] == ""
            subset_df = input_df.loc[country_code_match | country_code_missing]

            # Nothing to do continue to next country code
            if len(subset_df) == 0:
                continue

            # Temporarily assign country code if missing
            subset_df = tax_func(subset_df.assign(country_code=country_code))
            prefix = f"Tax_{country_code.lower()}"
            subset_df["labels"] = self.append_labels(
                prefix=prefix, input_df=subset_df, add_label_column="tax_label"
            )
        return input_df

    def apply_business_rules(self, input_df: pd.DataFrame) -> pd.DataFrame:
        """Apply business rules here, category column should be overwritten where business rules apply"""
        input_df = self.add_tax_labels(input_df=input_df)
        input_df = self.apply_test_description_rule(input_df=input_df)
        return input_df[["labels"]]

    def apply_test_description_rule(self, input_df: pd.DataFrame) -> pd.DataFrame:
        input_df.fillna("", inplace=True)
        is_test_transaction = input_df["description"].str.startswith(
            self.test_description_prefix
        )
        subset_df = input_df.loc[is_test_transaction]
        regex_pattern = "([A-Z][a-z_]+)"
        processed_labels = (
            subset_df["description"]
            .str.extractall(regex_pattern)
            .groupby(level=0)
            .agg(set)
        )
        input_df.loc[is_test_transaction, "labels"] = processed_labels.values
        return input_df

    def predict(self, *, df: pd.DataFrame):
        if df.empty:  # Nothing to predict so return empty dataframe
            scores = pd.Series(dtype=object)
        else:
            predictions = self.predict_function(**self.df_to_tensor_dict(df))
            scores = list(predictions.values())[0].numpy()

        return self.apply_business_rules(
            input_df=df.assign(
                labels=self.scores_to_labels(scores),
            )
        )


import curses

"""
Author: Leah Hackman
Date: April 2, 2014
"""

class Hangman_v:

    empty_hangman = [" _______ ",
                    "|/     | ",
                    "|        ",
                    "|        ",
                    "|        ",
                    "|        ",
                    "|        ",
                    "---------"]

    man = [[(2,7, "O")],[(3,7, "|"), (4,7, "|")], [(3,6, "\\")], [(3,8, "/")],  [(5,8, "\\")], [(5,6, "/")]]
    full_hangman= [" _______ ",
                    "|/     | ",
                    "|      O ",
                    "|     \|/",
                    "|      | ",
                    "|     / \\",
                    "|        ",
                    "---------"]

    def __init__(self, game, term_width, term_height,stdscr):
        self.game = game
        self.stdscr = stdscr

        """
        break up the screen into four panes:
        ----------------
        | a     |b     |
        |       |      |
        |-------|      |
        | c     |      |
        ----------------
        | d            |
        ----------------
        window a is for drawing the hangman
        window b is for recording the guesses
        window c is for showing the word (blanks and correct letters)
        window d is for communicating with the user
        """
        self.a = curses.newwin(int(term_height*4/6), int(term_width/2), 0,0)
        self.b = curses.newwin(int(term_height*4/6), int(term_width/2),
                                0, int(term_width/2))
        self.c = curses.newwin(int(term_height*1/6), term_width,
                                int(term_height*4/6), 0)
        self.d = curses.newwin(int(term_height*1/6), term_width,
                                int(term_height*5/6), 0)

        #initialize magenta colour pairing for printing out guessed letters.
        curses.init_pair(1, curses.COLOR_MAGENTA,curses.COLOR_BLACK)

    def draw_hangman(self):
        window = self.a
        v_offset = 1
        h_offset = 3
        for line in enumerate(Hangman_v.empty_hangman):
            window.addstr(line[0] +v_offset, h_offset, line[1])
        for i in range(min(self.game.num_errors, self.game.max_guesses)):
           for j in Hangman_v.man[i]:
               window.addstr(j[0]+v_offset, j[1]+h_offset, j[2])
        window.noutrefresh()

    def draw_wrong_letters(self):
        """
        Print out the letters they've guessed wrong
        and information about the number of guesses they've made
        """
        window = self.b
        window.addstr(0,0, "You've made {} guesses and {} errors so far".format(self.game.num_guesses, self.game.num_errors))
        window.addstr(1,0, "Wrong Letters Guessed: ")
        window.addstr(2, 0,", ".join(self.game.wrong_guesses()))

        #need to put the changes up for drawing but do not redraw them yet.
        window.noutrefresh()

    def draw_word_field(self):
        """
        Print out word field. Print ___ for characters we haven't guessed yet.
        Othewise print the character. If the most recent guessed letter was a
        hit, print it in MAGENTA :O
        """
        window= self.c
        for letter in enumerate(self.game.secret_word):
            if letter[1] in self.game.guesses:
                window.addstr(0, letter[0]*5 + 1," {} ".format(letter[1]), curses.A_UNDERLINE | (curses.color_pair(0) if letter[1] != self.game.most_recent_guess else curses.color_pair(1)) )
            else: window.addstr(0,letter[0]*5 + 1, "   ", curses.A_UNDERLINE)

        #need to put the changes up for drawing but do not redraw them yet.
        window.noutrefresh()

    def draw_ui_field(self):
        """
        Print some text to communicate with the user.
        Either ask for more input OR let them know the game is over.
        """
        window = self.d
        if self.game.solved():
            window.clear()
            window.addstr(0,0, "You've won! Congratulations!")
        elif self.game.game_over():
            window.clear()
            window.addstr(0,0, "It looks as though you've lost. Sorry!")
        else:
            if not self.game.most_recent_guess is None:
                window.addstr(0,0, "You guessed the letter {}. There are {} {}'s in the secret word.".format(self.game.most_recent_guess, self.game.num_matches(self.game.most_recent_guess), self.game.most_recent_guess))
            window.addstr(1,0, "Please enter a character to guess")

        #need to put the changes up for drawing but do not redraw them yet.
        window.noutrefresh()

    def draw_screen(self):
        #each draw command updates what should be drawn and then
        #waits to be drawn. The doupdate call draws all the changes
        #at once.
        self.draw_hangman()
        self.draw_wrong_letters()
        self.draw_word_field()
        self.draw_ui_field()
        curses.doupdate()


from django.core.management.base import BaseCommand
from hotel.models import Hotel

# command delete_hotel
class Command(BaseCommand):
    help = 'Deletes hotels based on criteria'

    def add_arguments(self, parser):
        parser.add_argument('hotel_name', nargs='+', type=str, help='Name of the hotel to delete')

    def handle(self, *args, **kwargs):
        hotel_names = kwargs['hotel_name']
        for name in hotel_names:
            try:
                hotel = Hotel.objects.get(name=name)
                # Delete associated rooms before deleting the hotel
                rooms = hotel.rooms.all()
                for room in rooms:
                    room.delete()
                hotel.delete()
                self.stdout.write(self.style.SUCCESS(f'Successfully deleted {name} and its rooms'))
            except Hotel.DoesNotExist:
                self.stdout.write(self.style.WARNING(f'Hotel with name {name} does not exist'))


aunt_sue_dict ={}
MFCSAM_Key = {'children': 3,
              'cats': 7,
              'samoyeds': 2,
              'pomeranians': 3,
              'akitas': 0,
              'vizslas': 0,
              'goldfish': 5,
              'trees': 3,
              'cars': 2,
              'perfumes': 1}

with open('Project16_2015') as file:
    for line in file:
        if "\n" in line:
            end_line = -1
        else:
            end_line = None
        name = line[:line.find(':')]
        attribute1 = line[line.find(':')+2:line.find(':', line.find(':')+1)]
        attribute1_value = int(line[line.find(':', line.find(':')+1)+2: line.find(',')])
        attribute2 = line[line.find(',')+2:line.find(':', line.find(','))]
        attribute2_value = int(line[line.find(':', line.find(','))+2: line.rfind(',')])
        attribute3 = line[line.rfind(',')+2:line.rfind(':')]
        attribute3_value = int(line[line.rfind(':')+1: end_line])
        aunt_sue_dict[name] = {attribute1: attribute1_value,
                               attribute2: attribute2_value,
                               attribute3: attribute3_value,
                               }

def dict_remover(dict_var):
    for sues in dict_var:
        break_status = False
        for keys in MFCSAM_Key:
            if keys in dict_var[sues]:
                if keys == 'cats' or keys == 'trees':
                    if MFCSAM_Key[keys] >= dict_var[sues][keys]:
                        dict_var.pop(sues, None)
                        break_status = True
                        break
                elif keys == 'goldfish' or keys == 'pomeranians':
                    if MFCSAM_Key[keys] <= dict_var[sues][keys]:
                        dict_var.pop(sues, None)
                        break_status = True
                        break
                else:
                    if MFCSAM_Key[keys] != dict_var[sues][keys]:
                        dict_var.pop(sues, None)
                        break_status = True
                        break
        if break_status:
            dict_remover(dict_var)
            break
    return dict_var

print(dict_remover(aunt_sue_dict))


from turtle import Turtle,register_shape

class HeroLaser(Turtle):

    def __init__(self):
        super().__init__()
        self.color("white")
        shape = ((-1,10),(-1,-10),(1,10),(1,-10))
        register_shape('hero-laser', shape)
        self.shape('hero-laser')
        self.penup()
        self.right(90)
        self.y_move = 10
        self.move_speed = 0.1
        self.goto(self.xcor(),-260)
        self.moving = False

    def fire(self):
        new_x = self.xcor()
        new_y = self.ycor() + self.y_move
        self.goto(new_x, new_y)
        self.moving = True

    def move(self, ship_x):
        if self.moving == True:
            new_x = self.xcor()
            new_y = self.ycor() + self.y_move
            self.goto(new_x, new_y)
        else:
            self.goto(ship_x,-260)
        if self.ycor() > 300:
            self.moving = False
            self.goto(ship_x,-260)

    def hit(self, ship_x):
        self.moving = False
        self.goto(ship_x,-260)


#! /usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import os

import numpy as np
import matplotlib.pyplot as plt

def _build_arg_parser():
    p = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter,
        description=__doc__)

    p.add_argument('in_binary_matrix')
    p.add_argument('in_ground_truth')
    p.add_argument('out_png')

    return p

def main():
    parser = _build_arg_parser()
    args = parser.parse_args()

    in_matrix = np.load(args.in_binary_matrix)
    gt = np.load(args.in_ground_truth)
    confusion_matrix = gt
    tp = 0
    fp = 0
    tn = 0
    fn = 0

    [i, j] = in_matrix.shape
    for x in range(0, i):
        for y in range(0, j):
            if in_matrix[x][y] == 1:
                if gt[x][y] == 1:
                    tp += 1
                    confusion_matrix[x][y] = 1
                if gt[x][y] == 0:
                    fp += 1
                    confusion_matrix[x][y] = 2
            if in_matrix[x][y] == 0:
                if gt[x][y] == 1:
                    fn += 1
                    confusion_matrix[x][y] = 4
                if gt[x][y] == 0:
                    tn += 1
                    confusion_matrix[x][y] = 3
    total = tp + fp + tn + fn

    print("True positives : " + str(100 * tp/total) + "%")
    print("False positives : " + str(100 * fp/total) + "%")
    print("True negatives : " + str(100 * tn/total) + "%")
    print("False negatives : " + str(100 * fn/total) + "%")

    fig, ax = plt.subplots()
    im = ax.imshow(confusion_matrix.T,
                   interpolation='nearest',
                   cmap='coolwarm')
    cbar = fig.colorbar(im, ticks=[1, 2, 3, 4])
    cbar.ax.set_yticklabels(['True Positives', 'False Positives',
                             'True Negatives', 'False Negatives'])

    plt.savefig(args.out_png, dpi=300, bbox_inches='tight')

if __name__ == "__main__":
    main()


import logging

import pygame

import environment
import globals
import script
from bush import animation, asset_handler, entity, physics, timer, util

loader = asset_handler.AssetHandler("sprites")
tile_loader = asset_handler.AssetHandler("tiled/tilesets")

logger = logging.getLogger(__name__)

class GameObject(entity.Actor):
    registry_groups = ("main",)

    def __init__(
        self,
        data,
        anim_dict=None,
        initial_state=None,
        physics_data=None,
        start_health=1,
        max_health=1,
        immunity=150,
        hit_effect=None,
    ):
        surface = data.surface
        if surface is None and anim_dict is not None:
            surface = anim_dict[min(anim_dict.keys())].image()
        super().__init__(
            data.pos,
            surface,
            (data.registry.get_group(i) for i in self.registry_groups),
            data.id,
            data.layer,
            data.topleft,
        )
        self.registry = data.registry
        self.anim_dict = {}
        if anim_dict is not None:
            self.anim_dict = anim_dict
        if initial_state is not None:
            self.state = initial_state
        elif self.anim_dict:
            self.state = "idle"
        else:
            self.state = None
        self.pushed_state = None
        if physics_data is not None:
            self.physics_data = physics_data
        else:
            self.physics_data = physics.PhysicsData(
                physics.TYPE_STATIC, pygame.sprite.Group()
            )
        self.current_health = start_health
        self.health_capacity = max_health
        self.desired_velocity = pygame.Vector2()
        self.move_state = "walk"
        self.idle_state = "idle"
        self.facing = "down"
        self.immunity_time = immunity
        self.immunity_timer = timer.DTimer(immunity)
        self.immunity_timer.finish()
        self.visual_effects = []
        self.hit_effect = hit_effect
        self.script_stack = []
        self.spawn_point = self.pos3.copy()
        self.spawn_groups = tuple(self.groups())

    def get_anim_key(self):
        return self.state

    def add_visual_effect(self, effect):
        self.visual_effects.append(effect)

    def heal(self, amount):
        self.current_health = min(self.health_capacity, self.current_health + amount)

    def respawn(self, health=None):
        self.pos3 = pygame.Vector3(self.spawn_point)
        if health is None:
            self.current_health = self.health_capacity
        else:
            self.current_health = pygame.math.clamp(health, 0, self.health_capacity)
        for group in self.spawn_groups:
            group.add(self)

    def hurt(self, amount):
        logger.info(f"Sprite {self._id} hurt by {amount}", self, amount)
        if self.immunity_timer.done():
            self.current_health -= amount
            if self.current_health <= 0:
                self.kill()
            self.immunify()

    def immunify(self):
        if self.hit_effect is not None:
            self.hit_effect.reset()
            self.add_visual_effect(self.hit_effect)
        self.immunity_timer.reset()

    def push_state(self, state):
        self.pushed_state = state

    def move(self, direction):
        self.desired_velocity = direction

    def stop(self):
        self.desired_velocity *= 0

    def run_script(self, script_name):
        self.script_stack.append(
            script.Script(self.get_id(), self.registry, script_name)
        )

    def update_state(self, dt):
        if self.desired_velocity:
            if self.state == self.idle_state:
                self.state = self.move_state
            self.facing = util.round_string_direction(
                util.string_direction(self.desired_velocity)
            )
        if not self.desired_velocity and self.state == self.move_state:
            self.state = self.idle_state
        if self.pushed_state:
            self.state = self.pushed_state

    def update_image(self, dt):
        if self.anim_dict:
            self.anim = self.anim_dict[self.get_anim_key()]
        if self.anim:
            self.image = self.anim.image()
        self.visual_effects = [e for e in self.visual_effects if not e.done()]
        for effect in self.visual_effects:
            effect.update(dt)
            self.image = effect.apply(self.image)

    def update_physics(self, dt):
        self.pos3 += self.velocity * dt
        self.update_rects()

    def update_behaviour(self, dt):
        pass

    def update_script(self, dt):
        if self.script_stack:
            self.script_stack[-1].update(dt)
            if self.script_stack[-1].finished():
                self.script_stack = self.script_stack[:-1]

    def update_rects(self):
        self.rect.center = self.pos

    def update(self, dt):
        self.update_state(dt)
        self.update_physics(dt)
        self.update_image(dt)
        self.update_behaviour(dt)
        self.update_script(dt)
        self.immunity_timer.update(dt)

    @staticmethod
    def spawn_particles(particles):
        globals.engine.stack.get_current().add_particles(particles)

class MobileGameObject(GameObject):
    def __init__(
        self,
        data,
        anim_dict=None,
        initial_state=None,
        physics_data=None,
        start_health=1,
        max_health=1,
        immunity=150,
        hit_effect=None,
        weight=0.5,
    ):
        super().__init__(
            data=data,
            anim_dict=anim_dict,
            initial_state=initial_state,
            physics_data=physics_data,
            start_health=start_health,
            max_health=max_health,
            immunity=immunity,
            hit_effect=hit_effect,
        )
        self.facing = "down"
        if physics_data is None:
            self.physics_data = physics.PhysicsData(
                physics.TYPE_STATIC, pygame.sprite.Group()
            )
        self.collision_rect = self.rect.copy()
        self.mobile = True
        self.knockback_remaining = pygame.Vector2()
        self.weight = weight
        self.force = pygame.Vector2()

    def knockback(self, from_pos, amount):
        self.force += (self.pos - from_pos).normalize() * amount * 200

    def get_current_environment(self):
        return environment.TERRAIN_DATA[
            self.registry.masks.collide_mask(
                pygame.Mask(self.collision_rect.size, True),
                self.collision_rect.topleft,
                *environment.TERRAIN_ORDER,
            )
            or "default"
        ]

    def mobilize(self):
        self.mobile = True

    def immobilize(self):
        self.mobile = False

    def face(self, dest):
        if isinstance(dest, str):
            self.facing = dest
        else:
            self.facing = util.string_direction(
                pygame.Vector2(util.direction_orthag(dest - self.pos))
            )

    def update_rects(self):
        self.collision_rect.center = self.pos
        self.rect.center = self.pos

    def get_anim_key(self):
        return f"{self.state} {self.facing}"

    def update_physics(self, dt):
        self.desired_velocity *= self.mobile
        self.update_rects()
        terrain = self.get_current_environment()
        # slippety-slide!
        if terrain.traction != 1:
            self.velocity += (self.desired_velocity * terrain.speed) * terrain.traction
        else:
            self.velocity = self.desired_velocity * terrain.speed
        self.force *= 1.5 - terrain.traction
        self.velocity += self.force

        physics.dynamic_update(self, dt)
        self.update_rects()

def get_anim_dict(path, size):
    frames = loader.load_spritesheet(path + ".png", size)
    return {
        "walk down": animation.Animation(frames[0:16:4], 150),
        "walk up": animation.Animation(frames[1:17:4], 150),
        "walk left": animation.Animation(frames[2:18:4], 150),
        "walk right": animation.Animation(frames[3:18:4], 150),
        "idle down": animation.Animation(frames[0:1]),
        "idle up": animation.Animation(frames[1:2]),
        "idle left": animation.Animation(frames[2:3]),
        "idle right": animation.Animation(frames[3:4]),
    }

def load_tile(tile_index, tileset="objects", tile_size=(16, 16)):
    return tile_loader.load_spritesheet(tileset + ".png", tile_size)[tile_index]

def load_tiles(tile_indices, tileset="objects", tile_size=(16, 16)):
    tiles = []
    for tile_index in tile_indices:
        tiles.append(load_tile(tile_index, tileset, tile_size))
    return tiles


import pandas

data = pandas.read_csv("2018_Central_Park_Squirrel_Census_-_Squirrel_Data.csv")
Gray_squierrels_count = len(data[data["Primary Fur Color"]== "Gray"])
Cinnamon_squierrels_count = len (data[data["Primary Fur Color"]== "Cinnamon"])
black_squirrels_count = len(data[data["Primary Fur Color"]== "Black"])

print (Gray_squierrels_count)
print (Cinnamon_squierrels_count)
print (black_squirrels_count )

dict_data = {"Primary Fur Color": ["Gray", "Cinnamon", "Black"],
              "cantidades de ardillas": [Gray_squierrels_count, Cinnamon_squierrels_count, black_squirrels_count]
}

DF = pandas.DataFrame(dict_data)
DF.to_csv("Squirrels_count.csv")


from bs4 import BeautifulSoup
import requests
from pprint import pprint
import os
import csv

def get_article_detail(url):#為了取得時間，必須先進入網頁在爬取當下標題的時間
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    time_element = soup.find_all("span", class_="article-meta-value")  # 找到包含完整時間的元素
    if time_element:
        return time_element[3].text.strip()
    else:
        return ""
def get_articles_data(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    articles = soup.find_all("div", class_="r-ent")#出來會是一團數據含有r-ent
    article_data = {}

    #用字典處理會比較容易，列表不好控制值
    #跑迴圈一一取出，如需要更深入的內容就繼續往下.find
    for article in articles:
        title_element = article.find("div", class_="title")
        if not title_element.a:  # 如果沒有超連結，跳過該篇文章
            continue

        title = title_element.text.strip()
        rate = article.find("div", class_="nrec").text.strip()
        article_url = "https://www.ptt.cc" + article.find("div", class_="title").a["href"]
        #以上取的方式也可以使用持續使用.find再利用屬性[""]，或是整理成一行
        #文章的時間卡在文章網頁裡頭，先透過func取的時間
        date = get_article_detail(article_url)#date的數據內容透過上面func幫我處理
        article_dict = {"title": title, "rate": rate, "date": date}
        article_data[title] = article_dict

    return article_data

base_url = "https://www.ptt.cc/bbs/movie/index.html"

# 先抓第一頁的東西及可以推演的內容
response = requests.get(base_url)
soup = BeautifulSoup(response.text, "html.parser")
prev_page_link = soup.find("a", string="‹ 上頁")["href"]#使用 ["href"] 就是從 <a> 元素中提取 href 屬性的值
page_index = int(prev_page_link[prev_page_link.index("index") + len("index"):prev_page_link.index(".html")])
#先抓index字的索引值然後針對字串[:]索引某數到某數，會求得一組index數字例如9640
article_data = get_articles_data(base_url)

# 拿可以推演的內容反推索引要更改的值依序扣一
for i in range(3):
    page_index -= 1
    prev_page_url = f"https://www.ptt.cc/bbs/movie/index{page_index}.html"
    article_data.update(get_articles_data(prev_page_url))#list時或是用+=類似extend
pprint(article_data)
print(article_data)
print(len(article_data))

output_file = "Week3/movies.csv"
fields=["title","rate","date"]
if not os.path.exists(output_file):
    with open(output_file, "w", newline="") as file:
        writer = csv.DictWriter(file, fieldnames=fields)
        writer.writeheader()
        for item in article_data.values():
            writer.writerow(item)
    print("movies.csv finished")
else:
    print("movies.csv exists")

output_file2= "Week3/movie.txt"
if not os.path.exists(output_file2):
    with open(output_file2, "w") as file:
        for value in article_data.values():
            line = f"{value['title']},{value['rate']},{value['date']}\n"
            file.write(line)
    print("movie.txt finished")
else:
    print("movie.txt exists")


from django.urls import path, re_path
from django.views.generic import TemplateView
from . import views

urlpatterns = [
    # Index page displays all posts + postform depending on whether user is authenticated. Renders Virtual DOM with React
    path("", TemplateView.as_view(
        template_name="network/index.html"), name="index"),

    # Login (display credentials form), logout and register
    path("login", views.login_view, name="login"),
    path("logout", views.logout_view, name="logout"),
    path("register", views.register, name="register"),

    # Almost all paths below render DOM with React (listingview does not)
    path("following", views.following, name="following"),
    path("<int:post_id>", views.viewpost, name="viewpost"),
    # regex to avoid matching api path
    re_path(r"^(?P<username>\w+(?<!api))$", views.profile, name="profile"),
    # Following and followers list page -- using regex to allow only /following or /followers and avoid media path
    re_path(r"^(?P<username>\w+(?<!api))\/(?P<list>follow(ers|ing))$",
            views.listingview, name="listing"),

    # API route
    path("api/update", views.update, name="update"),
    path("api/<str:filter>", views.send_data, name="send"),
    path("api", views.post, name="post"),

]


# coding=utf-8
"""
convert the ptb to linearized tree
"""
from __future__ import division
from __future__ import print_function

import argparse
import os
import sys

sys.path.append(".")

from preprocess.tree_linearization import tree_convert
from preprocess.tree_linearization import tree_to_s2b

def write_docs(fname, docs):
    with open(fname, 'w') as f:
        for doc in docs:
            f.write(str(doc))
            f.write('\n')

def remove_same(docs):
    check = {}
    res = []
    for doc in docs:
        if doc not in check:
            check[doc] = 1
            res.append(doc)
        else:
            pass
    print("same data filter:{}".format(len(docs) - len(res)))
    return res

def ptb_to_s2b(tree_file, rm_same=False):
    with open(tree_file, 'r') as tf:
        s2b_list = []
        for tree_str in tf.readlines():
            if len(tree_str.strip()) > 0:
                if not tree_str.strip().startswith("(TOP"):
                    tree_str = "(TOP " + tree_str.strip() + ")"
                try:
                    src, tgt = tree_to_s2b(tree_str.strip())
                    s2b_list.append("\t".join([src, tgt]))
                except:
                    print(tree_str.strip())

    if rm_same:
        s2b_list = remove_same(s2b_list)

    return s2b_list

def ptb_to_ae(tree_file):
    with open(tree_file, 'r') as tf:
        ae_list = []
        for tree_str in tf.readlines():
            if len(tree_str.strip()) > 0:
                if not tree_str.strip().startswith("(TOP"):
                    tree_str = "(TOP " + tree_str.strip() + ")"
                try:
                    src, tgt = tree_to_s2b(tree_str.strip())
                    ae_list.append("\t".join([src, src]))
                except:
                    print(tree_str.strip())

    return ae_list

def make_s2b_dataset(train_file, dev_file=None, test_file=None, tgt_dir=None, rm_same=False):
    if not os.path.exists(tgt_dir):
        os.makedirs(tgt_dir)

    train = ptb_to_s2b(train_file, rm_same)
    write_docs(fname=os.path.join(tgt_dir, "train.s2b"), docs=train)
    if dev_file is not None:
        dev = ptb_to_s2b(dev_file)
        write_docs(fname=os.path.join(tgt_dir, "dev.s2b"), docs=dev)
    if test_file is not None:
        test = ptb_to_s2b(test_file)
        write_docs(fname=os.path.join(tgt_dir, "test.s2b"), docs=test)

def make_ae_dataset(train_file, dev_file=None, test_file=None, tgt_dir=None, rm_same=False):
    if not os.path.exists(tgt_dir):
        os.makedirs(tgt_dir)

    train = ptb_to_ae(train_file)
    write_docs(fname=os.path.join(tgt_dir, "train.ae"), docs=train)
    if dev_file is not None:
        dev = ptb_to_ae(dev_file)
        write_docs(fname=os.path.join(tgt_dir, "dev.ae"), docs=dev)
    if test_file is not None:
        test = ptb_to_ae(test_file)
        write_docs(fname=os.path.join(tgt_dir, "test.ae"), docs=test)

def main_convert(tree_file, out_file=None, mode="s2t"):
    if out_file is None:
        out_file = tree_file + "." + mode
        print("linear tree out is: ", out_file)

    tree_convert_method = tree_convert(mode=mode)

    with open(tree_file, 'r') as tf:
        tree_list = []
        for tree_str in tf.readlines():
            if len(tree_str.strip()) > 0:
                if not tree_str.strip().startswith("(TOP"):
                    tree_str = "(TOP " + tree_str.strip() + ")"
                try:
                    word, tree = tree_convert_method(tree_str.strip())
                    tree_list.append("\t".join([word, tree]))
                except:
                    print(tree_str.strip())
    write_docs(fname=out_file, docs=tree_list)

if __name__ == "__main__":
    opt_parser = argparse.ArgumentParser()
    opt_parser.add_argument('--tree_file', dest="tree_file", type=str, help='tree file with Penn TreeBank Format[must]')
    opt_parser.add_argument('--out_file', dest="out_file", type=str, help='output path[optional]')
    opt_parser.add_argument('--mode', dest="mode", type=str, default="s2b", help="linearized tree format:[s2t,s2b,s2s],"
                                                                                "default is s2b")
    opt = opt_parser.parse_args()
    main_convert(tree_file=opt.tree_file, out_file=opt.out_file, mode=opt.mode)


#Import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
from sklearn.datasets import load_diabetes
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline
from mlxtend.plotting import plot_decision_regions
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
import ipywidgets as widgets
from JaverianaML.classification import LDA
from IPython.display import display
import scipy.stats as stats
import sklearn as skl

##Normal distribution for iris dataset
def Binary_Gaussian_Plot(DB, shared_variance=False):
    '''
    Density plot of P(petal_width|y=c) assuming Gaussin pdfs for each class
    '''
    fig, ax = plt.subplots(1,2,figsize=(20,7))
    iris = sns.load_dataset('iris')
    iris = iris.loc[iris.species != 'setosa', :]
    plt.figure(figsize=(15,7))

    mu1 = iris.groupby('species')['petal_width'].mean()['virginica']
    std1 = iris.groupby('species')['petal_width'].std()['virginica']

    mu2 = iris.groupby('species')['petal_width'].mean()['versicolor']
    std2 = iris.groupby('species')['petal_width'].std()['versicolor']

    if shared_variance:
        std =np.sqrt( 0.5*std2**2 +  0.5*std1**2 )
        std1 = std
        std2 = std

    x1 = np.linspace(mu1 - 3*std1, mu1 + 3*std1, 100)
    ax[0].fill_between(x1, stats.norm.pdf(x1, mu1, std1)*0.5, alpha=0.5, label="virginica")

    x2 = np.linspace(mu2 - 3*std2, mu2 + 3*std2, 100)
    ax[0].fill_between(x2, stats.norm.pdf(x2, mu2, std2)*0.5, alpha=0.5, label="versicolor")
    sns.scatterplot(data=iris, x='petal_width', y='species', hue='species', hue_order =['virginica', 'versicolor'], ax=ax[1])

    ax[0].vlines(DB,0,1)
    ax[1].vlines(DB,0,1)
    ax[0].text(0.7,0.9, r'$P(X|C_1)P(C_1)$', fontsize=20)
    ax[0].text(1.9,0.9, r'$P(X|C_2)P(C_2)$', fontsize=20)
    ax[0].set_xlabel("petal_width")
    ax[0].set_ylabel("Density")
    ax[0].legend()

def Multiclass_Gaussian_Plot(DB1, DB2, shared_variance=False):
    '''
    Density plot of P(petal_width|y=c) assuming Gaussin pdfs for each class
    '''
    fig, ax = plt.subplots(1,2,figsize=(25,7))
    iris = sns.load_dataset('iris')
    plt.figure(figsize=(25,7))

    mu1 = iris.groupby('species')['petal_width'].mean()['virginica']
    std1 = iris.groupby('species')['petal_width'].std()['virginica']

    mu2 = iris.groupby('species')['petal_width'].mean()['versicolor']
    std2 = iris.groupby('species')['petal_width'].std()['versicolor']

    mu3 = iris.groupby('species')['petal_width'].mean()['setosa']
    std3 = iris.groupby('species')['petal_width'].std()['setosa']

    if shared_variance:
        std =np.sqrt( (50/150)*std2**2 +  (50/150)*std1**2 + (50/150)*std3**2 )
        std1 = std
        std2 = std
        std3 = std

    x1 = np.linspace(mu1 - 3*std1, mu1 + 3*std1, 100)
    ax[0].fill_between(x1, stats.norm.pdf(x1, mu1, std1)*0.5, alpha=0.5, label="virginica")

    x2 = np.linspace(mu2 - 3*std2, mu2 + 3*std2, 100)
    ax[0].fill_between(x2, stats.norm.pdf(x2, mu2, std2)*0.5, alpha=0.5, label="versicolor")

    x3 = np.linspace(mu3 - 3*std3, mu3 + 3*std3, 100)
    ax[0].fill_between(x3, stats.norm.pdf(x3, mu3, std3)*0.5, alpha=0.5, label="versicolor")

    sns.scatterplot(data=iris, x='petal_width', y='species', hue='species', hue_order =['virginica', 'versicolor', 'setosa'], ax=ax[1])

    ax[0].vlines(DB1,0,1.8)
    ax[1].vlines(DB1,0,2)

    ax[0].vlines(DB2,0,1.8)
    ax[1].vlines(DB2,0,2)

    ax[0].text(0.9,1.0, r'$P(X|C_1)P(C_1)$', fontsize=15)
    ax[0].text(1.9,1.0, r'$P(X|C_2)P(C_2)$', fontsize=15)
    ax[0].text(0.0, 1.0, r'$P(X|C_3)P(C_3)$', fontsize=15)
    ax[0].set_xlabel("petal_width")
    ax[0].set_ylabel("Density")
    ax[0].legend()

def Gaussian_3d_plot(w0, w):

    from plotly.subplots import make_subplots
    import scipy.stats as stats
    import plotly.graph_objects as go
    import plotly.express as px
    iris = sns.load_dataset('iris')
    iris = iris.loc[iris.species != 'setosa', :]

    means = iris.groupby("species", as_index=False).mean()[['species','petal_width', 'petal_length']]
    mu1 = means.loc[means.species=='versicolor', ['petal_width', 'petal_length']].to_numpy().ravel()
    mu2 = means.loc[means.species=='virginica', ['petal_width', 'petal_length']].to_numpy().ravel()

    cov1 = iris.loc[iris.species=='versicolor' ,['petal_width', 'petal_length']].cov().to_numpy()
    cov2 = iris.loc[iris.species=='virginica' ,['petal_width', 'petal_length']].cov().to_numpy()

    cov = 0.5*cov1 + 0.5*cov2

    xs = np.linspace(0.5, 3.0, 100)
    zs = np.linspace(0, 1.4, 100)

    X, Z = np.meshgrid(xs, zs)
    Y = w*X + w0

    x1, y1 = np.mgrid[0.5:1.9:200j, 2.5:5.3:200j]
    x2, y2 = np.mgrid[1.3:2.6:200j, 4.5:7:200j]

    xy1 = np.column_stack([x1.flat, y1.flat])
    xy2 = np.column_stack([x2.flat, y2.flat])

    cmap = plt.get_cmap("tab10")
    colorscale = [[0, 'rgb' + str(cmap(1)[0:3])], [1, 'rgb' + str(cmap(2)[0:3])]]

    z1 = stats.multivariate_normal.pdf(xy1, mean=mu1, cov=cov)
    z1 = z1.reshape(x1.shape)

    z2 = stats.multivariate_normal.pdf(xy2, mean=mu2, cov=cov)
    z2 = z2.reshape(x2.shape)

    fig = make_subplots(rows=1, cols=2, specs=[[{'is_3d': True}, {'is_3d': False}]])
    trace_a = go.Surface(x=x1, y=y1, z=z1, cmin=0, cmax=1, surfacecolor=np.zeros(shape=z1.shape), colorscale= colorscale, showscale=False, name="versicolor", opacity=0.8)
    trace_b = go.Surface(x=x2, y=y2,z=z2, cmin=0, cmax=1, surfacecolor=np.ones(shape=z2.shape) , colorscale= colorscale, showscale=False, name="virginica", opacity=0.8)
    trace_c = go.Surface(x=X, y=Y,z=Z, cmin=0, cmax=1, surfacecolor=np.ones(shape=z2.shape)*0.5 , colorscale= colorscale, showscale=False, name="virginica")
    fig.add_traces([trace_a, trace_b, trace_c], rows=1, cols=1)
    fig.update_scenes(xaxis_title_text="petal_width", yaxis_title_text="petal_length")

    scatter_1 = go.Scatter(x=iris[iris.species=='versicolor'].petal_width, y=iris[iris.species=='versicolor'].petal_length,  mode='markers', name='versicolor', marker_color='red')
    scatter_2 = go.Scatter(x=iris[iris.species=='virginica'].petal_width, y=iris[iris.species=='virginica'].petal_length,  mode='markers', name='virginica', marker_color='green')

    x=np.linspace(1,2.5,100)
    y=w*x+w0
    line = go.Scatter(x=x, y=y,  mode='lines', name='virginica', marker_color='black')

    fig.add_traces([scatter_1, scatter_2, line], rows=1, cols=2)
    fig.update_yaxes(title_text="petal_length", row=1, col=2)
    fig.update_xaxes(title_text="petal_width", row=1, col=2)
    fig.update_layout(width=900*2, height=600)
    fig.show()

def LDA_on_different_datasets(dataset='blobs'):

    """
    This function fits LDA model using different synthetic datasets

    Arguments:
        dataset: string
            One of 'blobs', 'circle' or 'moons' which indicates the
            sythetic dataset to be created.
    """

    #Create sinthetic dataset
    if dataset == 'blobs':
        X, y = skl.datasets.make_blobs(n_samples=1000, cluster_std=3, centers=2, random_state=42)
    elif dataset == 'circle':
        X, y = skl.datasets.make_circles(n_samples=1000, noise=0.1, factor=0.5, random_state=42)
    elif dataset == 'moons':
        X, y = skl.datasets.make_moons(n_samples=1000, noise=0.1, random_state=42)

    #Fit linear discriminanat analysis model
    plt.figure(figsize=(10,8))
    plot_decision_regions(X, y, clf=LinearDiscriminantAnalysis().fit(X,y))
    plt.xlabel(r'$x_1$', fontsize=20)
    plt.ylabel(r'$x_1$', fontsize=20)

def QDA_on_different_datasets(dataset='blobs'):

    """
    This function fits QDA model using different synthetic datasets

    Arguments:
        dataset: string
            One of 'blobs', 'circle' or 'moons' which indicates the
            sythetic dataset to be created.
    """

    #Create sinthetic dataset
    if dataset == 'blobs':
        X, y = skl.datasets.make_blobs(n_samples=1000, cluster_std=3, centers=2, random_state=42)
    elif dataset == 'circle':
        X, y = skl.datasets.make_circles(n_samples=1000, noise=0.1, factor=0.5, random_state=42)
    elif dataset == 'moons':
        X, y = skl.datasets.make_moons(n_samples=1000, noise=0.1, random_state=42)

    #Fit linear discriminanat analysis model
    plt.figure(figsize=(10,8))
    plot_decision_regions(X, y, clf=QuadraticDiscriminantAnalysis().fit(X,y))
    plt.xlabel(r'$x_1$', fontsize=20)


from source.utilities.graphStringMuxer import GraphStringMuxer
from source.utilities.trajectoryCounts import TrajectoryCounts
from typing import List
from source.dataStructures import NoOpCompare, Vector

class BranchingGraphError(Exception):
    pass

class Graph(NoOpCompare):

    def __init__(self, graphVectors: List[Vector], vectorList: List[Vector], skipSpinUp=False):
        if skipSpinUp:
            return
        self.graphData = []
        self.weight = 0
        self.offendingIndex = 0
        self.lastChange = None
        self.vectorList = vectorList
        # self.trajectoryRegister = TrajectoryCounts.fillRegister(graphVectors)
        charList = ''
        for v in graphVectors:
            vector = vectorList[v]
            self.graphData.append(self.toKey(vector))
            self.weight += vector[2]
            if charList.find(vector[0]) == -1:
                charList += vector[0]
            else:
                raise BranchingGraphError

    def copy(self):
        g = Graph([], [], skipSpinUp=True)
        g.graphData = list(self.graphData)
        g.weight = self.weight
        g.lastChange = self.lastChange
        g.vectorList = self.vectorList
        # g.trajectoryRegister = self.trajectoryRegister
        g.offendingIndex = self.offendingIndex
        return g

    def goAcross(self, vector: Vector, weight: int) -> int:
        i = GraphStringMuxer.translate(self.lastChange[0])
        oldWeight = self.vectorList[self.graphData[i]][2]
        self.swapOutLastChange()
        self.replace(vector)
        return  weight - oldWeight + vector[2]

    def swapOutLastChange(self) -> None:
        self.replace(self.vectorList[self.lastChange])

    def goDeeper(self, vector, weight) -> int:
        self.replace(vector)
        return weight + vector[2]

    def replace(self, vector) -> None:
        i = GraphStringMuxer.translate(vector[0])
        v = self.graphData[i]
        self.lastChange = v
        self.weight += vector[2] - self.vectorList[v][2]
        # self.trajectoryRegister = TrajectoryCounts.createDeltaRegister(self.trajectoryRegister, v, vector)
        self.__assignVector(vector)

    def isValid(self) -> bool:
        # [consistent, offendingIndex] = TrajectoryCounts.demonstratesConsistency(self.trajectoryRegister)
        # self.offendingIndex = offendingIndex
        # if not consistent:
        #     return False
        visited = {}
        start = self.graphData[0][0]
        beginning = start
        while True:
            if visited.get(start, False):
                break
            visited[start] = True
            i = GraphStringMuxer.translate(start)
            start = self.graphData[i][3]
        return len(visited) == len(self.graphData) and start == beginning

    def getWeight(self):
        return self.weight

    def toVectorListString(self):
        s = "{"
        for v in self.graphData:
            s += str(self.vectorList[v])
        return s + "}"

    def __assignVector(self, vector):
        i = GraphStringMuxer.translate(vector[0])
        self.graphData[i] = self.toKey(vector)

    def __str__(self):
        visited = {}
        start = self.graphData[0][0]
        literal = "(" + start
        while True:
            if visited.get(start, False):
                break
            literal += GraphStringMuxer.arrowString
            visited[start] = True
            i = GraphStringMuxer.translate(start)
            start = self.destintationAt(i)
            literal += start
        return literal + "): " + str(self.weight)

    def __unicode__(self):
        return u"" + self.__str__()

    def toKey(self, v: Vector) -> str:
        return v[0] + GraphStringMuxer.arrowString + v[1]

    def destintationAt(self, i: int) -> str:
        return GraphStringMuxer.destintationCharacter(self.graphData[i])


import sys

sys.path.append("../../../Asset Wealth")

import os
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from osgeo import osr
from osgeo import gdal
from src.data_utils import calc_mean
from src.data_utils import calc_std

from src.config import clipping_values

def slice_to_input_size(array: np.ndarray, input_height: int, input_width: int):
    '''Slice image array to desired input shape.

    Args:
        array (np.ndarray): Numpy array containing image data
        input_height (int): Uniform image height to slice to
        input_width (int):  Uniform image width to slice to

    Returns:
        array (np.array): Numpy array containing image data in shape of input height/width and Bandwidth
    '''
    if len(array.shape) == 3:
        outer_pixels_y = int((array.shape[1] - input_height) / 2)
        outer_pixels_x = int((array.shape[2] - input_width) / 2)
        array = array[:, outer_pixels_y:-outer_pixels_y, outer_pixels_x:-outer_pixels_x]
        if array.shape[1] == input_height + 1:
            array = array[:, :-1, :]
        if array.shape[2] == input_width + 1:
            array = array[:, :, :-1]
    elif len(array.shape) == 2:
        outer_pixels_y = int((array.shape[0] - input_height) / 2)
        outer_pixels_x = int((array.shape[1] - input_width) / 2)
        array = array[outer_pixels_y:-outer_pixels_y, outer_pixels_x:-outer_pixels_x]
        if array.shape[0] == input_height + 1:
            array = array[:-1, :]
        if array.shape[1] == input_width + 1:
            array = array[:, :-1]
    return array

def standardize_resize(img: str, img_path: str,  input_height: str, input_width: str, clipping_values: list,
                     means=False, stds=False, add_img_path=False, standardize=False):
    '''Standardize and resize GeoTIFFs.
    Standardization is performed per band using Standard Scaler.
    Resizing is performed by slicing to the center of the image in shape of provided input Size.
    For VIIRS images, the band is tripled to fit RGB input shape of common CNNs.
    Standardized and resized images are stored in a new GeoTIFF.

    Args:
        img (str):                  Filename of image to normalize and resize
        img_path (str):             Path to image data
        input_height (int):         Desired input height
        input_width (int):          Desired input width
        clipping_values (list):     Interval of min and max values for clipping
        means (bool/np.ndarray):         Optional: Result of calc_mean: Mean of pixel values for each channel
        stds (bool/np.ndarray):          Optional: Result of calc_mean: Standard deviation of pixel values for each channel
        add_img_path (bool/str):    Optional: Path to image data to add (eg. for combining Sentinel-2 and VIIRS)
        standardize (bool):                Optional: Whether or not to standardize image data (e.g. standardization is not needed when already normalized Sentinel-2 and VIIRS data is merged)

    '''
    ds = gdal.Open(os.path.join(img_path, img))
    geotransform = ds.GetGeoTransform()
    array = ds.ReadAsArray()

    array[np.isnan(array)] = 0
    assert not np.any(np.isnan(array)), "Float"

    if add_img_path:
        ds = gdal.Open(os.path.join(add_img_path, img))
        geotransform = ds.GetGeoTransform()
        array_to_add = ds.ReadAsArray()

        array_to_add[np.isnan(array_to_add)] = 0
        assert not np.any(np.isnan(array_to_add)), "Float"
    # Clipping
    array = np.clip(array, a_min=clipping_values[0], a_max=clipping_values[1])

    assert not np.any(np.isnan(array)), "After clipping"
    # Normalize the array
    if standardize:
        if 'Sentinel' in img_path:
            array = ((array.transpose(1, 2, 0) - means) / stds).transpose(2, 0, 1)
        elif 'VIIRS' in img_path:
            array = ((array.transpose(0, 1) - means) / stds).transpose(1, 0)
        assert not np.any(np.isnan(array)), "Normalize"

    # Resize the array to ensure that all image arrays have the same size
    if len(array.shape) == 3:
        if array.shape != (array.shape[0], input_height, input_width):
            if array.shape[1] > input_height:
                array = slice_to_input_size(array, input_height, input_width)

        assert array.shape == (array.shape[0], input_height, input_width)
    elif len(array.shape) == 2:
        if array.shape != (input_height, input_width):
            if array.shape[0] > input_height:
                array = slice_to_input_size(array, input_height, input_width)

        assert array.shape == (input_height, input_width)
    # return normalized and resized image array

    # Create new geotiff
    if add_img_path:
        dst_ds = gdal.GetDriverByName('GTiff').Create(
            os.path.join('/mnt/datadisk/data/VIIRS_Sentinel2/preprocessed/asset/',
                         img_path.split('/')[-2],
                         img_path.split('/')[-1],
                         img), input_height, input_width,len(array) + 1, gdal.GDT_Byte)
    if 'Sentinel' in img_path:
        dst_ds = gdal.GetDriverByName('GTiff').Create(os.path.join(img_path, img), input_height, input_width,
                                                      len(array), gdal.GDT_Byte)
    elif 'VIIRS' in img_path:
        dst_ds = gdal.GetDriverByName('GTiff').Create(os.path.join(img_path, img), input_height, input_width, 3,
                                                      gdal.GDT_Byte)
    dst_ds.SetGeoTransform(geotransform)  # specify coords
    srs = osr.SpatialReference()  # establish encoding
    srs.ImportFromEPSG(4326)  # WGS84 lat/long
    dst_ds.SetProjection(srs.ExportToWkt())  # export coords to file
    if 'Sentinel' in img_path:
        for i in range(len(array)):
            dst_ds.GetRasterBand(i + 1).WriteArray(array[i])  # write band to the raster
    elif 'VIIRS' in img_path:
        array = np.concatenate((np.array([array]),) * 3, axis=0)

        for i in range(len(array)):
            dst_ds.GetRasterBand(i + 1).WriteArray(array[i])  # write band to the raster
    if add_img_path:
        for i in range(len(array)):
            dst_ds.GetRasterBand(i + 1).WriteArray(array[i])  # write band to the raster
        dst_ds.GetRasterBand(i + 2).WriteArray(array_to_add[0])  # write (viirs) band to the raster
    dst_ds.FlushCache()  # write to disk
    dst_ds = None

def main(img_path: str, ur: str, year: str, input_height: str, input_width: str,
         clipping_values: list, channels: list, add_img_path=False, standardize=False):
    '''Preprocessing of GeoTIFFs.
    Preprocessing includes slicing to desired input size and standardizing the image data.

    Args:
        img_path (str):             Path to image data
        ur (str):                   'u' for urban, 'r' for rural
        year (str):                 timespan (2012_2014 / 2016_2020) or all data
        input_height (int):         Desired input height
        input_width (int):          Desired input width
        clipping_values (list):     Interval of min and max values for clipping
        channels (list):            List of channels to use. [] to use all channels.
        add_img_path (bool/str):    Optional: Path to image data to add (eg. for combining Sentinel2 and VIIRS)
        standardize:                Optional: Whether or not to standardize image data (e.g. standardization is not needed when already
                                    normalized Sentinel-2 and VIIRS data is merged)
    '''

    # Get Splits to Normalize Train/Val and Test separately
    X_train_val, X_test = train_test_split([i for i in os.listdir(img_path) if i.endswith('.tif')], test_size=0.20,
                                           random_state=42)
    ds = gdal.Open(os.path.join(img_path, X_train_val[0]))
    geotransform = ds.GetGeoTransform()

    array = ds.ReadAsArray()
    array[np.isnan(array)] = 0
    assert not np.any(np.isnan(array)), "Float"

    # Clipping
    array = np.clip(array, a_min=clipping_values[0], a_max=clipping_values[1])

    assert not np.any(np.isnan(array)), "After clipping"
    # Normalize the array

    if standardize:
        # Calculate Means and Standard Deviations per Band
        if 'Sentinel' in img_path:
            if os.path.exists('./s2_means_stds.csv'):
                means_stds_df = pd.read_csv('./s2_means_stds.csv', )

                if ur + '_train_val_means' in means_stds_df.columns:
                    print('Mean and Standard Deviation Values found:')
                    train_val_means = means_stds_df[ur + '_train_val_means'].to_numpy()
                    train_val_stds = means_stds_df[ur + '_train_val_stds'].to_numpy()

                    test_means = means_stds_df[ur + '_test_means'].to_numpy()
                    test_stds = means_stds_df[ur + '_test_stds'].to_numpy()
                    print(train_val_means.shape, train_val_stds.shape, test_means.shape, test_stds.shape)
                else:
                    train_val_means = calc_mean(img_path, X_train_val, input_height, input_width, clipping_values, channels)
                    train_val_stds = calc_std(train_val_means, img_path, X_train_val, input_height, input_width,
                                              clipping_values, channels)

                    test_means = calc_mean(img_path, X_test, input_height, input_width, clipping_values, channels)
                    test_stds = calc_std(train_val_means, img_path, X_test, input_height, input_width, clipping_values,
                                         channels)

                    means_stds_df[ur + '_train_val_means'] = train_val_means
                    means_stds_df[ur + '_train_val_stds'] = train_val_stds
                    means_stds_df[ur + '_test_means'] = test_means
                    means_stds_df[ur + '_test_stds'] = test_stds
                    means_stds_df.to_csv('./s2_means_stds.csv')
            else:
                train_val_means = calc_mean(img_path, X_train_val, input_height, input_width, clipping_values, channels)
                train_val_stds = calc_std(train_val_means, img_path, X_train_val, input_height, input_width,
                                          clipping_values, channels)

                test_means = calc_mean(img_path, X_test, input_height, input_width, clipping_values, channels)
                test_stds = calc_std(train_val_means, img_path, X_test, input_height, input_width, clipping_values,
                                     channels)
                means_stds_df = pd.DataFrame(list(zip(train_val_means, train_val_stds, test_means, test_stds)),
                                             columns=[ur + '_train_val_means', ur + '_train_val_stds', ur + '_test_means',
                                                      ur + '_test_stds'])
                means_stds_df.to_csv('./s2_means_stds.csv')
        elif 'VIIRS' in img_path:
            if os.path.exists('./viirs_means_stds.csv'):
                means_stds_df = pd.read_csv('./viirs_means_stds.csv', )

                if ur + '_' + '_train_val_means' in means_stds_df.columns:
                    print('Mean and Standard Deviation Values found:')
                    train_val_means = means_stds_df[ur + '_' + year + '_train_val_means'].to_numpy()
                    train_val_stds = means_stds_df[ur + '_' + year + '_train_val_stds'].to_numpy()

                    test_means = means_stds_df[ur + '_' + year + '_test_means'].to_numpy()
                    test_stds = means_stds_df[ur + '_' + year + '_test_stds'].to_numpy()
                    print(train_val_means.shape, train_val_stds.shape, test_means.shape, test_stds.shape)
                else:
                    train_val_means = calc_mean(img_path, X_train_val, input_height, input_width, clipping_values, channels)
                    train_val_stds = calc_std(train_val_means, img_path, X_train_val, input_height, input_width,
                                              clipping_values, channels)

                    test_means = calc_mean(img_path, X_test, input_height, input_width, clipping_values, channels)
                    test_stds = calc_std(test_means, img_path, X_test, input_height, input_width, clipping_values, channels)

                    means_stds_df[ur + '_' + year + '_train_val_means'] = train_val_means
                    means_stds_df[ur + '_' + year + '_train_val_stds'] = train_val_stds
                    means_stds_df[ur + '_' + year + '_test_means'] = test_means
                    means_stds_df[ur + '_' + year + '_test_stds'] = test_stds
                    means_stds_df.to_csv('./viirs_means_stds.csv')
            else:
                train_val_means = calc_mean(img_path, X_train_val, input_height, input_width, clipping_values, channels)
                train_val_stds = calc_std(train_val_means, img_path, X_train_val, input_height, input_width,
                                          clipping_values, channels)

                test_means = calc_mean(img_path, X_test, input_height, input_width, clipping_values, channels)
                test_stds = calc_std(test_means, img_path, X_test, input_height, input_width, clipping_values, channels)
                means_stds_df = pd.DataFrame(list(zip(train_val_means, train_val_stds, test_means, test_stds)),
                                             columns=[ur + '_' + year + '_train_val_means',
                                                      ur + '_' + year + '_train_val_stds',
                                                      ur + '_' + year + '_test_means',
                                                      ur + '_' + year + '_test_stds'])
                means_stds_df.to_csv('./viirs_means_stds.csv')

    # Normalize and Resize Images and save the preprocessed Images for Training
    for image in tqdm(X_train_val):
        standardize_resize(img=image, img_path=img_path, input_height=input_height, input_width=input_width,
                           clipping_values=clipping_values, means=train_val_means, stds=train_val_stds,
                           standardize=True)

    for image in tqdm(X_test):
        standardize_resize(img=image, img_path=img_path, input_height=input_height, input_width=input_width,
                           clipping_values=clipping_values, means=test_means, stds=test_stds,
                           standardize=True)

if __name__ == '__main__':

    main(
        img_path='/mnt/datadisk/data/VIIRS/preprocessed/asset/urban/mozambique_2016_2021',
        ur='u',
        year='all',
        input_height=200,
        input_width=200,
        clipping_values=clipping_values,
        channels=[],
        standardize = True
    )


import logging
from pathlib import Path

from pyquery import PyQuery as pq

from config import config
from lian_jia import Community
from util.orm import Session

DATA_DIR = Path(__file__).parent.joinpath('../data/').resolve()

def parse_community_detail(community_id):
    save_file = DATA_DIR.joinpath(f'{community_id}.html')

    if save_file.exists():
        d = pq(save_file.read_text(encoding='utf-8'))

        keys = [e.text.rstrip('：') for e in d('span.hdic_key')]
        values = [e.text for e in d('span.hdic_value')]
        detail = dict(zip(keys, values))

        if not detail:
            logging.warning(f'# 页面已找到, 但其中未发现小区详情信息, community_id={community_id}')

        return detail

    else:
        logging.error(f'# 详情页面不存在, community_id={community_id}')

def parse_all_communities(city_id):
    db_session = Session()

    communities = db_session.query(Community).filter(
        Community.city_id == city_id,
        Community.detail == None,
        Community.page_fetched_at != None
    ).all()

    total_count = len(communities)
    logging.info(f'city_id={city_id}, 待分析={total_count}')

    for i, a_community in enumerate(communities):
        detail = parse_community_detail(a_community.id)

        if detail:
            a_community.detail = detail

        if (i + 1) % 100 == 0 or (i == total_count - 1):
            logging.info(f'进度={i + 1}/{total_count}, 剩余={total_count - i - 1}')
            db_session.commit()

    logging.info('已全部分析完成.')
    db_session.close()

def main():
    parse_all_communities(config.city_id)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Thu May 25 15:50:34 2023

@author: gabri
"""
import numpy as np
import matplotlib.pyplot as plt
from hamiltonians import Hamiltonian_soliton_A1u_sparse, charge_conjugation_operator_sparse
from functions import get_components
import scipy

L_x = 200
L_y = 200
t = 1
Delta = 1
mu = -2  #-2
Phi = 0.1*np.pi  #height of the phase soliton around flux pi
t_J = 1   #t
L = 30      #L_y//2
k = 4   #number of eigenvalues
Delta_Z = 0
theta = np.pi/2
phi = 0

H = Hamiltonian_soliton_A1u_sparse(t=t, mu=mu, L_x=L_x, L_y=L_y, Delta=Delta, t_J=t_J, Phi=Phi, L=L)
eigenvalues_sparse, eigenvectors_sparse = scipy.sparse.linalg.eigsh(H, k=k, sigma=0)
idx = eigenvalues_sparse.argsort()
eigenvalues_sparse = eigenvalues_sparse[idx]
eigenvectors_sparse = eigenvectors_sparse[:, idx]

from scipy.linalg import orth
eigenvectors_negative = scipy.linalg.orth(eigenvectors_sparse[:,:2]) #for negative eigenvalues
#eigenvectors_negative = eigenvectors_sparse[:,:2]
C = charge_conjugation_operator_sparse(L_x, L_y)
A = C.dot(eigenvectors_negative[:,0].conj()) #positive eigenvector
is_charge_conjugation_A = np.allclose(H @ A, -eigenvalues_sparse[0]*A)
B = C.dot(eigenvectors_negative[:,1].conj()) #positive eigenvector
is_charge_conjugation_B = np.allclose(H @ B, -eigenvalues_sparse[1]*B)
eigenvectors = np.append(eigenvectors_negative, A.reshape((len(A), 1)), axis=1)
eigenvectors = np.append(eigenvectors, B.reshape((len(B), 1)), axis=1)

eigenvector_up_negative = 1/np.sqrt(2)*(eigenvectors[:,0]+eigenvectors[:,1])
eigenvector_down_negative = 1/np.sqrt(2)*(eigenvectors[:,0]-eigenvectors[:,1])
eigenvector_up_positive = 1/np.sqrt(2)*(eigenvectors[:,2]+eigenvectors[:,3])
eigenvector_down_positive = 1/np.sqrt(2)*(eigenvectors[:,2]-eigenvectors[:,3])

eigenvectors_polarized = np.array([eigenvector_up_negative, eigenvector_down_negative, eigenvector_up_positive, eigenvector_down_positive]).T

index = np.arange(k)   #which zero mode (less than k)
probability_density = []
zero_state = []
localized_state_upper_left = [] # it is the site (L_x/2-1, (L_y+L)/2)
localized_state_upper_right = [] # it is the site (L_x/2, (L_y+L)/2)
localized_state_bottom_left = [] # it is the site (L_x/2-1, (L_y-L)/2)
localized_state_bottom_right = [] # it is the site (L_x/2, (L_y-L)/2)
localized_state_left = []
localized_state_right = []

for i in index:
    destruction_up, destruction_down, creation_down, creation_up = get_components(eigenvectors_sparse[:,i], L_x, L_y)
    probability_density.append((np.abs(destruction_up)**2 + np.abs(destruction_down)**2 + np.abs(creation_down)**2 + np.abs(creation_up)**2)/(np.linalg.norm(np.abs(destruction_up)**2 + np.abs(destruction_down)**2 + np.abs(creation_down)**2 + np.abs(creation_up)**2)))
    zero_state.append(np.stack((destruction_up, destruction_down, creation_down, creation_up), axis=2)) #positive energy eigenvector splitted in components
    localized_state_upper_left.append(zero_state[i][(L_y+L)//2, L_x//2-1,:])
    localized_state_upper_right.append(zero_state[i][(L_y+L)//2, L_x//2,:])
    localized_state_bottom_left.append(zero_state[i][(L_y-L)//2, L_x//2-1,:])
    localized_state_bottom_right.append(zero_state[i][(L_y-L)//2, L_x//2,:])
    localized_state_left.append(zero_state[i][:, L_x//2-1,:])
    localized_state_right.append(zero_state[i][:, L_x//2,:])


# -*- coding: utf-8 -*-

from unittest.mock import patch

from tests.fixtures.benchmark import ComparativeBenchmarkMother
from bench.app.benchmark.domain.config import NotificationsConfig
from bench.app.benchmark.domain.events import ComparativeBenchmarkFinished
from bench.app.benchmark.infrastructure.repositories import ComparativeBenchmarkInMemoryRepository
from bench.app.benchmark.use_cases.listeners.email_alert_listener import ComparativeBenchmarkFinishedEmailAlertListener
from bench.app.notifications.use_cases.commands import SendEmailCommand

notifications_config = NotificationsConfig(notification_email='some email')

@patch('bench.app.benchmark.domain.specification.SubjectLoadedSlowerThanAtLeastOneOfCompetitorsSpecification')
def test_it_should_send_email_if_subject_url_loaded_slower_than_at_least_one_of_competitors(specification_mock):
    # given
    specification_mock.is_satisfied_by.return_value = True
    benchmark = ComparativeBenchmarkMother.create_any()

    # and
    repository = ComparativeBenchmarkInMemoryRepository()
    repository.add(benchmark)

    # when
    listener = ComparativeBenchmarkFinishedEmailAlertListener(notifications_config, specification_mock, repository)
    result = listener.execute(ComparativeBenchmarkFinished(benchmark.benchmark_id))

    # then
    expected_command = SendEmailCommand('Benchmark alert', [notifications_config.notification_email], 'Your site is slow')
    assert expected_command == result

@patch('bench.app.benchmark.domain.specification.SubjectLoadedSlowerThanAtLeastOneOfCompetitorsSpecification')
def test_it_should_not_send_email_if_subject_url_loaded_faster_than_competitors(specification_mock):
    # given
    specification_mock.is_satisfied_by.return_value = False
    benchmark = ComparativeBenchmarkMother.create_any()

    # and
    repository = ComparativeBenchmarkInMemoryRepository()
    repository.add(benchmark)

    # when
    listener = ComparativeBenchmarkFinishedEmailAlertListener(notifications_config, specification_mock, repository)
    result = listener.execute(ComparativeBenchmarkFinished(benchmark.benchmark_id))

    # then
    assert result is None


'''
Downloads the desired number of images from the ImageNet database using a .txt file
with the urls of the images listed. Stores a testing set and a training set of these
images as numpy arrays in an .h5 file. Can be run from command line with click.
'''

import click
import h5py
from skimage import io
import os
import numpy as np
import urllib

'''
num_images - number of images to download
read_path - path to .txt file provided by ImageNet
returns - two lists with each entry being a url of an image, one for training
and the other for testing
'''

def choose_images(num_images, read_path, train_percent=0.8, val_percent=0.8):
    urls = []
    with open(read_path) as myfile:
        count = 0
        while count < num_images:
            x = next(myfile)
            if 'flickr' in x:
                count += 1
                url = x[x.find('h'):-1]
                urls.append(url)

    np.random.shuffle(urls)
    sep_index1 = int(num_images * train_percent)
    sep_index2 = int(sep_index1 * val_percent)

    return urls[:sep_index2], urls[sep_index2: sep_index1], urls[sep_index1:]

'''
Writes the images as a series of numpy arrays to an .h5 file

urls - list of urls
write_path - directory to write the .h5 file to
prefix - desired filename
'''

def image_to_h5(urls, write_path, prefix):
    if not os.path.exists(write_path):
        os.makedirs(write_path)

    filename = os.path.join(write_path, prefix + '.h5')

    with h5py.File(filename, "w") as f:
        for i in range(len(urls)):
            # some urls don't have images anymore and need to be skipped
            try:
                image = io.imread(urls[i])
                if i % 100 == 0:
                    print('Reached image {}'.format(i))
                name = 'img_{}'.format(i)
                f.create_dataset(name, data=image)
            except (urllib.error.URLError, ValueError):
                print('Could not download image {}'.format(i))

@click.command()
@click.option('--num_images', default=10000, help='Number of images to download')
@click.argument('read_path', type=click.Path(exists=True, readable=True))
@click.argument('write_path', type=click.Path(exists=False))
@click.argument('prefix')
def main(num_images, read_path, write_path, prefix):
    urls_train, urls_val, urls_test = choose_images(num_images, read_path)
    image_to_h5(urls_train, write_path, prefix + '_training')
    image_to_h5(urls_val, write_path, prefix + '_validation')
    image_to_h5(urls_test, write_path, prefix + '_testing')

if __name__ == '__main__':
    main()
    print('done')


import csv
import readline
import pandas as pd
import pymongo

conn  = pymongo.MongoClient('192.168.99.100', 32766)
db    = conn.get_database("mini_project") # 있으면 가져오고, 없으면 만들어
table = db.get_collection("korean_chulguk") # 있으면 가져오고, 없으면 만들어

f   = open('./resources/korean_chulguk.csv','r', encoding="euc-kr")

rdr = csv.reader(f)
print(rdr)

column = next(rdr) #[컬럼명 읽기]

for line in rdr:
    dict1 = dict()
    for idx, val in enumerate(line):
        dict1[column[idx]] = val
        print(line)

    table.insert_one(dict1)
conn.close()


import asyncio
import time
import pandas as pd
import jdatetime as jd
from datetime import timedelta
from tqdm import tqdm

from requests_html import HTMLSession, AsyncHTMLSession

# number of workers to run in parallel
WORKERS = 64
# number of pages to scrap
THRESHOLD = 5
# default URL template
URL_TEMPLATE = 'https://fidibo.com/category/literature/short-story?page='
DATE_FORMAT = '%Y/%m/%d'
HEADERS = {'X-Requested-With': 'XMLHttpRequest'}

session = HTMLSession()
session.headers.update(HEADERS)  # send HEADERS with all requests
r = session.get('https://fidibo.com/category/literature/short-story')

# get number of pages available
TOTAL_PAGES = r.json()['total_pages']
SIZE_PER_PAGE = r.json()['size']

# books that are going to be scraped
TOTAL_BOOKS = SIZE_PER_PAGE * THRESHOLD

# choose smaller between total_pages and threshold for threshold value
THRESHOLD = min(THRESHOLD, TOTAL_PAGES)

print(f'[i] Number of workers {WORKERS}')
print(f'[i] Total pages is {TOTAL_PAGES}')
print(f'[i] Number of pages to be scrapped {THRESHOLD}')

# dataset columns
all_books = []
all_comments = []

async def worker(job_queue: asyncio.Queue, response_queue: asyncio.Queue, pbar):
    """
    this function run asynchronously and GET the book pages
    of the provided URL and put() the Response object
    to the responses_queue.
    """
    while True:
        # get() the page number of the page to scrap
        page_number = await job_queue.get()
        # print(f'[i] getting page {page_number}')
        # create an session object which runs asynchronously
        asession = AsyncHTMLSession()
        asession.headers.update(HEADERS)
        response = await asession.get(URL_TEMPLATE + page_number)
        response_queue.put_nowait(response)
        job_queue.task_done()

        # update progress bar
        pbar.update(1)

async def comment_worker(job_queue: asyncio.Queue, response_queue: asyncio.Queue, p_bar):
    """
    this function run asynchronously and GET the comments of books
    of the provided URL and put() the Response object
    to the responses_queue.
    """
    while True:
        # get() the page number of the page to scrap
        data = await job_queue.get()
        # print(f'[i] getting page {page_number}')
        # create an session object which runs asynchronously
        asession = AsyncHTMLSession()
        asession.headers.update(HEADERS)
        response = await asession.post('https://fidibo.com/book/comment', data)
        response_queue.put_nowait((response, data))
        job_queue.task_done()
        # print(f'[i] got page {page_number}')

        # update progress bar
        p_bar.update(1)

async def parse(res_q: asyncio.Queue, comments_job: asyncio.Queue, book_q):
    """
    this function get book attributes and queue job for
    getting comment total pages to be scraped and then
    create jobs for comments to be scraped
    """
    job_queue = asyncio.Queue()
    response_queue = asyncio.Queue()

    # while the is still a result available in the Queue
    while not res_q.empty():
        res = await res_q.get()
        data = res.json()
        [
            book_q.put_nowait({'id': _['id'], 'title': _['title'], 'authors': _['authors'], 'url': _['url']})
            for _ in data['books']
        ]

        for _ in data['books']:
            book_id = _['id']
            page = 1
            post_data = {'book_id': book_id, 'page': page}

            job_queue.put_nowait(post_data)

    pbar = tqdm(total=job_queue.qsize())
    tasks = []
    for i in range(WORKERS):
        task = asyncio.create_task(comment_worker(job_queue, response_queue, pbar))
        tasks.append(task)

    # for progress bar
    await job_queue.join()
    pbar.close()

    # create jobs for comments to be scraped
    while not response_queue.empty():
        response, post_data = response_queue.get_nowait()
        total_comments_page = response.json()['comment_paging']['pageCount']

        for page in range(1, total_comments_page + 1):
            post_data['page'] = page
            await comments_job.put(post_data)

    for task in tasks:
        task.cancel()

async def get_books_comments(comments_res: asyncio.Queue, comments_q: asyncio.Queue):
    """
    create comment dict from previous results
    """
    while not comments_res.empty():
        cms, post_data = comments_res.get_nowait()
        comments = cms.json()
        for comment in comments.get('Comments'):
            comments_q.put_nowait({
                'id': comment.get('id'),
                'book_id': comment.get('book_id'),
                'comment': comment.get('comment')
            })

async def main():
    # create FIFO Queue for both jobs to be done
    # and the result of the jobs
    job_queue = asyncio.Queue()
    response_queue = asyncio.Queue()
    book_queue = asyncio.Queue()

    comments_jobs = asyncio.Queue()
    comments_response = asyncio.Queue()
    comments_queue = asyncio.Queue()

    # create all page numbers that need to be scraped
    for _ in range(THRESHOLD):
        job_queue.put_nowait(str(_ + 1))  # to start from 1

    pbar = tqdm(total=THRESHOLD)
    # Create WORKER number of worker tasks to process the queue concurrently.
    tasks = []
    for i in range(WORKERS):
        task = asyncio.create_task(worker(job_queue, response_queue, pbar))
        tasks.append(task)

    # Wait until the queue is fully processed.
    started_at = time.monotonic()
    await job_queue.join()
    total_slept_for = time.monotonic() - started_at
    # close progressbar for the worker()
    pbar.close()

    print('\n[i] getting book URLs...')
    await parse(response_queue, comments_jobs, book_queue)
    print('[i] got all book URLs.')

    while not book_queue.empty():
        all_books.append(book_queue.get_nowait())

    # run all jobs for comments of books to be scraped
    comment_pbar = tqdm(total=comments_jobs.qsize())
    comment_tasks = []
    for i in range(WORKERS):
        task = asyncio.create_task(comment_worker(comments_jobs, comments_response, comment_pbar))
        comment_tasks.append(task)

    await comments_jobs.join()
    comment_pbar.close()

    print('\n[i] getting book comments...')
    await get_books_comments(comments_response, comments_queue)
    print('[i] got all book comments.')

    while not comments_queue.empty():
        all_comments.append(comments_queue.get_nowait())

    # Cancel our worker tasks.
    for task in tasks:
        task.cancel()

    for task in comment_tasks:
        task.cancel()
    # Wait until all worker tasks are cancelled.
    await asyncio.gather(*tasks, return_exceptions=True)

    # Wait until all worker tasks are cancelled.
    await asyncio.gather(*comment_tasks, return_exceptions=True)

asyncio.run(main())

# create dataframe and save them
df = pd.DataFrame(all_comments)
df.to_excel('comments.xlsx')
df.to_csv('comments.csv')

df = pd.DataFrame(all_books)
df.to_excel('books.xlsx')
df.to_csv('books.csv')

# saved dataframes will be analysed using score_comments.py


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
PROJECT EULER - PROBLEM 17
NUMBER LETTER COUNTS

https://projecteuler.net/problem=17

If the numbers 1 to 5 are written out in words: one, two, three, four, five,
then there are 3 + 3 + 5 + 4 + 4 = 19 letters used in total.

If all the numbers from 1 to 1000 (one thousand) inclusive were written out in
words, how many letters would be used?

NOTE: Do not count spaces or hyphens. For example, 342 (three hundred and
forty-two) contains 23 letters and 115 (one hundred and fifteen) contains 20
letters. The use of "and" when writing out numbers is in compliance with
British usage.
"""

def letterCount(number):
    """Returns the letter count of any integer between 1 and 999 when written
    out in words.
    """

    digits = str(number).zfill(3)

    # Letter count of numbers up to 19
    letters1 = [0, 3, 3, 5, 4, 4, 3, 5, 5, 4, 3, 6, 6, 8, 8, 7, 7, 9, 8, 8]

    # Letter count of tens from 20 to 90
    letters10 = [0, 0, 6, 6, 5, 5, 5, 7, 6, 6]

    # Units and tens
    if int(digits[-2:]) < 20:
        count = letters1[int(digits[-2:])]
    else:
        count = letters10[int(digits[-2])] + letters1[int(digits[-1])]

    # Hundreds
    if digits[-3] != '0':
        count += letters1[int(digits[-3])] + 7
        if digits[-2:] != '00':
            count += 3

    return count

letterCountSum = 11 # Letter count of "one thousand", not included in loop
for number in range(1000):
    letterCountSum += letterCount(number)

print(letterCountSum)


#-1 is useful to indicate that a stack is empty

class Stack():
    def __init__(self, size: int):
        self.__size = size
        self.__pos = -1
        self.stack = [None] * self.__size

    def push(self, pData):
        if self.isFull():
            return IndexError

        self.__pos += 1
        self.stack[self.__pos] = pData
        return self.stack[self.__pos]

    def pop(self):
        if self.isEmpty():
            return IndexError

        self.stack[self.__pos] = None
        self.__pos -= 1
        return self.stack[self.__pos +1]

    def isFull(self):
        if(self.__size == self.__pos -1):
            return True

        return False

    def isEmpty(self):
        if(self.__pos == -1):
            return True

        return False

stack1 = Stack(10)

for i in range(5):
    print(stack1.push(10+i))


import sys
import os
import astropy.io.fits as pyfits
import numpy
from astLib import astWCS

if __name__ == "__main__":

    catfile = sys.argv[1]
    data = numpy.loadtxt(catfile)

    fitsfile = sys.argv[2]
    hdu = pyfits.open(fitsfile)

    for i in range(1, len(hdu)):
        wcs = astWCS.WCS(hdu[i].header, mode="pyfits")

        sel = (data[:,4] == i)

        data_sel = data[sel]

        for j in range(data_sel.shape[0]):
            x, y = data_sel[j,0], data_sel[j,1]
            _ra, _dec = wcs.pix2wcs(x-1, y-1)

            print(x, y, i, data_sel[j, 2], data_sel[j, 3], _ra, _dec)