#To scrape comments and rating from Rotten Tomatoes

import requests
from bs4 import BeautifulSoup
import re

#To get the url of the site
#url=input("URL:")
#year=input("Year:")
#typ=input("Type:")
user_agent={'User-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36','Accept-Language':'en-us'}
req=requests.Session()

#key={'vudu':'Vudu', 'peacock':'Peacock', 'amc-plus-us':'AMC+', 'netflix':'Netflix', 'hulu':'Hulu', 'amazon-prime-video-us':'Amazon Prime', 'disney-plus-us':'Disney+', 'hbo-max':'HBO-Max', 'paramount-plus-us':'Paramount+', 'apple-tv-plus-us':'Apple TV+', 'showtime':'Showtime', 'itunes':'Apple TV'}
key={'vudu':'v', 'peacock':'p', 'amc-plus-us':'amc', 'netflix':'nf', 'hulu':'h', 'amazon-prime-video-us':'ap', 'disney-plus-us':'dp', 'hbo-max':'hm', 'paramount-plus-us':'pp', 'apple-tv-plus-us':'atp', 'showtime':'s', 'itunes':'a'}

#Scrape Rating, year(for shows) and where to watch
def rt(url,year,typ):
	w2wdict={'v':None, 'p':None, 'amc':None, 'nf':None, 'h':None, 'ap':None, 'dp':None, 'hm':None, 'pp':None, 'atp':None, 's':None, 'a':None}
	if url==None:
		w2wdict.update({'seasoninfo':None, 'rt1':rt1,'rt2':rt2, 'year':year, 'r1rt':criticscore, 'r2rt':userscore, 'type':typ})
		return w2wdict
	r=req.get(url,headers=user_agent)
	soup=BeautifulSoup(r.content,'html5lib')

	#Where to watch
	w2w=soup.find('bubbles-overflow-container',{'data-qa':'section:w2w-items'}).contents
	#print('Watch here :')
	i=0
	for item in w2w:
		try:
			#print(item['affiliate'],item['href'])
			w2wdict[key[item['affiliate']]]=item['href']
		except:
			pass

	#Rating
	if int(typ)==0:
		#Year
		score=soup.find('h1',class_='title').span.text.strip()
		#print('Year -',score)
		year=score

		score=soup.find('div',{'data-qa':'tomatometer-container'})
		#print('Critics -',score.find('span', {'data-qa':'tomatometer'}).text.strip())
		criticscore=score.find('span', {'data-qa':'tomatometer'}).text.strip()
		score=score.parent
		#print('Audience -',score.find('span', {'data-qa':'audience-score'}).text.strip())
		userscore=score.find('span', {'data-qa':'audience-score'}).text.strip()

		soup=soup.find('section', id='seasonList').find_all('season-list-item')
		seasoninfo=""
		for ssn in soup:
			seasoninfo=seasoninfo+'[@#]'+ssn['seasontitle']+' - '+ssn['tomatometerscore']+'%'

		seasoninfo=seasoninfo[4:]

		dict_={'seasoninfo': seasoninfo, 'rt1':None,'rt2':None, 'year':year, 'r1rt':criticscore, 'r2rt':userscore, 'type':typ}
		dict_.update(w2wdict)
		return dict_
	else:
		score=soup.find('score-board')
		#print('Critics -',score['tomatometerscore'])
		criticscore=score['tomatometerscore']+'%'
		#print('Audience -',score['audiencescore'])
		userscore=score['audiencescore']+'%'

		#Scrape Critic scores
		r=req.get(url+'/reviews?sort=fresh',headers=user_agent)
		soup=BeautifulSoup(r.content,'html5lib')
		try:
			rt1=soup.find('div',class_='the_review').text.strip()
		except:
			rt1=None

		r=req.get(url+'/reviews?sort=rotten',headers=user_agent)
		soup=BeautifulSoup(r.content,'html5lib')
		try:
			rt2=soup.find('div',class_='the_review').text.strip()
		except:
			rt2=None

		dict_={'seasoninfo':None, 'rt1':rt1,'rt2':rt2, 'year':year, 'r1rt':criticscore, 'r2rt':userscore, 'type':typ}
		dict_.update(w2wdict)
		return dict_

#print(rt(url,year,typ))


#
# $Id$
#
# Script to setup configuration for an individual user
#

# import all the required M modules, scripts are run in its own namespace
import Minit, MAppBase, MProfile, MString

# show a welcome dialog
msg = "Setting up initial configuration..."
MAppBase.MDialog_Message(msg)

# get global profile (appconfig)
profileparent = MAppBase.wxGetApp().GetProfile()

profileparent.readEntry("test","default")
#inboxname = MString.String("INBOX")

#newprofile = MProfile.ProfileBase_CreateProfile(inboxname,profileparent)
#newprofile.writeEntry("Type",0)


# coding: utf-8

from __future__ import absolute_import
from datetime import date, datetime  # noqa: F401

from typing import List, Dict  # noqa: F401

from swagger_server.models.base_model_ import Model
from swagger_server import util

class Prediction(Model):
    """NOTE: This class is auto generated by the swagger code generator program.

    Do not edit the class manually.
    """

    def __init__(self, prediction: str=None):  # noqa: E501
        """Prediction - a model defined in Swagger

        :param prediction: The prediction of this Prediction.  # noqa: E501
        :type prediction: str
        """
        self.swagger_types = {
            'prediction': str
        }

        self.attribute_map = {
            'prediction': 'prediction'
        }

        self._prediction = prediction

    @classmethod
    def from_dict(cls, dikt) -> 'Prediction':
        """Returns the dict as a model

        :param dikt: A dict.
        :type: dict
        :return: The prediction of this Prediction.  # noqa: E501
        :rtype: Prediction
        """
        return util.deserialize_model(dikt, cls)

    @property
    def prediction(self) -> str:
        """Gets the prediction of this Prediction.

        :return: The prediction of this Prediction.
        :rtype: str
        """
        return self._prediction

    @prediction.setter
    def prediction(self, prediction: str):
        """Sets the prediction of this Prediction.

        :param prediction: The prediction of this Prediction.
        :type prediction: str
        """
        if prediction is not None and len(prediction) > 10:
            raise ValueError("Invalid value for `prediction`, length must be less than or equal to `10`")  # noqa: E501
        if prediction is not None and len(prediction) < 2:
            raise ValueError("Invalid value for `prediction`, length must be greater than or equal to `2`")  # noqa: E501

        self._prediction = prediction


import numpy as np
import tensorflow as tf

class DataIterator(object):

    def __init__(self, train_set, val_set=None, test_set=None):
        self.iterator = tf.data.Iterator.from_structure(train_set.output_types, train_set.output_shapes)
        self.next_op = self.iterator.get_next()
        self.reinitialize(train_set, val_set, test_set)

    def reinitialize(self, train_set, val_set=None, test_set=None):
        self.init_train_set_op = self.iterator.make_initializer(train_set)
        if val_set is not None:
            self.init_val_set_op = self.iterator.make_initializer(val_set)
        if test_set is not None:
            self.init_test_set_op = self.iterator.make_initializer(test_set)


"""
Methods for the users namespace

https://docs.github.com/en/rest/reference/users
"""
from __future__ import annotations

from typing import Any, Dict

from ..const import GitHubRequestKwarg
from ..models.response import GitHubResponseModel
from ..models.user import GitHubAuthenticatedUserModel, GitHubUserModel
from .base import BaseNamespace

class GitHubUsersNamespace(BaseNamespace):
    """Methods for the users namespace"""

    async def get(
        self,
        username: str | None = None,
        **kwargs: Dict[GitHubRequestKwarg, Any],
    ) -> GitHubResponseModel[GitHubAuthenticatedUserModel | GitHubUserModel]:
        """
         Get a user

         **Arguments**:

         `username` (Optional)

         The username to return, example "octocat", if not provided,
         the authenticated user is returned

        https://docs.github.com/en/rest/reference/users#get-a-user
        """
        response = await self._client.async_call_api(
            endpoint="/user" if not username else f"/users/{username}",
            **kwargs,
        )
        if username:
            response.data = GitHubUserModel(response.data)
        else:
            response.data = GitHubAuthenticatedUserModel(response.data)

        return response


import functools
import numpy as np
import torch
import torch.nn.functional as F
from fastai.data.transforms import ItemTransform
from torch import nn as nn
from torch.nn import functional as F

"""
Definitions for the architectures. See paper for more details.

The NN modules in this section assume that the I/O data have the following form:

In:  Batch size x no_features x input_time_len
Out: Batch size x no_features x output_time_len

As in paper, x1 is the known past and x2 is the known future.

"""

class SelectableBN(nn.Module):
    """
    Allows one to select what flavour of BN to apply

    Selects between:
      0 no BN
      1 Regular BatchNorm
      2 just a learnable scale+bias without normalizing first the inputs.

    """
    def __init__(self, no_features, style=1):
        super().__init__()
        self.no_features=no_features
        self.style = style
        if style == 0:
            self.act = nn.Identity()
        elif style == 1:
            self.act = nn.BatchNorm1d(self.no_features)
        else:
            self.weight = nn.Parameter(torch.ones(1, self.no_features, 1))
            self.bias = nn.Parameter(torch.zeros(1, self.no_features, 1))

    def forward(self, data):
        if self.style in [0, 1]:
            out = self.act(data)
        if self.style == 2:
            batch_size = data.size(0)
            time_len = data.size(2)
            out = data * self.weight.expand(batch_size, self.no_features, time_len) +\
                  self.bias.expand(batch_size, self.no_features, time_len)
        return out

class AffineTransform(nn.Module):
    """
    An affine transform using some learned parameters.

    The one_kernel_per_feature parameter is ignored.

    """
    def __init__(self, no_features, input_time_len, output_time_len=None, one_kernel_per_feature=None, bn_type=1):
        super().__init__()
        if output_time_len is None:
            output_time_len = input_time_len
        if one_kernel_per_feature == True:
            output_time_len = input_time_len
        self.no_features = no_features
        self.input_time_len = input_time_len
        self.output_time_len = output_time_len
        self.translate = nn.Parameter(torch.zeros(self.no_features))
        self.scale = nn.Parameter(torch.zeros(self.no_features))
        self.bn = SelectableBN(self.no_features, bn_type)
        self.init_params()

    def init_params(self):
        self.translate.data = torch.randn(self.no_features) * 0.1
        self.scale.data = torch.randn(self.no_features) * 0.15

    def forward(self, data):
        batch_size = data.shape[0]
        affine = torch.zeros(self.no_features, 2, 3, dtype=torch.float32, device=self.scale.device)
        # Scale, then translate
        scale_factor = torch.exp(self.scale / 5)
        affine[:, 0, 0] = scale_factor
        affine[:, 1, 1] = 1
        affine[:, 0, 2] = scale_factor * self.translate
        grid_tensor = torch.nn.functional.affine_grid(affine, [self.no_features, 1, 1, self.output_time_len],
                                                          align_corners=False)
        grid_tensor_rep = grid_tensor.repeat(batch_size, 1, 1, 1)
        data_rep = data.reshape(-1, 1, self.input_time_len).unsqueeze(2)
        filtered_signal = torch.nn.functional.grid_sample(data_rep, grid_tensor_rep, mode='bilinear', align_corners=False)
        filtered_signal.squeeze(2)
        filtered_signal = filtered_signal.view(batch_size, self.no_features, self.output_time_len)
        scaled = self.bn(filtered_signal)
        return scaled

class LogGauss(nn.Module):
    """
    A fixed logNormal kernel is affine transformed using some learned parameters.

    """
    def __init__(self, no_features, input_time_len, output_time_len=None, one_kernel_per_feature=True, bn_type=1):
        super().__init__()
        if one_kernel_per_feature == False:
            if output_time_len is None:
                output_time_len = input_time_len
        self.no_features = no_features
        self.input_time_len = input_time_len
        self.one_kernel_per_feature = one_kernel_per_feature
        self.output_time_len = output_time_len
        self.kernel_width, self.pad = LogGauss.get_kernel_size_and_padding(self.input_time_len, one_kernel_per_feature)
        if one_kernel_per_feature == False:
            self.internal_no_features = self.no_features * self.output_time_len
        else:
            self.internal_no_features = self.no_features
        self.register_buffer("mother_kernel", self.gen_base_kernel())
        self.translate = nn.Parameter(torch.zeros(self.internal_no_features))
        self.scale = nn.Parameter(torch.zeros(self.internal_no_features))
        self.bn = SelectableBN(self.internal_no_features, bn_type)
        self.init_params()

    def init_params(self):
        self.translate.data = torch.randn(self.internal_no_features) * 0.1
        self.scale.data = torch.randn(self.internal_no_features) * 0.5
        if self.one_kernel_per_feature == False:
            wid_div = 0.9
            self.translate.data = self.translate.data - \
                                  torch.linspace(-wid_div, wid_div, self.output_time_len).repeat(self.no_features)

    def gen_base_kernel(self):
        no_samples = 100
        max_range = 20
        base = torch.linspace(1e-5, max_range, no_samples, requires_grad=False, dtype=torch.float32)
        loggauss = 1 / ((base)) * torch.exp(-np.square(np.log(base)) / 2)
        loggauss[torch.isnan(loggauss)] = 0
        filter_mode = no_samples / max_range * np.exp(-1)
        new_start_pos = int(no_samples / 2 - filter_mode) - 1
        loggauss = torch.roll(loggauss, new_start_pos)
        loggauss[:new_start_pos] = 0
        loggauss = loggauss.flip(0)
        return loggauss

    def forward(self, data):
        batch_size = data.shape[0]
        scaled_filters = self.get_scaled_filters()
        filtered_signal = nn.functional.conv1d(data, scaled_filters, padding=self.pad, groups=self.no_features)
        scaled = self.bn(filtered_signal)
        if not self.one_kernel_per_feature:
            scaled = scaled.view(batch_size, self.no_features, self.output_time_len)
        return scaled

    def get_scaled_filters(self):
        affine = torch.zeros(self.internal_no_features, 2, 3, dtype=torch.float32, device=self.scale.device)
        scale_factor = torch.exp(self.scale) + 3
        affine[:, 0, 0] = scale_factor
        affine[:, 1, 1] = 1
        affine[:, 0, 2] = scale_factor * self.translate
        grid_tensor = torch.nn.functional.affine_grid(affine, [self.internal_no_features, 1, 1, self.kernel_width],
                                                      align_corners=False)
        base_kern = self.mother_kernel
        base_kern = base_kern.expand(self.internal_no_features, 1, 1, base_kern.shape[0])
        scaled_kern = torch.nn.functional.grid_sample(base_kern, grid_tensor, mode='bilinear', align_corners=False)
        scaled_kern = scaled_kern.squeeze(2)  # Remove W dimension
        scaled_kern = scaled_kern / (torch.sum(scaled_kern, dim=2).view(-1, 1, 1) + 1e-7)
        return scaled_kern

    @staticmethod
    def get_kernel_size_and_padding(input_size, one_kernel_per_feature=True):
        if one_kernel_per_feature:
            offset = (input_size + 1) % 2
            nk = input_size + offset
            pad = int((input_size - input_size - 1 + nk) / 2)  # compute padding so out signal len == in signal len
        else:
            assert input_size is not None, "There must be a non null output size specified!"
            nk = input_size   # The conv1d will return ONE value per convolution
            pad = 0
        return nk, pad

class GaborFilter(nn.Module):
    """
    Gabor filters with fixed bandwidth. They output the magnitute and orientation of the complex convolution.

    Unstable for now in deeper architectures.

    """
    def __init__(self, no_features, input_time_len, output_time_len=None, one_kernel_per_feature=True,  bn_type=1, bandwidth=2.5,):
        super().__init__()
        self.one_kernel_per_feature = one_kernel_per_feature
        self.no_features = no_features
        self.input_time_len = input_time_len
        self.internal_no_features = 1
        self.bandwidth = bandwidth
        dtype = torch.float32
        if output_time_len is None:
            output_time_len = input_time_len
        self.output_time_len = output_time_len
        if self.one_kernel_per_feature:
            self.internal_no_features = no_features
        else:
            self.internal_no_features = no_features * output_time_len
        self.phase = 0
        self.nk, self.pad = GaborFilter.get_kernel_size_and_padding(self.input_time_len, self.one_kernel_per_feature)
        #         print(f"nk:{self.nk} pad: { self.pad}")
        self.support_range = max(self.nk / 2, 6)  # To avoid aliasing, but limits the offset range.
        self.frequency = nn.Parameter(torch.zeros(self.internal_no_features, dtype=dtype, requires_grad=True))
        self.mu = nn.Parameter(torch.zeros(self.internal_no_features, dtype=dtype, requires_grad=True))
        self.bn_mag = SelectableBN(self.internal_no_features, bn_type)
        self.bn_orient = SelectableBN(self.internal_no_features, bn_type)

        self.kernel_support = nn.Parameter(torch.zeros(self.nk, dtype=dtype, requires_grad=False))
        self.init_params()

    def compute_kernel(self):
        mu = self.mu
        frequency = torch.sigmoid(self.frequency)*(0.5 - 1.0/self.nk) + 1.0/self.nk  # min frequency =1/T, max frequency 1/2
        mu = mu.view(-1, 1)
        frequency = frequency.view(-1, 1)
        sigma = self.bw_constant / frequency
        real_comp = torch.cos((self.kernel_support - mu) * frequency * self.two_pi_constant)
        imag_comp = torch.sin((self.kernel_support - mu) * frequency * self.two_pi_constant)
        exp_comp = torch.exp(-torch.square((self.kernel_support - mu) / sigma) / 2)
        gabor_kernels_real = real_comp * exp_comp
        gabor_kernels_imag = imag_comp * exp_comp

        gabor_kernels_real = gabor_kernels_real.flip(1).unsqueeze(1)
        gabor_kernels_imag = gabor_kernels_imag.flip(1).unsqueeze(1)
        return gabor_kernels_real, gabor_kernels_imag

    def forward(self, data):
        batch_size = data.shape[0]
        gabor_kernels_real, gabor_kernels_imag = self.compute_kernel()
        if torch.isnan(gabor_kernels_real).any():
            print(f"NaNs detected in gabor_kernel_real")
        if torch.isnan(gabor_kernels_imag).any():
            print(f"NaNs detected in gabor_kernels_imag")

        filtered_signal_real = nn.functional.conv1d(data, gabor_kernels_real, padding=self.pad, groups=self.no_features)
        filtered_signal_imag = nn.functional.conv1d(data, gabor_kernels_imag, padding=self.pad, groups=self.no_features)
        magnitude = torch.sqrt(torch.square(filtered_signal_real) + torch.square(filtered_signal_imag))
        orientation = torch.atan2(filtered_signal_imag, filtered_signal_real)

        scaled_magnitude = self.bn_mag(magnitude)
        scaled_orientation = self.bn_orient(orientation)
        if not self.one_kernel_per_feature:
            scaled_magnitude = scaled_magnitude.view(batch_size, self.no_features, self.output_time_len)
            scaled_orientation = scaled_orientation.view(batch_size, self.no_features, self.output_time_len)
        return scaled_magnitude, scaled_orientation

    def init_params(self):
        self.kernel_support.data = torch.linspace(-self.support_range, self.support_range, self.nk)
        self.frequency.data = torch.randn(self.internal_no_features) * 1 + 2
        self.mu.data = (torch.rand(self.internal_no_features) - 0.5) * 0.1 * self.support_range
        if self.one_kernel_per_feature == False:
            wid_div = self.support_range * 0.9
            self.mu.data -= torch.linspace(-wid_div, wid_div, self.output_time_len).repeat(self.no_features)
        two_df = np.power(2, self.bandwidth)
        self.bw_constant = np.sqrt(np.log(2) / np.pi) * (two_df + 1)/(two_df - 1)
        self.two_pi_constant = np.pi * 2

    @staticmethod
    def get_kernel_size_and_padding(input_size, one_kernel_per_feature=True):
        if one_kernel_per_feature:
            offset = (input_size + 1) % 2
            nk = input_size + offset
            pad = int((input_size - input_size - 1 + nk) / 2)  # compute padding so out signal len == in signal len
        else:
            assert input_size is not None, "There must be a non null output size specified!"
            nk = input_size   # The conv1d will return ONE value per convolution
            pad = 0
        return nk, pad

class IdentityTransform(nn.Module):
    """
    An identity transform that forwards the input to the output.

    If one_kernel_per_feature parameter is False and the output_time_len < input_time_len the output is a selected
    view of the input. Only the last part of the input signal is sent to the output.

    No batch norm is perfomed. (bn_type is ignored)
    """

    def __init__(self, no_features, input_time_len, output_time_len=None, one_kernel_per_feature=True, bn_type=None):
        super().__init__()
        if output_time_len is None:
            output_time_len = input_time_len
        self.no_features = no_features
        self.input_time_len = input_time_len
        self.output_time_len = output_time_len
        self.one_kernel_per_feature = one_kernel_per_feature

    def forward(self, data):
        batch_size = data.shape[0]
        if (self.one_kernel_per_feature == False) and (self.input_time_len != self.output_time_len):
            return data[:, :, -self.output_time_len:]
        return data

class CausalTemporalConv(nn.Module):
    """
    Causal temporal convolution.

    It will "shrink" the input, along time, from input_time_len to output_time_len.

    There is a convolution layer that performs this transformation. Each feature will have its own learnable
    convolution kernel. There is no crossover between featuers.

    The one_kernel_per_feature must be false for compatibility with other filters.

    """
    def __init__(self, no_features, input_time_len, output_time_len, one_kernel_per_feature=False):
        super().__init__()
        assert (one_kernel_per_feature == False), "one_kernel_per_feature must be false so output_time_len makes sense"
        assert (output_time_len <= input_time_len), "The desired output must be smaller than the input"
        kernel_size = input_time_len - output_time_len + 1
        self.conv = nn.Conv1d(in_channels=no_features, out_channels=no_features, kernel_size=kernel_size, padding=0,
                              groups=no_features)

    def forward(self, data):
        out = self.conv(data)
        return out

class GaussFilter(nn.Module):
    """
    Gaussian filter.

    """
    def __init__(self, no_features, input_time_len, output_time_len=None, one_kernel_per_feature=True, bn_type=1):
        super().__init__()
        self.one_kernel_per_feature = one_kernel_per_feature
        self.no_features = no_features
        self.input_time_len = input_time_len
        self.internal_no_features = 1
        dtype = torch.float32
        if output_time_len is None:
            output_time_len = input_time_len
        self.output_time_len = output_time_len
        if self.one_kernel_per_feature:
            self.internal_no_features = no_features
        else:
            self.internal_no_features = no_features * output_time_len
        self.nk, self.pad = GaborFilter.get_kernel_size_and_padding(self.input_time_len, self.one_kernel_per_feature)
        self.support_range = max(self.nk / 2, 3)  # To avoid aliasing, but limits the offset range.
        self.sigma = nn.Parameter(torch.zeros(self.internal_no_features, dtype=dtype, requires_grad=True))
        self.mu = nn.Parameter(torch.zeros(self.internal_no_features, dtype=dtype, requires_grad=True))
        self.bn = SelectableBN(self.internal_no_features, bn_type)
        self.kernel_support = nn.Parameter(torch.zeros(self.nk, dtype=dtype, requires_grad=False))
        self.init_params()

    def forward(self, data):
        batch_size = data.shape[0]
        gauss_kernel = self.compute_kernel()
        filtered_signal = nn.functional.conv1d(data, gauss_kernel, padding=self.pad, groups=self.no_features)
        scaled = self.bn(filtered_signal)
        if not self.one_kernel_per_feature:
            scaled = scaled.view(batch_size, self.no_features, self.output_time_len)
        return scaled

    def init_params(self):
        self.kernel_support.data = torch.linspace(-self.support_range, self.support_range, self.nk)
        self.sigma.data = torch.randn(self.internal_no_features) * 0.1
        self.mu.data = (torch.randn(self.internal_no_features)) * 0.01 * self.support_range
        if self.one_kernel_per_feature == False:
            wid_div = self.support_range * 0.9
            self.mu.data -= torch.linspace(-wid_div, wid_div, self.output_time_len).repeat(self.no_features)

    def compute_kernel(self):
        mu = self.mu
        sigma = self.sigma
        mu = mu.view(-1, 1)
        sigma = sigma.view(-1, 1)
        gauss_kernel = torch.exp(-torch.square((self.kernel_support - mu) / (torch.exp(sigma))) / 2)
        gauss_kernel = gauss_kernel.flip(1).unsqueeze(1)
        return gauss_kernel

    @staticmethod
    def get_kernel_size_and_padding(input_size, one_kernel_per_feature=True):
        if one_kernel_per_feature:
            offset = (input_size + 1) % 2
            nk = input_size + offset
            pad = int((input_size - input_size - 1 + nk) / 2)  # compute padding so out signal len == in signal len
        else:
            assert input_size is not None, "There must be a non null output size specified!"
            nk = input_size   # The conv1d will return ONE value per convolution
            pad = 0
        return nk, pad

class BankedFilters(nn.Module):
    """
    One will want more filters per feature and this class organize these filters.

    All filters here have the same meta-properties (eg filtering type) but different learnable parameters.

    The only parameters that is not forwarded is number_of_filters that specify how many filters are in this bank.

    """
    def __init__(self, no_features, input_time_steps, filter_class, number_of_filters=8, one_kernel_per_feature=True, bn_type=1):
        super().__init__()
        assert number_of_filters >= 1, "There must be at least one filter in the filter bank"
        filter_bank = []
        assert filter_class in [IdentityTransform, GaussFilter, AffineTransform, GaborFilter, LogGauss]
        self.filter_class = filter_class
        self.no_features = no_features
        self.number_of_filters = number_of_filters
        for k in range(number_of_filters):
            f1 = self.filter_class(no_features, input_time_steps, input_time_steps, one_kernel_per_feature=one_kernel_per_feature, bn_type=bn_type)
            filter_bank.append(f1)
        self.filter_bank = nn.ModuleList(filter_bank)

    def get_number_of_out_features(self):
        no_out_features = self.no_features * self.number_of_filters
        if self.filter_class == GaborFilter:
            return 2 * no_out_features
        return no_out_features

    def forward(self, data):
        """
        data have shape: Batch x Features x Time
        """
        acc_features = []
        if self.filter_class == GaborFilter:
            for gf in self.filter_bank:
                mag, orient = gf(data)
                acc_features.extend([mag, orient])
        else:
            for gf in self.filter_bank:
                filter_out = gf(data)
                acc_features.append(filter_out)
        out = torch.cat(acc_features, dim=1)
        return out

class FeatureAggregationStack(nn.Module):
    """
    Transforms a set of features into an output using a fully connected network.

    The input is of shape batch x no_features x time_len and the FCN is applied only accross features.
    The weights are shared for all temporal steps (along time_len dimension)

    The FCN is implemented using a series of 1D convolutional kernels.

    One can select the number of linear layers that do the transform and the intermediate size of those layers.
    The number of layers is no_layers+1:

    If no_layers is set to zero, only a linear operation is applied.

    If no_layers is one, two linear transformations will be present.

    Between the layers is leaky_relu nonlinearity

    No activation is performed on the output. So, if no_layers is 0, this module will perform a linear transformation.

    An optional (do_initial_bn) BatchNorm is performed on the input.

    Inputs:  Batch x no_features x time_len
    Outputs: Batch x no_out_features x time_len

    """
    def __init__(self, no_features, no_out_features, no_intermediate_features, no_layers, do_initial_bn=True):
        super().__init__()
        self.do_initial_bn = do_initial_bn
        if self.do_initial_bn:
            self.bn = nn.BatchNorm1d(no_features)
        self.no_layers = no_layers

        if no_layers <= 0:
            self.input = nn.Conv1d(no_features, no_out_features, kernel_size=1, bias=True)
        else:
            self.input = nn.Conv1d(no_features, no_intermediate_features, kernel_size=1, bias=False)
            self.output = nn.Conv1d(no_intermediate_features, no_out_features, kernel_size=1, bias=True)
            self.interm_layers = nn.ModuleList([nn.Conv1d(no_intermediate_features, no_intermediate_features, kernel_size=1, bias=True) for _ in range(no_layers - 1)])

    def forward(self, data):
        data_bn = data
        if self.do_initial_bn:
            data_bn = self.bn(data)
        if self.no_layers == 0:
            out = self.input(data_bn)
        else:
            interm_data = F.leaky_relu(self.input(data_bn))
            for k,layer in enumerate(self.interm_layers):
                interm_data = F.leaky_relu(layer(interm_data))
            out = self.output(interm_data)
        return out

class ConstructDelayNet(nn.Module):
    """
    Instantiate a heavily customizable Delay type architecture.

    The Delay architecture (shown approx in Fig. 5 in the paper) have several blocks, each block with parameters.
    This class allows one to instantiate everything in one go.

    Parameters like no_features, no_commands, no_targets, past_length, future_length are specific to the inputs and outputs,
    the rest of the parameters are sent to their specific blocks:

    filter_* to BankedFilters class
    aggregator_* to TimeWiseLinearStack
    temporal_contractor to CausalTemporalConv or directly to one of the filters, having one_kernel_per_feature == False
    The number of I/O features for the filters is deduced automatically based on other parameters.

    aggregator_low_out_bottleneck is the **Fc** parameter in the paper. It decides how many features the aggregator_low
    block will output.

    The bn_type_* allows one to experiment with the batch normalization after filtering. Setting it to 1 is a sensitive choice.
    Check SelectableBN for some options.

    out_interpolator allows some interpolation for the final output, slightly reducing the network capacity. A sensitive
    choice is to not use it.

    The input shapes for this networ are:

    In:  A tuple tensor: (Batch size x no_features x past_length , Batch size x no_commands x (past_length + future_length) )
    Out: A tensor: Batch size x no_targets x future_length

    """
    def __init__(self, no_features, no_commands, no_targets, past_length, future_length,
                 filter_low_classname, filter_low_filter_count, filter_low_one_kernel_per_feature,
                 aggregator_low_expansion, aggregator_low_layers, aggregator_low_out_bottleneck,
                 temporal_contractor_classname,
                 filter_high_classname, filter_high_filter_count, filter_high_one_kernel_per_feature,
                 aggregator_high_expansion, aggregator_high_layers,
                 bn_type_low=1, bn_type_temporal=1, bn_type_high=1, out_interpolator=None):
        super().__init__()
        self.no_features = no_features
        self.no_commands = no_commands
        self.no_targets = no_targets
        self.past_timelen = past_length
        self.future_timelen = future_length
        self.total_time = past_length + future_length
        if out_interpolator is None:
            out_interpolator = self.future_timelen

        self.internal_future_timelen = out_interpolator

        self.filter_low_classname = filter_low_classname
        self.filter_low_filter_count = filter_low_filter_count
        self.aggregator_low_expansion = aggregator_low_expansion
        self.aggregator_low_layers = aggregator_low_layers
        self.aggregator_low_out_bottleneck = aggregator_low_out_bottleneck
        self.temporal_contractor_classname = temporal_contractor_classname
        self.filter_high_classname = filter_high_classname
        self.filter_high_filter_count = filter_high_filter_count
        self.aggregator_high_expansion = aggregator_high_expansion
        self.aggregator_high_layers = aggregator_high_layers

        self.filter_low = BankedFilters(self.no_features, self.past_timelen, eval(self.filter_low_classname),
                                        self.filter_low_filter_count, filter_low_one_kernel_per_feature, bn_type=bn_type_low)
        no_features_after_expander_low = self.filter_low.get_number_of_out_features()
        self.aggregator_low = FeatureAggregationStack(no_features_after_expander_low,
                                                      self.aggregator_low_out_bottleneck,
                                                      int(no_features_after_expander_low * self.aggregator_low_expansion),
                                                      self.aggregator_low_layers, do_initial_bn=False)
        if self.temporal_contractor_classname in ["IdentityTransform", "GaussFilter", "AffineTransform", "GaborFilter", "LogGauss"]:
            self.temporal_contractor = eval(self.temporal_contractor_classname)(self.aggregator_low_out_bottleneck,
                                                                                self.past_timelen, self.internal_future_timelen,
                                                                                one_kernel_per_feature=False, bn_type=bn_type_temporal)
        elif self.temporal_contractor_classname == "CausalTemporalConv":
            self.temporal_contractor = CausalTemporalConv(self.aggregator_low_out_bottleneck, self.past_timelen, self.internal_future_timelen)
        else:
            assert False, "temporal_contractor_classname is wrong. Check documentation"
        self.high_feature_count = self.aggregator_low_out_bottleneck + self.no_commands
        self.filter_high = BankedFilters(self.high_feature_count, self.internal_future_timelen, eval(self.filter_high_classname),
                                         self.filter_high_filter_count, filter_high_one_kernel_per_feature, bn_type=bn_type_high)
        self.high_feature_count_expanded = self.filter_high.get_number_of_out_features()
        self.aggregator_high = FeatureAggregationStack(self.high_feature_count_expanded, self.no_targets,
                                                       int(self.high_feature_count_expanded * self.aggregator_high_expansion),
                                                       self.aggregator_high_layers, do_initial_bn=False)

    def forward(self, *data):
        feature_block = data[0]
        command_block = data[1]
        batch_size = feature_block.shape[0]

        feat_delay_low = self.filter_low(feature_block)
        feature_mid = F.leaky_relu(self.aggregator_low(feat_delay_low))
        feature_mid_future = F.leaky_relu((self.temporal_contractor(feature_mid)))
        if self.internal_future_timelen != self.future_timelen:
            command_block = nn.functional.interpolate(command_block, size=(self.internal_future_timelen, ), mode='linear', align_corners=False)
        feature_mid_all = torch.cat([feature_mid_future, command_block], dim=1)
        feature_high_delayed = F.leaky_relu(self.filter_high(feature_mid_all))
        out_aggregated = self.aggregator_high(feature_high_delayed)
        if self.internal_future_timelen != self.future_timelen:
            out_aggregated = nn.functional.interpolate(out_aggregated.unsqueeze(1), size=(1, self.future_timelen, ), mode='bilinear', align_corners=False)
            out_aggregated = out_aggregated.squeeze(1)
        return out_aggregated

class ICCP_Wrap_AttentionModel(nn.Module):
    """
    This model is a small wrapper around the previously developed Attention model.

    The models developed in previous iterations had fixed sizes (of one) for the number of command features and
    number of target features. In current iteration, one can have more than one command and can predict more than one
    target.

    When there are more commands and targets, only the first command and target are sent to the Attention network.

    """
    def __init__(self, no_features, no_commands, no_targets, past_length, future_length,
                 hidden_size, num_layers=1, dropout=0.5):
        super().__init__()
        in_features = no_features
        self.base_network = AttentionModel(in_features, past_length, future_length,
                                                        hidden_size, num_layers, dropout)

    def forward(self, *data):
        feature_block = data[0]
        command_block = data[1]
        out = self.base_network((feature_block, command_block.squeeze(1)))
        out = out.unsqueeze(1)
        return out

class FeatureItemizer(ItemTransform):
    """
    Generates the features, commands and ground truth data from list of continuous sequences of data.

    The data list is parsed by the fastai Transform and the encodes() function receive one sample at a time.
    One assumes that the input is a blend of features, commands, and targets (eg output of the system).
    Up to past_time these are known. The commands are known for the future_time too.

    Input for encodes(): Time x K array, where K is no of features + no of commands + no of targets
    Output: Several arrays:
        len(known_set) x past_time representing the known features+commands+targets
        len(comands_set) x future_len representing the future proposed commands
        len(target_set) x future_time, future targets, for the future period. Used as ground truth in learning

    In setups(), the method will calculate avg and std for each group in feature_groups. If some features are not in
    feature_groups, each one will be treated as separate groups.

    If average_range is >0, then the last average_range values from the known past, of the target, are averaged and this
    average is then substracted from all the features in the first feature group. So, make sure that the target_set is
    in the feature_groups[0]

    If average_range == 0 this step is ignored.

    Use decode_predictions_from_the_network() to reconstruct the samples

    Constructor parameters: future_len, comands_set, target_set are lists of column indices that will be exported to
    the output (eg vectors X1, X2, Y from paper)

    """

    def __init__(self, future_len, known_set, comands_set, target_set, feature_groups, average_range=0.2):
        super().__init__()
        self.future_len = future_len
        self.known_set = known_set
        self.comands_set = comands_set
        self.target_set = target_set
        self.feature_groups = feature_groups
        self.average_range = average_range
        self.avgs = None

    def encodes(self, input):
        # input of shape Time x Features
        features_input = input - self.avgs  # Only one block of inputs
        features_input /= self.stds
        N = features_input.shape[0]
        past_len = N - self.future_len

        if self.average_range > 0:
            count_to_subst = int(past_len *(1-self.average_range))
            val_to_substract = np.average(features_input[count_to_subst:past_len, self.target_set])
            features_input[:, self.feature_groups[0]] -= val_to_substract

        known_past = features_input[:past_len, self.known_set]
        future_commands = features_input[past_len:N, self.comands_set]
        targets_future = features_input[past_len:N, self.target_set]

        return torch.tensor(known_past, dtype=torch.float).transpose(0, 1), \
               torch.tensor(future_commands, dtype=torch.float).transpose(0, 1), \
               torch.tensor(targets_future, dtype=torch.float).transpose(0, 1),

    def setups(self, items):
        if self.avgs is not None:  # avoid re-setup, to be refactored with dataset_index
            return
        fg_new = []  # lone features that will be grouped
        item = items[0]
        no_features = item.shape[1]
        items_arr = np.reshape(np.array(items), (-1, no_features))
        grouped_features = functools.reduce(set.union, self.feature_groups, set())
        for k in range(no_features):
            if k not in grouped_features:
                fg_new.append((k,))
        self.feature_groups.extend(fg_new)
        self.avgs = np.zeros(no_features)
        self.stds = np.ones(no_features)
        for group in self.feature_groups:
            group = list(group)
            features = items_arr[:, group].ravel()
            avgs = np.average(features)
            stds = np.std(features)
            self.avgs[group] = avgs
            self.stds[group] = stds

def decode_predictions_from_the_network(network_output, data_sample_array, itemizer):
    """
    Takes a network output generated by the  fastai get_preds() method (either target or preds) and the list of samples (in an array form)
    and the itemizer instance that coded the instances.

    It returns the network output scaled back and with eventual extracted average added.

    See the demo notebook on how to use it.

    """
    N = data_sample_array.shape[1]
    past_len = N - itemizer.future_len
    val_to_substract = 0
    if itemizer.average_range > 0:
        count_to_subst = int(past_len *(1-itemizer.average_range))
        to_avg = data_sample_array[:, count_to_subst:past_len, itemizer.target_set]
        a1 = np.average(to_avg, axis=1)
        a2 = itemizer.avgs[itemizer.target_set]
        val_to_substract = a1 - a2
    stds = itemizer.stds[itemizer.target_set]
    avgs = itemizer.avgs[itemizer.target_set]
    pred_set_np = network_output.squeeze(1).detach().cpu().numpy()
    decoded = stds * pred_set_np + avgs + val_to_substract
    return decoded

############################
#
# Att_* network definitions. Used indirectly, through ICCP_Wrap_AttentionModel()
#
############################

class AttentionEncoding(nn.Module):
    """
    The encoder from Attention architecture. It works only on the input x1

    """
    def __init__(self, in_features, sample_len, hidden_size, num_layers, dropout):
        super().__init__()
        self.in_features = in_features
        self.sample_len = sample_len
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        if num_layers == 1:
            dropout=0
        self.bn = nn.BatchNorm1d(num_features=in_features)
        self.gru = nn.LSTM(in_features, hidden_size, num_layers=num_layers, dropout=dropout,  batch_first=True)

    def forward(self, data):
        bn = self.bn(data)          # B, F, S
        bn_t = bn.transpose(2, 1)   # B, S, F, needed for LSTM
        output, hidden = self.gru(bn_t) # Output: (B, S, H),  2 x (num_layers, B, H)
        return output, hidden

class AttentionDecoding(nn.Module):
    """
    The decoder, from the architecture. Implements the attention mechanism.

    It expects one "output" at a time
    """
    def __init__(self, hidden_size, sample_len, input_features, output_size, num_layers, dropout):
        """

        :param hidden_size: Same as for encoder
        :param sample_len: The "future" size, T
        :param input_features:  The number of input features, F2 (here, == 2)
        :param output_size: The output size, here == 1
        :param num_layers:  Same as for encoder.
        :param dropout: Ignored if num_layers == 1
        """
        super().__init__()
        self.hidden_size = hidden_size
        self.sample_len = sample_len
        self.input_features = input_features
        self.num_layers = num_layers
        self.hidden_flat = hidden_size * num_layers
        self.output_size = output_size

        self.attn = nn.Linear(self.input_features + self.hidden_flat, self.sample_len)
        self.attn_combine = nn.Linear(self.input_features + self.hidden_size, self.hidden_size)
        self.bn_input_attn = nn.BatchNorm1d(num_features=self.input_features+self.hidden_size)
        self.gru = nn.LSTM(self.hidden_size, self.hidden_size, num_layers=num_layers, dropout=dropout, batch_first=True)
        self.out = nn.Linear(self.hidden_size, self.output_size)

    def forward(self, input, encoder_outputs, hidden):
        """

        :param input: size B, 1, input_features, the previous prediction
        :param encoder_outputs: BS, sample_len, hidden_size , the otuput of the encoder
        :param hidden: *  the output of the encoder.
        :return:
        """
        bs = input.size(0)              # batch size B
        if type(self.gru) is nn.LSTM:
            hidden_for_cat = hidden[0]      # Because this is where LSTM keeps its hidden states
        else:
            hidden_for_cat = hidden         # GRU does not output a tuple for the hidden states.
        # We flatten the hidden states, batch-wise
        hidden_for_cat = hidden_for_cat.view(1, bs, self.hidden_flat) # 1, B, num_layers * H
        # Attach the previous known output
        cat_hidden_input = torch.cat((input.squeeze(1), hidden_for_cat.squeeze(0)), 1) # B, num_layers * H + input_features
        # Compute the weights for the attention. One weight for each past timestamp
        attn_weights = self.attn(cat_hidden_input) # B, S
        # One of the past items is relevant, so softmax to find it.
        attn_weights_smax = torch.softmax(attn_weights, dim=1) # attn_weights_smax shape (B, S)
        # Apply the attention to the encoder outputs and outuput the relevant past information
        attn_applied = torch.bmm(attn_weights_smax.unsqueeze(1),
                                 encoder_outputs)  #attn_applied shape: (BS, 1, Hidden)
        # Attach the previous output to the relevant past information
        att_plus_input = torch.cat((input.squeeze(1), attn_applied.squeeze(1)), 1)
        # Regularize the responses, otherwise there is no learning for num_layers > 1
        att_plus_input = self.bn_input_attn(att_plus_input)
        # Generate the input for LSTM, with a linear layer
        att_plus_input_comb = self.attn_combine(att_plus_input)
        att_plus_input_comb = torch.sigmoid(att_plus_input_comb).unsqueeze(1)
        output, hidden_out = self.gru(att_plus_input_comb, hidden) # att_plus_input_comb shape (BS, 1, H),hidden: 2 x (num_layers, BS, H)
        output = torch.tanh(output)
        output = self.out(output.squeeze(1))
        # output shape (B, output_size)
        # hidden_out shape 2  x (num_layers, B, Hidden) -> This will be injected back
        return output, hidden_out

class AttentionModel(nn.Module):
    """
    Main attention module. Will call AttentionEncoding and AttentionDecoding

    Handles a bit of housekeeping and data iteration.

    Does NOT implement teacher forcing, each future time step is based on the network's previous output.

    """

    def __init__(self, in_features, past_steps, future_steps, hidden_size, num_layers=1, dropout=0.5):
        """

        :param in_features: F
        :param past_steps:  S
        :param future_steps:  T
        :param hidden_size:  H
        :param num_layers:  Number of LSTM layers, same for encoder and decoder
        :param dropout: Dropout between LSTM layers. Ignored if num_layers == 1
        """
        super().__init__()
        if num_layers == 1:
            dropout=0
        self.in_features = in_features
        self.past_steps = past_steps
        self.hidden_size = hidden_size
        self.future_steps = future_steps
        self.num_layers = num_layers
        self.encoder = AttentionEncoding(in_features, past_steps, hidden_size, num_layers, dropout)
        self.decoder = AttentionDecoding(hidden_size, past_steps, input_features=2, output_size=1,
                                         num_layers=num_layers, dropout=dropout)

    def forward(self, input):
        x1, x2 = input
        encoded, hidden_encoded = self.encoder(x1)
        # We collect the outputs
        output_lst = []
        dec_hidden = hidden_encoded
        dec_out_recursive = torch.zeros((x1.size(0), 1, 2), requires_grad=False, device=x1.device)
        for k in range(self.future_steps):
            # First feature in F2 is the previous prediciton
            # Second feature in F2 is the known future at current step.
            dec_out_recursive[:, 0, 1] = (x2[:, k]).detach()
            dec_out, dec_hidden = self.decoder(dec_out_recursive, encoded, dec_hidden)
            output_lst.append(dec_out)
            # The gradient will not propagate back from the input of the next step
            dec_out_detached = dec_out.detach()
            # Write the current prediction for the next step
            dec_out_recursive[:, :, 0] = dec_out_detached
        output = torch.cat(output_lst, axis=1)
        output = output.squeeze(1)
        return output

def compute_model_param_count(model):
    """
    Computes the total number of *trainable* weights/params into a model.

    For some reason, default parameter count in pytorch skips some layers.

    """
    net_params = filter(lambda p: p.requires_grad, model.parameters())
    weight_count = 0
    for param in net_params:
        weight_count += np.prod(param.size())
    return weight_count

def print_model_weights_rec(model, crt_level=0, max_level=2):
    """
    Pretty prints a network structure showing the weights at each sub-level.

    """
    crt_sum = 0
    prefix = "      " * crt_level
    network_name = str(type(model))
    no_params = compute_model_param_count(model)
    print(f"{prefix}{network_name} with {no_params} parameters:")
    for cm in model.children():
        if crt_level < max_level:
            print_model_weights_rec(cm, crt_level+1, max_level)
    print(f"{prefix}Parameters: {no_params}\n{prefix}------------------")


"""
    Chapter 6: Paint Application
        Developing a Tiny Framework

Tkinter GUI Application Development Blueprints
"""
import tkinter as tk

class Framework():

    """
    GUIFramework is a class that provides a higher level of abstraction for
    the development of Tkinter graphic user interfaces (GUIs).
    Every class that uses this GUI framework must inherit from this class
    and should pass the root window as an argument to this class by calling
    the super method as follows:
        super().__init__(root)

    Building Menus:
    To build a menu, call build_menu() method with one argument for
    menu_definition, where menu_definition is a tuple where each item is a string of the
    format:
        'Top Level Menu Name - MenuItemName/Accelrator/Commandcallback/Underlinenumber'.

        MenuSeparator is denoted by a string 'sep'.

    For instance, passing this tuple as an argument to this method

        menu_definition = (
                      'File - &New/Ctrl+N/new_file, &Open/Ctrl+O/openfile, &Save/Ctrl+S/save, Save&As//saveas, sep, Exit/Alt+F4/close',
                      'Edit - Cut/Ctrl+X/cut, Copy/Ctrl+C/copy, Paste/Ctrl+V/paste, Sep',
                      )

    will generate a File and Edit Menu Buttons with listed menu items for each of the buttons.
    """
    menu_items = None

    def __init__(self, root):
        self.root = root

    def build_menu(self, menu_definitions):
        menu_bar = tk.Menu(self.root)
        for definition in menu_definitions:
            menu = tk.Menu(menu_bar, tearoff=0)
            top_level_menu, pull_down_menus = definition.split('-')
            menu_items = map(str.strip, pull_down_menus.split(','))
            for item in menu_items:
                self._add_menu_command(menu, item)
            menu_bar.add_cascade(label=top_level_menu, menu=menu)
        self.root.config(menu=menu_bar)

    def _add_menu_command(self, menu, item):
        if item == 'sep':
            menu.add_separator()
        else:
            menu_label, accelrator_key, command_callback = item.split('/')
            try:
                underline = menu_label.index('&')
                menu_label = menu_label.replace('&', '', 1)
            except ValueError:
                underline = None
            menu.add_command(label=menu_label, underline=underline,
                             accelerator=accelrator_key, command=eval(command_callback))

class TestThisFramework(Framework):

    def new_file(self):
        print('new tested OK')

    def open_file(self):
        print ('open tested OK')

    def undo(self):
        print ('undo tested OK')

    def options(self):
        print ('options tested OK')

    def about(self):
        print ('about tested OK')

if __name__ == '__main__':

    root = tk.Tk()
    menu_items = (
        'File- &New/Ctrl+N/self.new_file, &Open/Ctrl+O/self.open_file',
        'Edit- Undo/Ctrl+Z/self.undo, sep, Options/Ctrl+T/self.options',
        'About- About//self.about'
    )
    app = TestThisFramework(root)
    app.build_menu(menu_items)
    root.mainloop()


import re
import html
import jieba
import jieba.analyse
from sklearn.metrics.pairwise import cosine_similarity

class CosineSimilarity(object):

    #余弦相似度

    @staticmethod
    def extract_keyword(content):  # 提取关键词
        # 正则过滤 html 标签
        re_exp = re.compile(r'(<style>.*?</style>)|(<[^>]+>)', re.S)
        content = re_exp.sub(' ', content)
        # html 转义符实体化
        content = html.unescape(content)
        # 切割
        seg = [i for i in jieba.cut(content, cut_all=True) if i != '']
        # 提取关键词
        keywords = jieba.analyse.extract_tags("|".join(seg), topK=200, withWeight=False)
        return keywords

    @staticmethod
    def one_hot(dict, keywords):  # oneHot编码
        lens = len(dict)
        code = [0]*lens
        for word in keywords:
            code[dict[word]] += 1
        return code

    def __init__(self, x, y):
        self.s1 = x
        self.s2 = y

    def main(self):
        # 编码
        dict = {}
        i = 0
        # 去除停用词
        jieba.analyse.set_stop_words(r'C:\test\stopwords.txt')

        # 提取关键词
        keywords1 = self.extract_keyword(self.s1)
        keywords2 = self.extract_keyword(self.s2)
        # 词的并集
        union = set(keywords1).union(set(keywords2))
        for word in union:
            dict[word] = i
            i += 1
        # oneHot编码
        code1 = self.one_hot(dict, keywords1)
        code2 = self.one_hot(dict, keywords2)
        # 余弦相似度计算
        sample = [code1, code2]
        # 除零处理
        try:
            sim = cosine_similarity(sample)
            return sim[1][0]
        except Exception as e:
            print(e)
            return 0.0

            # 测试
if __name__ == '__main__':
    with open(r'C:\test\orig.txt', 'r',encoding='utf-8') as x, open(r'C:\test\orig_0.8_add.txt', 'r',encoding='utf-8') as y:
        content_x = x.read()
        content_y = y.read()
        similarity = CosineSimilarity(content_x, content_y)
        similarity = similarity.main()
        result = similarity*100
        print('相似度: %.1f%%' % result)
        f3 = open(r'C:\test\result.txt','w')
        f3.write('文本相似度:'+str(result)+'%')


name = "sai"
age = 16

# using expressions
print(f"my name is: {name} and age {age}")
print(f"my name is: {name:10} and age {age:10d}")

# str() , repr()
t = (1,"test")
print(str(t))

#
x = 65
# !a ascii !s str()   !r  repr()
print(f"my name is: {x!a} and age {age}")
# using format
print("{0} and {1}".format(name,age))

# create math table
table = 5
for x in range(1,11):
    print("{0:3d} X {1:3d} = {2:3d}".format(table, x, table*x))

#writing to file
with open('c:\\temp\\test.txt','w') as f:
    f.writelines(["first line","second line"])

# read to file
with open('c:\\temp\\test.txt','r') as f:
    for l in f.readlines():
        print(l)

# JSON
l = ["one", "two", "three"]
import json
print(json.dumps(l))
with open('c:\\temp\\test.txt','w') as f:
    f.write(json.dumps(l))


#Last Digit of the Sum of Squares of Fibonacci Numbers
#Compute the last digit of 𝐹2 0 + 𝐹2 1 + · · · + 𝐹2 𝑛.

n = int(input())
B=[] #Array of Fibonacci numbers
B.append(0)
B.append(1)

A=[] #Array of Fibonacci square sums
A.append(0)
A.append(1)

for i in range(2,60):
    b = (B[i-1]+B[i-2])%10
    B.append(b)
    A.append((A[i-1]+b**2)%10)
print(A[n%60])


import sys
sys.stdin = open('input.txt')
input = sys.stdin.readline
v, e = map(int, input().split())
edges = []
parent = dict()
rank = dict()
cnt = 0

for _ in range(e):
    a, b, w = map(int, input().split())
    edges.append([w,a,b])
edges.sort()
for i in range(1,v+1):
    parent[i] = i
    rank[i] = 0

def find(node):
    if parent[node] == node:
        return node
    return find(parent[node])

def union(v, u):
    v_p = find(v)
    u_p = find(u)
    if rank[v_p] > rank[u_p]:
        parent[u_p] = v_p
    else:
        parent[v_p] = u_p
        if rank[u_p] == rank[v_p]:
            rank[u_p] += 1

for edge in edges:
    w, v, u = edge
    if find(v) != find(u):
        union(v,u)
        cnt += w

print(cnt)


from django.urls import path
from . import views

app_name = 'productos'

urlpatterns = [
    path("", views.Listado, name="listado"),
    path("crear", views.CrearProducto, name="crearproducto"),
    path("<int:id_producto>/modificar", views.ModificarProducto, name="modificarproducto"),
    path("<int:id_producto>/eliminar", views.EliminarProducto, name="eliminarproducto"),
]


import torch
import torch.nn as nn

class conv_bn_re(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, is_re=True):
        super(conv_bn_re, self).__init__()
        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding)
        self.bn = nn.BatchNorm2d(out_channels)
        self.re = nn.ReLU()
        self.is_re = is_re

    def forward(self, x):
        if self.is_re:
            return self.re(self.bn(self.conv(x)))
        else:
            return self.bn(self.conv(x))

class basicblock(nn.Module):
    def __init__(self, in_channels, out_channels, stride, kernel_size=3, padding=1):
        super(basicblock, self).__init__()
        self.conv1 = conv_bn_re(in_channels, out_channels, kernel_size, stride, padding)
        self.conv2 = conv_bn_re(out_channels, out_channels, kernel_size, 1, padding, is_re=False)
        if stride == 2:
            self.downsample = conv_bn_re(in_channels, out_channels, kernel_size=1, stride=2, is_re=False)
        else:
            self.downsample = None
        self.re = nn.ReLU(inplace=True)

    def forward(self, x):
        y = self.conv1(x)
        y = self.conv2(y)
        if self.downsample:
            x = self.downsample(x)
        return self.re(y + x)

class SElayer(nn.Module):
    def __init__(self,channel,reduction=16):
        super(SElayer, self).__init__()
        self.pool = nn.AdaptiveAvgPool2d(1)
        self.func = nn.Sequential(
            nn.Conv2d(
                in_channels=channel,
                out_channels=int(channel/reduction)+1,
                kernel_size=1,
            ),
            nn.ReLU(inplace=True),
            nn.Conv2d(
                in_channels=int(channel/reduction)+1,
                out_channels=channel,
                kernel_size=1,
            ),
            nn.Sigmoid()
        )

    def forward(self,x):
        atten_x = self.pool(x)
        atten_x = self.func(atten_x).view(x.shape[0],-1,1,1)
        y = x * atten_x
        return y

class SpatialAttention(nn.Module):
    def __init__(self, kernel_size=7):
        super(SpatialAttention, self).__init__()

        assert kernel_size in (3, 7), 'kernel size must be 3 or 7'
        padding = 3 if kernel_size == 7 else 1

        self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        avg_out = torch.mean(x, dim=1, keepdim=True)
        max_out, _ = torch.max(x, dim=1, keepdim=True)
        x = torch.cat([avg_out, max_out], dim=1)
        x = self.conv1(x)
        return self.sigmoid(x)

class cbam(nn.Module):
    def __init__(self,channels):
        super(cbam, self).__init__()
        self.cattn = SElayer(channels)
        self.sattn = SpatialAttention()

    def forward(self,x):
        out = self.cattn(x)
        out = self.sattn(out) * out
        return out

class SCVfliter(nn.Module):
    def __init__(self, dict_len, embed_size, num_hiddens, num_layers, nclass):
        super(SCVfliter, self).__init__()
        self.emb = nn.Embedding(dict_len, embed_size)
        self.conv = nn.Sequential(
            nn.Conv2d(1, 16, 3, 1),
            basicblock(16, 32, 2, 3, 1),
            nn.MaxPool2d(1, 2),
            basicblock(32, 64, 2, 3, 1),
            cbam(64),
            basicblock(64, 128, 2, 3, 1),
            nn.MaxPool2d(1, 2),
            basicblock(128, 256, 2, 3, 1),
            cbam(256),
            basicblock(256, 512, 2, 3, 1),
            nn.MaxPool2d(1, 2),
            basicblock(512, 512, 1, 3, 1),
            cbam(512),
        )
        self.lstm = nn.LSTM(
            input_size=118,
            hidden_size=num_hiddens,
            num_layers=num_layers,
            bidirectional=True,
            batch_first=True,
            dropout=0.3
         )

        self.fc1 = nn.Linear(2 * num_hiddens, num_hiddens)
        self.re = nn.ReLU()
        self.fc2 = nn.Linear(num_hiddens, nclass)
        self.drop = nn.Dropout(0.2)

    def forward(self, x, return_layer=0):
        emb = self.emb(x)
        if return_layer == 1:
            return emb
        emb = self.conv(emb.unsqueeze(1))
        emb = emb.squeeze(dim=3)
        if return_layer == 2:
            return emb
        out, _ = self.lstm(emb)
        out = out.permute(1, 0, 2)
        encoding = out[-1]
        if return_layer == 3:
            return encoding
        fearures = self.re(self.drop(self.fc1(encoding)))
        outs = self.fc2(fearures)

        return outs

if __name__ == '__main__':
    x = torch.ones((2, 30100), dtype=int)
    net = SCVfliter(6, 20, 100, 2, 2)
    print(net(x).shape)


from pymongo import MongoClient
client = MongoClient('localhost', 27017)

db = client.test

posts = db.posts

post_data_1 = {
	'title': 'Python and MongoDB',
    'content': 'PyMongo is fun, you guys',
    'author': 'Scott'
}

post_data_2 = {
    'title': 'Virtual Environments',
    'content': 'Use virtual environments, you guys',
    'author': 'Scott'
}

post_data_3 = {
    'title': 'Learning Python',
    'content': 'Learn Python, it is easy',
    'author': 'Bill'
}

res = posts.insert(post_data_1)

print('The First Post: '.format(res.inserted_id))

new_res = posts.insert([post_data_1,post_data_2, post_data_3])

print('All of the posts: '.format(new_res.inserted_ids))


from py2neo import Graph
from py2neo import Node
from py2neo import NodeMatcher
from py2neo import Relationship

'''
创建节点
文件提供了创建节点的八个的函数
'''

graph = Graph("bolt: // localhost:7687", auth=("neo4j", "12345678"))

# 创建user的函数
def create_user(infos):
    for info in infos:
        # Uid，性别，年龄，邮箱
        user_node = Node('user', Uid=info[0], gender=info[1], age=info[2], email=info[3])
        graph.create(user_node)

# 创建application的函数
def create_application(infos):
    for info in infos:
        # Aid,名称，版本，开发者，简述，占用空间大小
        # 外键 databaseserver.DBSid
        application_node = Node('application', Aid=info[0], name=info[1], version=info[2], dev=info[3], profile=info[4],
                                size=info[5])
        graph.create(application_node)

        # 外键处理
        matcher = NodeMatcher(graph)
        databaseserver_node = matcher.match("databaseserver", DBSid=info[-1])
        if len(databaseserver_node) == 0:
            continue
        App_DBS = Relationship(application_node, 'App_DBS', databaseserver_node.first())
        graph.create(App_DBS)

# 创建category的函数
def create_category(infos):
    for info in infos:
        # Cid,名称,说明
        category_node = Node('category', Cid=info[0], name=info[1], profile=info[2])
        graph.create(category_node)

# 创建databaseserver的函数
def create_databaseserver(infos):
    for info in infos:
        # DBSid,名称,IP地址,DBMS类型,版本,处理器类型,内存容量
        databaseserver_node = Node('databaseserver', DBSid=info[0], name=info[1], IP=info[2], DB_class=info[3],
                                   version=info[4], pro_class=info[5], size=info[6])
        graph.create(databaseserver_node)

# 创建loadbalancer的函数
def create_loadbalancer(infos):
    for info in infos:
        # LBid,名称,IP地址,端口,负载分发算法,SSL加速,会话保持
        # 外键rack.Rid
        loadbalancer_node = Node('loadbalancer', LBid=info[0], name=info[1], IP=info[2], port=info[3], load=info[4],
                                 SSL=info[5], session=info[6])
        graph.create(loadbalancer_node)

        # 外键处理
        matcher = NodeMatcher(graph)
        rack_node = matcher.match("rack", Rid=info[-1])
        if len(rack_node) == 0:
            continue
        LdB_Rack = Relationship(loadbalancer_node, 'LdB_Rack', rack_node.first())
        graph.create(LdB_Rack)

# 创建virtualmachine的函数
def create_virtualmachine(infos):
    for info in infos:
        # VMid，名称，版本，访问控制，认证配置、容错设置
        virtualmachine_node = Node('virtualmachine', VMid=info[0], name=info[1], version=info[2], visitcontrol=info[3],
                                   auth=info[4], allowerror=info[5])
        graph.create(virtualmachine_node)

        # 外键处理
        matcher = NodeMatcher(graph)
        server_node = matcher.match("server", Sid=info[-1])
        if len(server_node) == 0:
            continue
        VM_Srv = Relationship(virtualmachine_node, 'VM_Srv', server_node.first())
        graph.create(VM_Srv)

# 创建rack的函数
def create_rack(infos):
    for info in infos:
        # Rid，名称，位置，容量
        rack_node = Node('rack', Rid=info[0], name=info[1], address=info[2], capacity=info[3])
        graph.create(rack_node)

# 创建server的函数
def create_server(infos):
    for info in infos:
        # Sid，名称，IP地址，操作系统，处理器类型、内存容量
        # rack.Rid
        server_node = Node('server', Sid=info[0], name=info[1], IP=info[2], operationsystem=info[3], processor=info[4],
                           memoryspace=info[5])
        graph.create(server_node)

        # 外键处理
        matcher = NodeMatcher(graph)
        rack_node = matcher.match("rack", Rid=info[-1])
        if len(rack_node) == 0:
            continue
        server_Rack = Relationship(server_node, 'server_Rack', rack_node.first())
        graph.create(server_Rack)


from typing import Dict, List, Callable, Tuple

from PySide6.QtGui import Qt
from PySide6.QtWidgets import QTableWidget, QTableWidgetItem

from api.registerdevice.models.device import Device
from gui.actions.models import (SyncTableWithRegistryParams,
                                TableWidgetCallbackIdentifiers,
                                BatteryLevelToNotify,
                                BatteryNotificationParams)
from gui.notifications import notify_device_charged

DEVICE_NAME_COLUMN = 0
BATTERY_LEVEL_COLUMN = 1
BATTERY_LEVEL_TO_NOTIFY_COLUMN = 2
_DEFAULT_BATTERY_LEVEL_TO_NOTIFY = 80

_FilteredDevicesInfo = Tuple[Dict[str, Device], List[int], List[int]]

def _apply_table_widget_item_styles(item: QTableWidgetItem):
    def _set_cell_as_immutable(item: QTableWidgetItem):
        item.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)

    _set_cell_as_immutable(item)
    item.setTextAlignment(Qt.AlignCenter)

def sync_shared_cache_with_gui(devices_table: QTableWidget,
                               sync_table_with_registry_params: SyncTableWithRegistryParams):
    (registered_devices_dict,
     updating_devices_info,
     evicting_devices_row_id) = _identify_evicting_and_registered_devices(devices_table,
                                                                          sync_table_with_registry_params)

    remove_related_entries: Callable[[int], None] = sync_table_with_registry_params.callbacks.get_callback(
        TableWidgetCallbackIdentifiers.REMOVE_MAP_ENTRIES_ASSOCIATED_TO_TABLE_ROW_ID)

    _remove_devices_from_table_and_related_maps(devices_table, evicting_devices_row_id, remove_related_entries)
    _update_devices_from_table_and_related_maps(devices_table, updating_devices_info, sync_table_with_registry_params)
    _insert_new_devices_to_table_and_related_maps(devices_table, registered_devices_dict,
                                                  sync_table_with_registry_params)

def _identify_evicting_and_registered_devices(devices_table: QTableWidget,
                                              sync_table_with_registry_params: SyncTableWithRegistryParams) -> _FilteredDevicesInfo:
    registered_devices_dict: Dict[str, Device] = {}
    updating_devices_info: List[int] = []
    evicting_devices_row_id: List[int] = []

    no_of_devices_in_table: int = devices_table.rowCount()

    device_registry = sync_table_with_registry_params.devices_registry
    table_id_device_map = sync_table_with_registry_params.table_id_device_map

    if no_of_devices_in_table <= 0:
        return registered_devices_dict, updating_devices_info, evicting_devices_row_id

    for i in range(no_of_devices_in_table):
        device_id: str = table_id_device_map[i]

        if device_id not in device_registry:
            evicting_devices_row_id.append(i)
            continue

        registered_devices_dict[device_id] = device_registry[device_id]

        device_in_registry: Device = device_registry[device_id]
        device_battery_level = _get_device_battery_level_from_table(devices_table, i)

        if device_battery_level == device_in_registry.battery_level:
            continue

        updating_devices_info.append(i)

    return registered_devices_dict, updating_devices_info, evicting_devices_row_id

def _remove_devices_from_table_and_related_maps(devices_table: QTableWidget,
                                                evicting_devices_row_id: List[int],
                                                remove_related_entries: Callable) -> None:
    for row_id in evicting_devices_row_id:
        _remove_device_from_table(devices_table, row_id)
        remove_related_entries(row_id)

def _update_devices_from_table_and_related_maps(devices_table: QTableWidget,
                                                updating_devices_info: List[int],
                                                sync_table_with_registry_params: SyncTableWithRegistryParams):
    devices_registry = sync_table_with_registry_params.devices_registry
    table_id_device_map = sync_table_with_registry_params.table_id_device_map

    for row_id in updating_devices_info:
        device_id: str = table_id_device_map[row_id]
        device_in_registry: Device = devices_registry[device_id]
        updated_battery_level: str = device_in_registry.battery_level

        _update_device_battery_in_table(devices_table, row_id, updated_battery_level)

def _insert_new_devices_to_table_and_related_maps(devices_table: QTableWidget,
                                                  registered_devices_dict: Dict[str, Device],
                                                  sync_table_with_registry_params: SyncTableWithRegistryParams):
    devices_registry: Dict[str, Device] = sync_table_with_registry_params.devices_registry
    new_devices: Dict[str, Device] = _identify_new_devices(registered_devices_dict, devices_registry)

    insert_related_entries: Callable[[int, str, BatteryLevelToNotify],
                                     None] = sync_table_with_registry_params.callbacks.get_callback(
        TableWidgetCallbackIdentifiers.MAP_TABLE_ROW_ID_TO_DEVICE
    )

    for device_id, device in new_devices.items():
        inserted_row_id: int = _insert_device_to_table(devices_table, device)
        new_battery_notification_status = BatteryLevelToNotify(_DEFAULT_BATTERY_LEVEL_TO_NOTIFY)

        insert_related_entries(inserted_row_id, device_id, new_battery_notification_status)

def _identify_new_devices(registered_devices_dict: Dict[str, Device], devices_registry: Dict[str, Device]) -> dict:
    registered_devices_set: set = set(registered_devices_dict.items())
    devices_registry_set: set = set(devices_registry.items())

    new_devices_set: set = devices_registry_set - registered_devices_set

    return dict(new_devices_set)

def _get_device_battery_level_from_table(devices_table: QTableWidget, row: int) -> str:
    return _get_item_from_table(devices_table, row, BATTERY_LEVEL_COLUMN)

def _get_device_name_from_table(devices_table: QTableWidget, row: int) -> str:
    return _get_item_from_table(devices_table, row, DEVICE_NAME_COLUMN)

def _get_item_from_table(devices_table: QTableWidget, row: int, column: int) -> str:
    return devices_table.item(row, column).text()

def _remove_device_from_table(devices_table: QTableWidget, row: int) -> None:
    devices_table.removeRow(row)

def _update_device_battery_in_table(devices_table, row: int, item: str):
    updated_item = QTableWidgetItem(item)
    _apply_table_widget_item_styles(updated_item)

    _update_device_in_table(devices_table, row, BATTERY_LEVEL_COLUMN, updated_item)

def _update_device_in_table(devices_table: QTableWidget, row: int, column: int, item: QTableWidgetItem):
    devices_table.setItem(row, column, item)

def _insert_device_to_table(devices_table: QTableWidget, device: Device) -> int:
    row_position = devices_table.rowCount()
    devices_table.insertRow(row_position)

    device_name_item: QTableWidgetItem = QTableWidgetItem(device.device_name)
    _apply_table_widget_item_styles(device_name_item)

    battery_level: QTableWidgetItem = QTableWidgetItem(device.battery_level)
    _apply_table_widget_item_styles(battery_level)

    battery_level_to_notify: QTableWidgetItem = QTableWidgetItem(str(_DEFAULT_BATTERY_LEVEL_TO_NOTIFY))
    _apply_table_widget_item_styles(battery_level_to_notify)

    _update_device_in_table(devices_table, row_position, DEVICE_NAME_COLUMN, device_name_item)
    _update_device_in_table(devices_table, row_position, BATTERY_LEVEL_COLUMN, battery_level)
    _update_device_in_table(devices_table, row_position, BATTERY_LEVEL_TO_NOTIFY_COLUMN, battery_level_to_notify)

    return row_position

def notify_device_battery_level_hits_threshold(devices_table: QTableWidget,
                                               battery_notification_params: BatteryNotificationParams):
    row_count: int = devices_table.rowCount()

    if row_count <= 0:
        return

    for i in range(row_count):
        device_id: str = battery_notification_params.table_row_id_device_map[i]
        device_battery_notification_status: BatteryLevelToNotify = \
            battery_notification_params.device_battery_level_to_notify_map[device_id]

        device_battery_level_in_table: int = int(_get_device_battery_level_from_table(devices_table, i))

        if (device_battery_level_in_table < device_battery_notification_status.battery_level_to_notify or
                device_battery_notification_status.has_notified):
            continue

        device_name: str = _get_device_name_from_table(devices_table, i)
        _send_message(device_id, device_name, battery_notification_params.interface_title)

        device_battery_notification_status.has_notified = True
        battery_notification_params.on_update_battery_notification_status(device_id, device_battery_notification_status)

def _send_message(device_id: str, device_name: str, interface_title: str):
    payload: str = _build_notification_payload(device_name)
    notify_device_charged(payload, interface_title)

    print(f'GUI: \tNotified about device {device_id}, {device_name}')

def _build_notification_payload(device_name: str) -> str:
    message: str = f'Device: {device_name} is charged. \n' \
                   'You can unplug it from the charger now.'

    return message


# [9, 12, 2, 4, 5]

# [9, 12, 17, 18, 2, 4]

def shifted_arr_search(shiftArr, num):
  n = len(shiftArr)

  def binaryPivot(shiftArr):
    l, r = 0, n-1
    while l < r:
      m = (l + r) // 2
      if shiftArr[m] > shiftArr[m+1]:
        return m
      elif shiftArr[r] < shiftArr[m]:
        l = m + 1
      else:
        r = m

    return -1

  def binarySearch(arr, l, r, num):
    while l <= r:
      m = (l + r) // 2
      print(l, r, m)
      if arr[m] == num:
        return m
      if arr[m] > num:
        r = m - 1
      else:
        l = m + 1

    return -1

  pivot = binaryPivot(shiftArr)
  # print(pivot)
  if pivot == -1:
    return binarySearch(shiftArr, 0, n-1, num)
  else:
    if shiftArr[0] <= num and shiftArr[pivot] >= num:
      return binarySearch(shiftArr, 0, pivot, num)
    else:
      return binarySearch(shiftArr, pivot+1, n-1, num)

print(shifted_arr_search([2], 2))
print(shifted_arr_search([9, 12, 17, 18, 2, 4], 2))


import ssl
from bs4 import BeautifulSoup
#import requests
from urllib.request import Request, urlopen

#note some site block the urllib library to protect from bot/spider so
# we use known browser agent like mozilla , chrome and so on.
# in this code too we need that so here is alterntive for that
req = Request(
     url= 'https://www.scrapethissite.com/pages/simple/',
     headers = {'User-Agent': 'Mozilla/5.0'}
     )

html = urlopen(req).read()
soup = BeautifulSoup(html,'html.parser')
all = soup('h3')

for tag in all:
    print(tag.text.strip())


n = input()
n = int(n)
'''
import math
for a in range(n):
    presentes = []
    peso = []
    for i in range(int(input())):
        presentes.append(input())
        peso.append(float(input()))
    capacidade = float(input())
    entry = input()
    n = int(input())
    carga = 0
    naoListados = []
    while entry != '-':
        flag = True
        for j in range(len(presentes)):
            if entry == presentes[j]:
                carga += n*peso[j]
                flag = False
        if flag:
            naoListados.append(entry)
        entry = input()
        n = int(input())
    for i in range(len(naoListados)):
        print("NAO LISTADO:", naoListados[i])
    print("Peso total:","{0:.2f}".format(carga),"kg")
    print("Numero de trenos:",math.ceil(carga/capacidade))
    print("")
'''


import numpy as np
import numpy.testing as npt
import pytest

from pulse2percept.utils import (center_image, shift_image, scale_image,
                                 trim_image)

def test_shift_image():
    img = np.zeros((5, 5), dtype=np.uint8)
    img[2, :] = 255
    # Top row:
    top = shift_image(img, 0, -2)
    npt.assert_almost_equal(top[0, :], 255)
    npt.assert_almost_equal(top[1:, :], 0)
    # Bottom row:
    bottom = shift_image(img, 0, 2)
    npt.assert_almost_equal(bottom[:4, :], 0)
    npt.assert_almost_equal(bottom[4, :], 255)
    # Bottom right pixel:
    bottom = shift_image(img, 4, 2)
    npt.assert_almost_equal(bottom[4, 4], 255)
    npt.assert_almost_equal(bottom[:4, :], 0)
    npt.assert_almost_equal(bottom[:, :4], 0)
    with pytest.raises(ValueError):
        shift_image(np.ones(10), 1, 1)
    with pytest.raises(ValueError):
        shift_image(np.ones((3, 4, 5, 6)), 1, 1)

def test_center_image():
    # Create a horizontal bar:
    img = np.zeros((5, 5), dtype=np.uint8)
    img[2, :] = 255
    # Center phosphene:
    npt.assert_almost_equal(img, center_image(img), decimal=3)
    npt.assert_almost_equal(img, center_image(shift_image(img, 0, 2)))
    with pytest.raises(ValueError):
        center_image(np.ones(10))
    with pytest.raises(ValueError):
        center_image(np.ones((3, 4, 5, 6)))

def test_scale_image():
    # Create a horizontal bar:
    img = np.zeros((5, 5), dtype=np.uint8)
    img[2, :] = 255
    # Scale phosphene:
    npt.assert_almost_equal(img, scale_image(img, 1))
    npt.assert_almost_equal(scale_image(img, 0.1).ravel()[12], 255)
    npt.assert_almost_equal(scale_image(img, 0.1).ravel()[:12], 0)
    npt.assert_almost_equal(scale_image(img, 0.1).ravel()[13:], 0)
    with pytest.raises(ValueError):
        scale_image(np.ones(10), 1)
    with pytest.raises(ValueError):
        scale_image(np.ones((3, 4, 5, 6)), 1)
    with pytest.raises(ValueError):
        scale_image(img, 0)

@pytest.mark.parametrize('n_channels', (1, 3))
def test_trim_image(n_channels):
    img = np.squeeze(np.zeros((13, 29, n_channels), dtype=float))
    shape = img.shape
    img[1:-1, 1:-1, ...] = 0.1
    img[2:-2, 2:-2, ...] = 0.2
    npt.assert_equal(trim_image(img).shape[:2], (shape[0] - 2, shape[1] - 2))
    npt.assert_equal(trim_image(img, tol=0.05).shape[:2],
                     (shape[0] - 2, shape[1] - 2))
    npt.assert_equal(trim_image(img, tol=0.1).shape[:2],
                     (shape[0] - 4, shape[1] - 4))
    npt.assert_equal(trim_image(img, tol=0.2).shape[:2], (1, 0))
    npt.assert_equal(trim_image(img, tol=0.1).shape[:2],
                     trim_image(trim_image(img), tol=0.1).shape[:2])
    trimmed, rows, cols = trim_image(img, return_coords=True)
    npt.assert_almost_equal(trimmed, trim_image(img))
    npt.assert_equal(rows, (1, 12))
    npt.assert_equal(cols, (1, 28))

    with pytest.raises(ValueError):
        trim_image(np.ones(10))
    with pytest.raises(ValueError):
        trim_image(np.ones((3, 4, 5, 6)))
    with pytest.raises(ValueError):
        trim_image(img, tol=-1)


import streamlit as st
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import sqlite3

df = pd.read_csv('data/data.csv')

def data_transform(data):
	meses = {
    'enero': 'January',
    'febrero': 'February',
    'marzo': 'March',
    'abril': 'April',
    'mayo': 'May',
    'junio': 'June',
    'julio': 'July',
    'agosto': 'August',
    'septiembre': 'September',
    'octubre': 'October',
    'noviembre': 'November',
    'diciembre': 'December'}
	data['Fecha_Update'] = data['Fecha_Update'].apply(lambda x :x.split(',')[1].strip())
	data['Fecha_Update'] = data['Fecha_Update'].replace(meses, regex=True)
	data['Fecha_Update'] = pd.to_datetime(data['Fecha_Update'] , format='%d de %B de %Y')
	data['Day'] = data['Fecha_Update'].dt.day_name()
	data.loc[(data.Estatus.isnull())&(data.TXN>0),'Estatus'] = 'TRANSACTION'
	data.loc[(data.Estatus.isnull())&(data.TXN==0),'Estatus'] = 'NO_RESPONSE'

	return data

def graph_cum(data):
	data_grouped_month = data.loc[(data.Estatus== 'APPROVED'),:].groupby(pd.Grouper(key='Fecha_Update', freq='M'))['Numero_cliente'].count().reset_index()
	data_grouped_month['conteo_acumulativo'] = data_grouped_month['Numero_cliente'].cumsum()

	fig, ax = plt.subplots(figsize=(10, 6))
	sns.lineplot(x='Fecha_Update', y='conteo_acumulativo', data=data_grouped_month, linewidth=2.5, color='blue', ax=ax)
	ax.set_title('Users with credit card', fontsize=12)
	ax.set_xlabel('Fecha', fontsize=14)
	ax.set_ylabel('Valor', fontsize=14)
	ax.legend(['Cummulative count'], loc='upper left', fontsize=12)

	return fig

def graph_pie(data):
	data_grouped_motivo = data.loc[(df.Estatus== 'APPROVED'),:].groupby('Motivo')['Numero_cliente'].count().reset_index()
	fig, ax = plt.subplots(figsize=(10, 6))
	ax.pie(data_grouped_motivo['Numero_cliente'], labels=data_grouped_motivo['Motivo'], autopct='%1.1f%%', startangle=90)
	ax.axis('equal')
	plt.title('Credit card Type', fontsize=12)

	return fig

def graph_bar_c(data):
	data = pd.DataFrame(pd.pivot_table(data.loc[df.Estatus.isin(['RESPONSE','NO_RESPONSE'])], values = 'Id_product',index='Estatus', columns='canal_venta', aggfunc='count', margins = True))
	data.reset_index(inplace = True)
	ind = np.arange(2)
	width = 0.09
	fig = plt.figure(figsize=(10,6)); ax = plt.axes()
	ax.bar(ind+ 0.00, data.Marketing[0:2].values, width, color='r') # barra roja.
	ax.bar(ind+ 0.15, data.Operaciones[0:2].values, width, color='b') # barra azul
	ax.bar(ind+ 0.30, data['Servicio al cliente'][0:2].values, width, color='g') # barra azul
	ax.set_ylabel('Clients');
	ax.set_title('Conversion')
	ax.set_xticks(ind)
	ax.set_xticklabels(['NO_RESPONSE', 'RESPONSE'])
	ax.legend(labels=['Marketing', 'Operaciones', 'Servicio al cliente'])

	return fig

def graph_den(data):
	fig = plt.figure(figsize=(10,6))
	sns.set_style('whitegrid')
	sns.kdeplot(data=data.loc[(data.Estatus== 'TRANSACTION')&(data.TXN<27000),:], x='TXN', shade=True)
	return fig

def trx_week(data):
	fig, ax = plt.subplots(figsize=(10, 6))
	sns.countplot(x='Day', data=data.loc[(data.Estatus== 'TRANSACTION'),:])
	ax.set_ylabel('Transctions')
	ax.set_title('Frequency of transactions by day of the week')

	return fig

def create_clients_table(data):
	df_clientes_b0 = pd.DataFrame(pd.pivot_table(data.loc[(data.Estatus== 'TRANSACTION')], values = ['TXN','Id_product'],index='Numero_cliente', aggfunc={'TXN':np.mean, 'Id_product':'count'}))
	df_clientes_b0.reset_index(inplace = True)
	df_clientes_b1 = pd.DataFrame(pd.pivot_table(data.loc[(data.Estatus== 'APPROVED')], values = ['Importe','Tasa_Interes','CAT'],index='Numero_cliente', aggfunc={'Importe':np.max, 'Tasa_Interes':np.max, 'CAT':np.max}))
	df_clientes_b1.reset_index(inplace = True)
	df_clientes_b2 = pd.DataFrame(pd.pivot_table(data.loc[(data.Estatus== 'DELIVERED')], values = ['CP', 'Puntuacion'],index='Numero_cliente', aggfunc={'CP':np.max, 'Puntuacion':np.max}))
	df_clientes_b2.reset_index(inplace = True)
	df_clientes_b3 = data.loc[(data.Estatus== 'APPROVED'),['Numero_cliente' ,'Motivo']]
	df_merged = pd.merge(df_clientes_b1, df_clientes_b0, on='Numero_cliente', how='left')
	df_merged_2 = pd.merge(df_clientes_b3, df_clientes_b2, on='Numero_cliente', how='left')
	df_clients = pd.merge(df_merged, df_merged_2, on='Numero_cliente', how='left')
	df_clients.rename(columns = {'Numero_cliente':'CUSTOMER_ID', 'Importe':'AMOUNT_CREDIT_GRANTED', 'Id_product': 'TRANSACTIONS', 'TXN':'MEAN_AMOUNT_TRANSACTION', 'Puntuacion':'DELIVERY_SCORE','Motivo':'CARD_TYPE'}, inplace = True)
	df_clients.DELIVERY_SCORE = df_clients.DELIVERY_SCORE.astype(float)
	return df_clients

def metrics_clients(data):
    mean_transactions = np.round(data.TRANSACTIONS.mean(),2)
    mean_transactions_amount = np.round(data.MEAN_AMOUNT_TRANSACTION.mean(),2)
    mean_delivery_score = np.round(data.DELIVERY_SCORE.mean(),2)
    mean_amount_g = np.round(data.AMOUNT_CREDIT_GRANTED.mean(),2)
    metrics = pd.DataFrame({'METRIC': ['MEAN_TRANSACTIONS', 'MEAN_TRANSACTIONS_AMOUNT', 'MEAN_DELIVERY_SCORE', 'MEAN_AMOUNT_CREDIT_GRANTED'],
                  'VALUE': [mean_transactions, mean_transactions_amount, mean_delivery_score, mean_amount_g]})
    return metrics

st.set_page_config(page_title='Business Challenge')

nav = st.sidebar.selectbox("Menu",['Introduction', 'KPI´s and Data analysis'])

if nav == 'Introduction':

	st.subheader('Business Challenge Credit Cards')
	col1 , col2, col3 = st.columns([3,1,1])
	col1.markdown('Iván Serrano Zapata')
	col2.image('images/python.png',width = 50, use_column_width = False)
	col3.image('images/st.png',width = 80, use_column_width = False)
	col1.subheader('')

	with st.expander("Requirements and objectives of the challenge"):
	    	st.write("""

        	1. Define and present KPI's for different stakeholders that help monitor product performance, describe the current state and facilitate future strategy

        	2. Generate an understanding data and translates into business knowledge

        	3. Generate data tables that can be used for the consumption of different business areas based on their needs

        	""")

	with st.expander("Brief explanation"):
	    	st.write("""
	    		1. The dashboard is developing in [Streamlit](https://streamlit.io/),  open-source Python library used for building data science web applications. It allows you to create interactive web apps directly from Python scripts.

	    		2. For the data manipulation part i used python and conventional libraries for manipulation and visualization.

	    		3. In this [repository](https://github.com/iserranoz) you can review all the notebooks and scripts, in this application i will only show the results.

	    		""")

	with st.expander('Summary of the data'):

		st.markdown('This database contains credit card information and transactions from multiple customers')
		st.table(pd.read_csv('data/info.csv'))

if nav == 'KPI´s and Data analysis':

	df = pd.read_csv('data/data.csv')
	df = data_transform(df)
	rate_conv = '{:.2f}%'.format(100*np.round(df.loc[df.Estatus.isin(['RESPONSE']),'Numero_cliente'].count()/df.loc[df.Estatus.isin(['RESPONSE', 'NO_RESPONSE']),'Numero_cliente'].count(),4))
	rate_apro = '{:.2f}%'.format(100*np.round(df.loc[df.Estatus.isin(['APPROVED']),'Numero_cliente'].count()/df.loc[df.Estatus.isin(['RESPONSE']),'Numero_cliente'].count(),4))
	rate_f = '{:.2f}%'.format(100*np.round(df.loc[df.Estatus.isin(['APPROVED']),'Numero_cliente'].count()/df.loc[df.Estatus.isin(['RESPONSE', 'NO_RESPONSE']),'Numero_cliente'].count(),4))
	df_clients = create_clients_table(df)
	mean_amount_rx = data=df.loc[(df.Estatus== 'TRANSACTION')&(df.TXN<27000),'TXN'].mean()
	client_metrics = metrics_clients(df_clients)
	st.subheader('KPI´s and Data analysis')
	st.subheader('')
	st.subheader('')
	with st.expander("Customer acquisition"):
		st.subheader('')

		col1, col2 = st.columns([4,2])
		with col1:
			st.pyplot(graph_cum(df))
			st.subheader('')
			st.subheader('')
			st.pyplot(graph_pie(df))
			st.subheader('')
			st.subheader('')
			st.pyplot(graph_bar_c(df))
			st.subheader('')
			st.subheader('')

		with col2:
			st.write("""There was an accelerated growth of approved clients in the last two months of 2019, but it has slowed down considerably afterwards.""")
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.write("""Our clients prefer the physical card, but the penetration of the digital card is good.""")
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.write(f"""The conversion ratio of the campaigns is {rate_conv}, which means that almost 7 out of 10 customers
				are interested in purchasing our product. This ratio seems to be stable in the different sales channels.
				Of that percentage, {rate_apro} are approved to have a card. Our final acquisition ratio is {rate_f}, this means that for every 10 customers we contact, around 4 get a card.""")

	with st.expander("Customer behavior"):
		st.subheader('')

		col1, col2 = st.columns([4,2])
		with col1:
			st.pyplot(graph_den(df))
			st.subheader('')
			st.subheader('')
			st.pyplot(trx_week(df))
			st.subheader('')
			st.subheader('')
			st.table(df_clients.head(10))
			st.table(client_metrics)
			st.subheader('')

		with col2:
			st.write(f"""This graph shows the distribution of transaction amounts, the average transaction amount is ${np.round(mean_amount_rx,2)}.""")
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.write("""This shows the frequency of use of the card on the days of the week, it seems that there is no type of preference in the days of card use.""")
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.write(f"""This table is very important, it groups
				the main characteristics of our users, number of transactions made, average amount of transactions, zip code,
				type of card, interest rate and delivery score..""")
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.subheader('')
			st.write(f"""These metrics describe the behavior of our users, and can give us an idea of the type of customer that purchases the card""")

	with st.expander("Additional KPI´s proposals"):
		st.subheader('')
		st.markdown('Performance metrics:')
		st.write("""

        	1. New customers ratio: the percentage increase in new card users versus the previous month

        	2. Campaign Response Rate: The percentage of users who respond to marketing campaigns

        	3. Active Users: Percentage of users who used the credit card versus the total number of users in a month

        	4. Average delivery time: average time it takes for a card to be delivered

        	5. Usage recurrence: the average number of transactions a user makes in X time

        	These metrics tend to perform better when based on some type of customer segmentation, which gives us information from different populations.""")


from collections import deque

def solution(people, limit):
    people = deque(sorted(people))
    answer = 0
    while people:
        right = people.pop()
        if len(people) != 0 and people[0] + right <= limit:
            people.popleft()
        answer += 1
    return answer


"""Utilities for formatting and printing data."""

import os
import random

def usd(n):
    """Convert a number into a USD currency string."""
    return "${:.2f}".format(n)

def float_of_usd(n):
    """Convert a USD currency string into a number."""
    datum = n.replace(",", "")
    if datum[0:1] == "$":
        datum = datum[1:]
    return float(datum)

def random_price(min_price, max_price):
    """Pick a random price between a min and a max."""
    selection = random.uniform(float(min_price), float(max_price))
    return usd(selection)

def pad(value, width):
    """Pads a value with spaces to make the string a certain width."""
    value_width = len(str(value))
    if value_width > width:
        return value
    else:
        padding = width - value_width
        return "{}{}".format(value, " " * padding)

def table(data):
    """Print a dictionary as a table."""
    if len(data) == 0:
        return ""
    sample = data[0]
    headers = sample.keys()
    widths = {k: 0 for k in headers}
    for header in headers:
          best = widths[header]
          size = len(header)
          if size > best:
              widths[header] = size
    for row in data:
        for k, v in row.items():
            best = widths[k]
            size = len(str(v))
            if size > best:
                widths[k] = size
    out = []
    inner_rule = "+".join(["-" * (wd + 2) for wd in widths.values() ])
    rule = "+{}+".format(inner_rule)
    inner_header = " | ".join(
        [pad(k.upper(), widths[k]) for k in headers])
    header = "| {} |".format(inner_header)
    out.append(rule)
    out.append(header)
    out.append(rule)
    for row in data:
        inner_content = " | ".join(
                [pad(v, widths[k]) for k, v in row.items()])
        content = "| {} |".format(inner_content)
        out.append(content)
        out.append(rule)
    return os.linesep.join(out)


class A:
    def __init__(self, a= 0, b= 3):
        self.a = a
        self.b = b
    def __repr__(self):
        return '{},{}'.format(self.a, self.b)

a = A(5, 3)
b = A(3, 7)
l =[a ,b]
l1 = sorted(l, key =lambda t: t.a)
print(l1)


import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches

def bilinear_upsampler(tensor, new_shape):
	return tf.image.resize(tensor, new_shape)

#Reference : https://github.com/tadax/m2det/blob/master/utils/generate_priors.py
def generate_anchors(num_scales=3, anchor_scale=2.0, image_size=320, shapes=[40, 20, 10, 5, 3, 1]):
	anchor_configs = {}
	for shape in shapes:
		anchor_configs[shape] = []
		for scale_octave in range(num_scales):
			for aspect_ratio in [(1, 1), (1.41, 0.71), (0.71, 1.41)]:
				anchor_configs[shape].append(
					(
						image_size / shape,
						scale_octave / float(num_scales),
						aspect_ratio
					)
				)

	boxes_all = []
	for _, configs in anchor_configs.items():
		boxes_level = []
		for config in configs:
			stride, octave_scale, aspect = config
			base_anchor_size = anchor_scale * stride * (2 ** octave_scale)
			anchor_size_x_2 = base_anchor_size * aspect[0] / 2.0
			anchor_size_y_2 = base_anchor_size * aspect[1] / 2.0
			x = np.arange(stride / 2, image_size, stride)
			y = np.arange(stride / 2, image_size, stride)
			xv, yv = np.meshgrid(x, y)
			xv = xv.reshape(-1)
			yv = yv.reshape(-1)
			boxes = np.vstack(
				(
					yv - anchor_size_y_2, xv - anchor_size_x_2,
					yv + anchor_size_y_2, xv + anchor_size_x_2)
				)
			boxes = np.swapaxes(boxes, 0, 1)
			boxes_level.append(np.expand_dims(boxes, axis=1))
		boxes_level = np.concatenate(boxes_level, axis=1)
		boxes_level /= image_size
		boxes_all.append(boxes_level.reshape([-1, 4]))

	anchor_boxes = np.vstack(boxes_all)

	return anchor_boxes

def clip_boxes(boxes, img_shape=(320, 320)):
	"""Clip boxes to image boundaries."""
	# x1 >= 0
	boxes[:, 0] = np.maximum(boxes[:, 0], 0)
	# y1 >= 0
	boxes[:, 1] = np.maximum(boxes[:, 1], 0)
	# x2 < im_width
	boxes[:, 2] = np.minimum(boxes[:, 2], img_shape[0] - 1)
	# y2 < img_height
	boxes[:, 3] = np.minimum(boxes[:, 3], img_shape[1] - 1)
	return boxes

def iou(anchors, boxes):
	if len(boxes.shape) == 2:
		boxes = boxes[0,:]
	if len(anchors.shape) == 1:
		anchors = anchors.reshape(1,-1)
	x1 = np.maximum(anchors[:, 0], boxes[0])
	y1 = np.maximum(anchors[:, 1], boxes[1])
	x2 = np.minimum(anchors[:, 2], boxes[2])
	y2 = np.minimum(anchors[:, 3], boxes[3])

	width = (x2 - x1)
	height = (y2 - y1)
	area_overlap = np.maximum(width*height, 0)
	l = (anchors[:, 2] - anchors[:, 0])
	b = (anchors[:, 3] - anchors[:, 1])
	area_an = l * b
	l = (boxes[2] - boxes[0])
	b = (boxes[3] - boxes[1])
	area_bx = l * b
	union = np.maximum((area_an + area_bx) - area_overlap,0)
	iou = area_overlap / (union)

	#setting iou to 0 which doesn't have overlap with ground truth
	iou[iou == np.inf] = 0
	iou[iou == -np.inf] = 0
	iou[anchors[:,2] < boxes[0]] = 0
	iou[anchors[:,0] > boxes[2]] = 0
	iou[anchors[:,3] < boxes[1]] = 0
	iou[anchors[:,1] > boxes[3]] = 0

	return iou

def nms(dets, confidence, thresh=0.4):
	outputs = []
	while len(dets) > 0:
		ix = np.argmax(confidence)
		result = dets[ix]
		outputs.append(result)
		dets = np.delete(dets, ix, axis=0)
		box1 = np.expand_dims(result, axis=0)
		to_delete = []
		for jx in range(len(dets)):
			box2 = np.expand_dims(dets[jx], axis=0)
			iou_ = iou(box1, box2)
			if iou_ >= thresh:
				to_delete.append(jx)
		for jx in to_delete[::-1]:
			dets = np.delete(dets, jx, axis=0)
	return np.array(outputs, dtype=np.float32)

def visualize(img, boxes):
	if len(boxes.shape) == 1:
		boxes = boxes.reshape(1,-1)
	fig, ax = plt.subplots(1)
	ax.imshow(img)
	boxes_xy_wh = boxes
	boxes_xy_wh[:, 0] = boxes[:, 0]
	boxes_xy_wh[:, 1] = boxes[:, 1]
	boxes_xy_wh[:, 2] = boxes[:, 2] - boxes[:, 0]
	boxes_xy_wh[:, 3] = boxes[:, 3] - boxes[:, 1]
	for i in range(len(boxes_xy_wh)):
		k = 0
		if i == 0:
			ec = 'r'
		else:
			ec = 'g'
		s = patches.Rectangle(
			(boxes_xy_wh[i][k], boxes_xy_wh[i][k+1]),
			boxes_xy_wh[i][k+2], boxes_xy_wh[i][k+3],
			linewidth=1, edgecolor=ec,
			facecolor="none"
			)
		ax.add_patch(s)
	plt.show()

if __name__ == "__main__":
	gt_box = np.array([[0, 0, 99, 99]])
	boxes = np.array(
		[[0, 0, 74, 74],
		[0, 0, 24, 24],
		[0, 0, 70, 70],
		[0, 0, 50, 50],
		[10, 20, 60, 70]]
		)
	ans = np.array(
		[0.5, 0.062, 0.5, 0.25]
		)
	iou_ = iou(boxes, gt_box)
	check = iou_[:4]/ans
	print (iou_)
	print("iou cases %s"%(np.greater(check, 0.9)))
	nms_box = nms(boxes, iou_)
	print ("nms cases are %s"%(np.allclose(nms_box, boxes[:2, :])))