## Get error rate.
import pandas as pd
import os
import sys

list_files_names = []
list_y_true = []
list_y_pred = []
list_error_rate = []
list_ASA_error_rate = []
dice_coef_list = []
TP_list = []
FN_list = []
FP_list = []
precision_list = []
recall_list = []
f1_score_list = []
errorRateBasedOnConfusionMatrix = []
accuracy = []

path2Experiment = '../test_exp'
pathToManual = r'../data/Annotation_disector_countFiles/countFiles'

foldNames = ['Iteration_1','Iteration_2','Iteration_3','Iteration_4','Iteration_5','Iteration_6','Iteration_7']
for foldName in foldNames:
    #MiceNames = foldName.split('_')[1:]
    MiceNames = [foldName]
    for mouseName in MiceNames:
        mouseName = mouseName+'_'
        pathToPredictions1 = os.path.join(path2Experiment,foldName,'PredictedMasks2','predMasks','OldPostprocessing_0.5')
        pathToResults = path2Experiment
        pathToPredictions = pathToPredictions1
        def getDiceCoef(diceCoef):
            dice_mean = 0.0
            zeros = 0
            #print(diceCoef['dice'].values)
            for item in diceCoef['dice'].values:
                if item == 0:
                    zeros = zeros + 1
                    continue
                else:
                    dice_mean += item
            dice_mean = dice_mean / (diceCoef.shape[0] - zeros)
            return dice_mean
        def processConfMatrix(df):
            df.columns =['Name','TP','FN','FP']
            df['TP'] = [int(item.split(',')[0].split('  ')[1]) for item in df['TP'].tolist()]
            df['FN'] = [int(item.split(',')[0].split(' ')[2]) for item in df['FN'].tolist()]
            df['FP'] = [int(item.split('  ')[2]) for item in df['FP'].tolist()]
            return df
        folderName = 'Error_analysis_final4'
        if not os.path.isdir(os.path.join(pathToPredictions1,folderName)):
            os.mkdir(os.path.join(pathToPredictions1,folderName))

        for file in os.listdir(pathToPredictions):
            print(file)
            if file.endswith('_count.txt'):
                print('Now serving {}'.format(file))
                fileparts = file.split('_count')
                diceCoef = pd.read_csv(os.path.join(pathToPredictions,fileparts[0]+'_diceCoef.txt'),sep="\t",header=None)
                confMatrix = pd.read_csv(os.path.join(pathToPredictions,fileparts[0]+'_confusionMatrix.txt'),sep="\t",header=None)
                Predicted = pd.read_csv(os.path.join(pathToPredictions,file), sep="\t", header=None)
                #print('dice shape is {}  confMatrix shape is {}  Predicted shape is {}'.format(diceCoef.shape,confMatrix.shape,Predicted.shape))
                Predicted.columns = ['Name', 'Count_predicted']
                diceCoef.columns = ['Name', 'dice']

                limit = Predicted.shape[0]
                print('limit is {}'.format(limit))
                loc = limit   # for lines that has total info
                #dice = diceCoef.iloc[loc,:][1]
                dice_mean = getDiceCoef(diceCoef)
                df_confusion = processConfMatrix(confMatrix)
                TP = df_confusion['TP'].sum()
                FN = df_confusion['FN'].sum()
                FP = df_confusion['FP'].sum()

                Manual = pd.read_csv(os.path.join(pathToManual,'FullDataSetNeuNSingleStainManualAnnotationCount_v2_correctFinalSep6th2018.csv'),sep=",") #FullDataSetNeuNSingleStainManualAnnotationCount_v2_Sep6th2018
                ASA = pd.read_csv(os.path.join(pathToManual,'ASA_count_final_Sep_4_2018.csv'),sep=",")

                print('Predicted shape is {}'.format(Predicted.shape))

                Predicted = Predicted.iloc[0:limit,:]
                diceCoef = diceCoef.iloc[0:limit,:]

                newNames = []
                for item in Predicted['Name'].values:
                    itemParts = item.split('_pred')
                    newNames.append(itemParts[0])
                Predicted['Name']= newNames
                #dice_mean = 0.0
                zeros = 0

                Manual_list = Manual['Name'].values
                Predicted_list= Predicted['Name'].values
                #print(Manual_list)
                #print(Predicted_list)
                intersection = [value for value in Manual_list if value in Predicted_list]
                #print('length of intersection is {}'.format(len(intersection)))
                ## Joining two dataframes.
                #Manual.join(Predicted,on=['Name'],how='inner')
                #result = pd.merge(Manual, Predicted, on='Name',how='inner')
                result = pd.merge(Manual, Predicted, on ='Name', how = 'inner')
                ASA_result = pd.merge(ASA,Predicted,on ='Name', how = 'inner')

                #print(result.head())
                #print(ASA_result.head())
                fractionater  = pd.DataFrame()
                Manual_Unet_ASA = pd.merge(result,ASA_result, on='Name', how='inner')

                #print(Manual_Unet_ASA.head())
                Manual_Unet_ASA['LU'] = Manual_Unet_ASA['Name'].str.split('_').str[0]
                Manual_Unet_ASA['Section'] = Manual_Unet_ASA['Name'].str.split('_').str[1]
                Manual_Unet_ASA['Stack'] = Manual_Unet_ASA['Name'].str.split('_').str[2]
                fractionater['LU'] = Manual_Unet_ASA['LU']
                fractionater['Section'] = Manual_Unet_ASA['Section']
                fractionater['Manual_count'] = Manual_Unet_ASA['Count_x']
                fractionater['Unet_count'] = Manual_Unet_ASA['Count_predicted_x']
                fractionater['ASA_count'] = Manual_Unet_ASA['Count_y']

                sf = fractionater.groupby('LU')['Unet_count'].sum()
                sf1 = fractionater.groupby('LU')['Manual_count'].sum()
                sf2 = fractionater.groupby('LU')['ASA_count'].sum()
                fractionater_total = pd.DataFrame({'LU':sf.index, 'Unet_count':sf.values,'Manual_count':sf1.values,'ASA_count':sf2.values})
                #print(fractionater_total.shape)
                #print(fractionater_total.head())
                fractionater_total.to_csv(os.path.join(pathToPredictions1,folderName,fileparts[0]+'_PerCaseCount_newCount.csv'),sep=',',index=False)

                result.to_csv(os.path.join(pathToPredictions1,folderName,fileparts[0]+'_oldCount.csv'),index=False)
                #result['LU'] = Manual_Unet_ASA['Name'].str.split('_').str[0]
                #result['Section'] = Manual_Unet_ASA['Name'].str.split('_').str[1]
                sf3 = fractionater.groupby(['LU','Section'])['Manual_count'].sum()
                sf4 = fractionater.groupby(['LU','Section'])['Unet_count'].sum()
                sf5 = fractionater.groupby(['LU','Section'])['ASA_count'].sum()
                index = [i[0]+'_'+i[1] for i in sf3.index]
                print(index)
                Count_per_section = pd.DataFrame({'LU_Section':index, 'Manual_count':sf3.values,'Unet_count':sf4.values,'ASA_count':sf5.values})

                #print(result.head())
                #print(Count_per_section.head())
                Count_per_section[['LU_Section','Manual_count','Unet_count','ASA_count']].to_csv(os.path.join(pathToPredictions1,folderName,fileparts[0]+'_PerSectionCount_newCount.csv'),sep=',',index=False)
                #Count_per_section[['LU_Section','Manual_count','Unet_count','ASA_count']].to_csv(os.path.join(pathToPredictions1,'_PerSectionCount.csv'),sep=',',index=False)
                #print(result)
                count_true = result['Count'].values
                count_pred = result['Count_predicted'].values
                count_ASA = ASA_result['Count'].values
                total_true = result['Count'].sum()
                total_pred = result['Count_predicted'].sum()
                error_rate = (abs(total_true - total_pred)/float(total_true))*100
                #print(ASA_result.head())
                #sys.exit()
                ASA_error_rate = (abs(total_true - ASA_result['Count'].sum())/float(total_true))*100
                #print('total_true {} , SUM {} error rate {}'.format(total_true,ASA_result['Count'].sum(),ASA_error_rate))
                #sys.exit()
                TP = int(TP)
                FP = int(FP)
                FN = int(FN)

                FN = abs((TP+FN) - total_true) + FN
                #assert(FN+TP == total_true,'Error in FN count\n')
                #print('new FN is {}'.format(FN))
                precision = TP/ float(TP + FP)
                recall = TP / float(TP + FN)
                #print('temperary remember to remove')
                #precision = 1
                #recall = 1
                f1_score = 2 *precision*recall/float(precision+recall)
                print('Dice coef is {}'.format(dice_mean))
                print('total_true is {}'.format(total_true))
                print('total_pred is {}'.format(total_pred))
                print('total ASA count is {}'.format(ASA_result['Count'].sum()))
                print('Error rate is {}'.format(error_rate))
                print('precision is {}'.format(precision))
                print('recall is {}'.format(recall))
                print('f1_score is {}'.format(f1_score))
                print('ASA error rate {}'.format(ASA_error_rate))
                print('TP {}'.format(TP))
                print('FP {}'.format(FP))
                print('FN {}'.format(FN))
                print('Error new based on confusion Matrix {}'.format((FP+FN)/float(total_pred)))
                print('Accuracy {}'.format(1- ((FP+FN)/float(total_pred))))
                #sys.exit()
                print('--'*30)
                list_files_names.append(mouseName)
                list_y_true.append(total_true)
                list_y_pred.append(total_pred)
                list_error_rate.append(error_rate)
                list_ASA_error_rate.append(ASA_error_rate)
                dice_coef_list.append(dice_mean)
                TP_list.append(TP)
                FN_list.append(FN)
                FP_list.append(FP)
                precision_list.append(precision)
                recall_list.append(recall)
                f1_score_list.append(f1_score)
final = pd.DataFrame()
final['fold'] = list_files_names
final['y_true'] = list_y_true
final['y_pred'] = list_y_pred
final['error_rate(%)'] = list_error_rate
final['ASA_error_rate(%)'] = list_ASA_error_rate
final['diceCoef'] = dice_coef_list
final['TP'] = TP_list
final['FN'] = FN_list
final['FP'] = FP_list
final['precision'] = precision_list
final['recall'] = recall_list
final['f1_score'] = f1_score_list
final2 = pd.DataFrame()

final2['fold']= ['Avg/Sum']
final2['y_true'] = [final['y_true'].sum()]
final2['y_pred'] = [final['y_pred'].sum()]
final2['error_rate(%)'] = [final['error_rate(%)'].mean()]
final2['ASA_error_rate(%)'] = [final['ASA_error_rate(%)'].mean()]
final2['diceCoef'] = [final['diceCoef'].mean()]
final2['TP'] = [final['TP'].sum()]
final2['FN'] = [final['FN'].sum()]
final2['FP'] = [final['FP'].sum()]
final2['precision'] = [final['precision'].mean()]
final2['recall'] = [final['recall'].mean()]
final2['f1_score'] = [final['f1_score'].mean()]

#parentFolder = os.path.dirname(pathToPredictions1)
if not os.path.exists(os.path.join(pathToResults,"Error_analysis")):
    os.makedirs(os.path.join(pathToResults,"Error_analysis"))
#print(pathToResults)
final.to_csv(os.path.join(pathToResults,"Error_analysis",'error.csv'),sep=',',index=False)
final2.to_csv(os.path.join(pathToResults,"Error_analysis",'errorAvg.csv'),sep=',',index=False)
print('##'*30)
print('Average DL error of folds is {}'.format(final['error_rate(%)'].mean()))
print('True count is {}'.format(final['y_true'].sum()))
print('DL predicted count is {}'.format(final['y_pred'].sum()))
print('DL Std of folds is {}'.format(final['error_rate(%)'].std()))
print('Average ASA error is {}'.format(final['ASA_error_rate(%)'].mean()))
print('ASA Std of folds is {}'.format(final['ASA_error_rate(%)'].std()))
print('Avg DL dice Coef is {}'.format(final['diceCoef'].mean()))
print('DL TP {}'.format(final['TP'].sum()))
print('DL FP {}'.format(final['FP'].sum()))
print('DL FN {}'.format(final['FN'].sum()))
print('Avg DL precision is {}'.format(final['precision'].mean()))
print('Avg DL recall is {}'.format(final['recall'].mean()))
print('Avg DL f1 score is {}'.format(final['f1_score'].mean()))


def StructuralComparison(root1,root2):
    node1,node2= root1,root2
    if (not node1.left) and (not node1.right) and (not node2.left) and (not node2.right) and node1.data == node2.data:
        return True
    if node1.data!=node2.data or (node1.left and not node2.left) or (node1.right and not node2.right) or (not node1.left and node2.left)or \
        (not node1.right and  node2.right):
        return False
    left = StructuralComparison(node1.left,node2.left) if node1.left and node2.left else True
    right = StructuralComparison(node1.right,node2.right) if node1.right and node2.right else True
    return left,right


"""Day 21: Dirac Dice"""

from functools import cache
from itertools import cycle, product
from pathlib import Path

PATH = Path(__file__).parent

class Dice:

    def __init__(self, position1, position2):
        self._pos1 = position1
        self._pos2 = position2

    def play(self):
        return self._play(self._pos1, self._pos2)

    @staticmethod
    def move(position, roll):
        return 1 + (position + roll - 1) % 10

    def _play(self, pos1, pos2, score1=0, score2=0):
        raise NotImplementedError

class DeterministicDice(Dice):

    win_points = 1000

    @staticmethod
    def rolls():
        roll = cycle(range(1, 101))
        while True:
            yield sum(next(roll, 0) for _ in range(3))

    def _play(self, pos1, pos2, score1=0, score2=0):
        rolls = self.rolls()
        count = 0

        for p1_roll, p2_roll in zip(rolls, rolls):
            pos1 = self.move(pos1, p1_roll)
            score1 += pos1
            count += 3
            if score1 >= self.win_points:
                break

            pos2 = self.move(pos2, p2_roll)
            score2 += pos2
            count += 3
            if score2 >= self.win_points:
                break

        return count * min(score1, score2)

class DiracDice(Dice):

    win_points = 21

    @staticmethod
    def rolls():
        return (sum(rolls) for rolls in product((1, 2, 3), repeat=3))

    @cache
    def _play(self, pos1, pos2, score1=0, score2=0):
        if score1 >= self.win_points:
            return 1, 0

        if score2 >= self.win_points:
            return 0, 1

        p1_wins = p2_wins = 0
        for p1_roll in self.rolls():
            new_pos1 = self.move(pos1, p1_roll)
            new_score1 = score1 + new_pos1
            if new_score1 >= self.win_points:
                p1_wins += 1
                continue

            for p2_roll in self.rolls():
                new_pos2 = self.move(pos2, p2_roll)
                new_score2 = score2 + new_pos2
                _p1_wins, _p2_wins = self._play(
                    new_pos1, new_pos2, new_score1, new_score2
                )
                p1_wins += _p1_wins
                p2_wins += _p2_wins

        return p1_wins, p2_wins

def parse(path):
    with path.open(encoding='utf-8') as input_file:
        positions = []
        for line in input_file:
            _, position = line.split(':')
            positions.append(int(position))

        return positions

def test_example1():
    number = DeterministicDice(*parse(PATH / 'example')).play()
    assert number == 739785

def test_example2():
    wins = DiracDice(*parse(PATH / 'example')).play()
    assert max(wins) == 444356092776315

def test_part1():
    number = DeterministicDice(*parse(PATH / 'input')).play()
    assert number == 998088

def test_part2():
    wins = DiracDice(*parse(PATH / 'input')).play()
    assert max(wins) == 306621346123766


import socket

sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("0.0.0.0", 4444))
sock.listen(2)
print("Waiting for the connection! You connecting?")
c, addr = sock.accept()

print("Client that tried to connect with me -> {} ".format(addr))

sock.close()


import time
from PIL import Image

def animate(left_eye, right_eye, left_mouth, right_mouth, delay=0.5):

    eye_top_left = Image.open('routines/displays/images/eyes/pointy_top_left.png')
    eye_left = Image.open('routines/displays/images/eyes/pointy_left.png')
    eye_center = Image.open('routines/displays/images/eyes/pointy_center.png')
    eye_right = Image.open('routines/displays/images/eyes/pointy_right.png')
    eye_top_right = Image.open('routines/displays/images/eyes/pointy_top_right.png')
    eye_top_center = Image.open('routines/displays/images/eyes/pointy_top_center.png')

    mouth_left = Image.open('routines/displays/images/mouth/open_left.png')
    mouth_right = Image.open('routines/displays/images/mouth/open_right.png')

    left_mouth.image(mouth_left)
    right_mouth.image(mouth_right)

    while True:
        set_eyes_and_delay(left_eye, right_eye, delay, eye_center)
        set_eyes_and_delay(left_eye, right_eye, delay, eye_right)
        set_eyes_and_delay(left_eye, right_eye, delay, eye_top_right)
        set_eyes_and_delay(left_eye, right_eye, delay, eye_top_center)
        set_eyes_and_delay(left_eye, right_eye, delay, eye_top_left)
        set_eyes_and_delay(left_eye, right_eye, delay, eye_left)

def set_eyes_and_delay(left_eye, right_eye, delay, image):
    left_eye.image(image)
    right_eye.image(image)
    time.sleep(delay)


from .table import Table
from .player import User, Dealer

def main():
    """Main function for blackjack application

    blackjack table and players are preapred
    game will be played until there is no player on the table
    """
    table = Table()

    player = User("player1")
    # table.add_player(player)

    print(f"There are {table.get_num_players()} players on the table")

    num_game = 0
    playing = True
    while playing:
        print(f"Starting game {num_game+1}")

        # table.play_game()

        if table.get_num_players() == 0:
            print(f"Every player has left the table")
            break

    print(f"Good bye")

if __name__ == "__main__":
    main()


import numpy as np
import cv2
import matplotlib.pyplot as plt
from skimage.color import rgb2gray
from skimage import data
from skimage.filters import gaussian
from skimage.segmentation import active_contour

#LBP is faster (a few times faster) but less accurate. (10-20% less than Haar).
# If you want to detect faces on an embedded system, I think LBP is the choice,
# because it does all the calculations in integers. Haar uses floats, which is a killer for embedded/mobile.

cap = cv2.VideoCapture(0)

# Create the haar or lbp cascade
#faceCascade = cv2.CascadeClassifier("data/haarcascade_frontalface_default.xml")
faceCascade = cv2.CascadeClassifier("data/lbpcascade_frontalface.xml")

while(True):
	# Capture frame-by-frame
	ret, frame = cap.read()

	# Our operations on the frame come here
	gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

	# Detect faces in the image
	faces = faceCascade.detectMultiScale(
		gray,
		scaleFactor=1.1,
		minNeighbors=5,
		minSize=(30, 30)
		#flags = cv2.CV_HAAR_SCALE_IMAGE
	)

	print("Found {0} faces!".format(len(faces)))

	# a=[]
	# b=[]
	s = np.linspace(0, 2*np.pi, 400)
	a = 190 + 150*np.cos(s)
	b = 320 + 150*np.sin(s)
	init = np.array([a, b]).T
	# Draw a rectangle around the faces
	for (x, y, w, h) in faces:
	    cv2.ellipse(frame,(x+w/2,y+h/2),(w,h),0,0,360,(0,255,0))
	    # a.append(x)
	    # b.append(y)
        # cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
	# snake=active_contour(gaussian(frame, 3),
                       # init, alpha=0.015, beta=10, gamma=0.001)

	# Display the resulting frame
	cv2.imshow('frame', frame)
	if cv2.waitKey(1) & 0xFF == ord('a'):
		break

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()


class Task:
    """A class which represents a task.

    Arguments:
        name(str): The task name as defined in the configuration file.
        params(dict, optional): If not `None`, stores the task-relevant
            arguments as well.

    """
    def __init__(self, name, params=None):
        assert name.count(':') == name.count('-') == 1, \
            "Task name should be in <prefix>:<src1,...,srcN>-<trg1,...,trgN> format"

        self.name = name
        self.params = params
        self.prefix, topology = self.name.split(':')
        srcs, trgs = topology.split('-')
        self.sources = srcs.split(',')
        self.targets = trgs.split(',')
        self.multi_source = len(self.sources) > 1
        self.multi_target = len(self.targets) > 1

        self.src, self.trg = None, None

        if not self.multi_source:
            self.src = self.sources[0]

        if not self.multi_target:
            self.trg = self.targets[0]

    def __repr__(self):
        return f'Task({self.name}, srcs={self.sources!r}, trgs={self.targets!r})'

class Batch:
    r"""A custom batch object used through data representation.

    Args:
        data(dict): A dictionary with keys representing the data sources
            as defined by the configuration file and values being the
            already collated `torch.Tensor` objects for a given batch.

        task(str): The task name associated with the batch.

        device(`torch.device`, optional): If given, the batch will be moved
            to the device right after instance creation.

        non_blocking(bool, optional): If `True`, the tensor copies will
            use the `non_blocking` argument.

    Notes:
        In the current code, moving to an appropriate device is handled
        by an explicit call to `.to()` method from the main loop or
        other relevant places.

    Returns:
        a `Batch` instance indexable with bracket notation. The batch size
        is accessible through the `.size` attribute.

    """
    def __init__(self, data, task, device=None, non_blocking=False):
        if device:
            self.data = {k: v.to(device, non_blocking=non_blocking) for k, v in data.items()}
        else:
            self.data = data

        self.task = Task(task)

        dim1s = set([x.size(1) for x in data.values()])
        assert len(dim1s) == 1, \
            "Incompatible batch dimension (1) between modalities."
        self.size = dim1s.pop()

    def __getitem__(self, key):
        return self.data[key]

    def to(self, device, non_blocking=False):
        self.data.update(
            {k: v.to(device, non_blocking=non_blocking) for k, v in self.data.items()})

    def __repr__(self):
        s = f"Batch(size={self.size}, task={self.task})\n"
        for key, value in self.data.items():
            s += f"  {key:10s} -> {value.shape} - {value.device}\n"
        return s[:-1]


"""Code for recording and playing back audio"""
from src.audio.recording import Recording
from src.plotting.plot_recording import plot_recording
from src.file_io.utils import get_recording_file_path
import os

def run():
    """main loop"""
    file_name_short = input("File name to save (.wav): ")
    file_name_full = get_recording_file_path(file_name_short + ".wav")

    duration = int(input("Duration of recording (seconds): "))

    recording = Recording.from_mic(duration=duration)
    recording.save(file_name_full)

    plot_recording(recording)

if __name__ == "__main__":
    print("\nTeam Wapiti - Record\n~~~~~~~~~~~~~~~~~~~\n")
    run()


import torch
import torch.nn as nn
import copy
import time
import numpy as np
from math import ceil, sqrt
from collections import OrderedDict
from utils import strdict_to_dict
from fnmatch import fnmatch, fnmatchcase
from layer import LayerStruct
from .utils import get_pr_model, get_constrained_layers, pick_pruned_model, get_kept_filter_channel, replace_module, get_next_bn
from .utils import get_masks

class MetaPruner:
    def __init__(self, model, args, logger, passer):
        self.model = model
        self.args = args
        self.logger = logger
        self.logprint = logger.log_printer.logprint if logger else print
        self.netprint = logger.log_printer.netprint if logger else print

        # set up layers
        self.LEARNABLES = (nn.Conv2d, nn.Linear) # the layers we focus on for pruning
        layer_struct = LayerStruct(model, self.LEARNABLES)
        self.layers = layer_struct.layers
        self._max_len_ix = layer_struct._max_len_ix
        self._max_len_name = layer_struct._max_len_name
        self.layer_print_prefix = layer_struct.print_prefix

        # set up pr for each layer
        self.raw_pr = get_pr_model(self.layers, args.stage_pr, skip=args.skip_layers, compare_mode=args.compare_mode)
        self.pr = copy.deepcopy(self.raw_pr)

        # pick pruned and kept weight groups
        self.constrained_layers = get_constrained_layers(self.layers, self.args.same_pruned_wg_layers)
        print(f'Constrained layers: {self.constrained_layers}')

    def _get_kept_wg_L1(self, align_constrained=False):
        # ************************* core pruning function **************************
        self.pr, self.pruned_wg, self.kept_wg = pick_pruned_model(self.model, self.layers, self.raw_pr,
                                                        wg=self.args.wg,
                                                        criterion=self.args.prune_criterion,
                                                        compare_mode=self.args.compare_mode,
                                                        sort_mode=self.args.pick_pruned,
                                                        constrained=self.constrained_layers,
                                                        align_constrained=align_constrained)
        # ***************************************************************************

        # print
        print(f'*********** Get pruned wg ***********')
        for name, layer in self.layers.items():
            logtmp = f'{self.layer_print_prefix[name]} -- Got pruned wg by L1 sorting ({self.args.pick_pruned}), pr {self.pr[name]}'
            ext = f' -- This is a constrained layer. Its pruned/kept indices have been adjusted.' if name in self.constrained_layers else ''
            self.netprint(logtmp + ext)
        print(f'*************************************')

    def _prune_and_build_new_model(self):
        if self.args.wg == 'weight':
            self.masks = get_masks(self.layers, self.pruned_wg)
            return

        new_model = copy.deepcopy(self.model)
        for name, m in self.model.named_modules():
            if isinstance(m, self.LEARNABLES):
                kept_filter, kept_chl = get_kept_filter_channel(self.layers, name, pr=self.pr, kept_wg=self.kept_wg, wg=self.args.wg)

                # decide if renit the current layer
                reinit = False
                for rl in self.args.reinit_layers:
                    if fnmatch(name, rl):
                        reinit = True
                        break

                # get number of channels (can be manually assigned)
                num_chl = self.args.layer_chl[name] if name in self.args.layer_chl else len(kept_chl)

                # copy weight and bias
                bias = False if isinstance(m.bias, type(None)) else True
                if isinstance(m, nn.Conv2d):
                    new_layer = nn.Conv2d(num_chl, len(kept_filter), m.kernel_size,
                                    m.stride, m.padding, m.dilation, m.groups, bias).cuda()
                    if not reinit:
                        kept_weights = m.weight.data[kept_filter][:, kept_chl, :, :]

                elif isinstance(m, nn.Linear):
                    kept_weights = m.weight.data[kept_filter][:, kept_chl]
                    new_layer = nn.Linear(in_features=len(kept_chl), out_features=len(kept_filter), bias=bias).cuda()

                if not reinit:
                    new_layer.weight.data.copy_(kept_weights) # load weights into the new module
                    if bias:
                        kept_bias = m.bias.data[kept_filter]
                        new_layer.bias.data.copy_(kept_bias)

                # load the new conv
                replace_module(new_model, name, new_layer)

                # get the corresponding bn (if any) for later use
                next_bn = get_next_bn(self.model, m)

            elif isinstance(m, nn.BatchNorm2d) and m == next_bn:
                new_bn = nn.BatchNorm2d(len(kept_filter), eps=m.eps, momentum=m.momentum,
                        affine=m.affine, track_running_stats=m.track_running_stats).cuda()

                # copy bn weight and bias
                if self.args.copy_bn_w:
                    weight = m.weight.data[kept_filter]
                    new_bn.weight.data.copy_(weight)
                if self.args.copy_bn_b:
                    bias = m.bias.data[kept_filter]
                    new_bn.bias.data.copy_(bias)

                # copy bn running stats
                new_bn.running_mean.data.copy_(m.running_mean[kept_filter])
                new_bn.running_var.data.copy_(m.running_var[kept_filter])
                new_bn.num_batches_tracked.data.copy_(m.num_batches_tracked)

                # load the new bn
                replace_module(new_model, name, new_bn)
        self.model = new_model

        # print the layer shape of pruned model
        LayerStruct(new_model, self.LEARNABLES)
        return new_model


#oq e o Yfinance
import yfinance as yf
import pandas as pd
from datetime import datetime

import streamlit as st
#como usar?
st.markdown('# Analisando empresas')

st.text_input('Preencha o Ticker Code: ',key ='tickercode',value = 'GOOG')
st.markdown(f'## Ultimas noticias da {st.session_state.tickercode} :')

ticker = st.session_state.tickercode

data = yf.Ticker(ticker)

data_news=pd.DataFrame(data.news)

data_news2 = data_news[['title','publisher','link','relatedTickers']]
st.dataframe(data_news2)

end_date = datetime.now().strftime('%Y-%m-%d')
data_hist = data.history(period='max',start='2019-03-16',end =end_date,interval='5d' )
data_hist = data_hist.reset_index()

st.markdown('# Contrua seu grafico: ')
ey= st.selectbox('Eixo y:',data_hist.columns)
ex = st.selectbox('Eixo x:',data_hist.columns)

st.markdown(f'## Grafico {ey} x {ex}:')
st.line_chart(data_hist,x = ex,y=ey)

print(data_hist)


"""
Model exported as python.
Name : 添加字段
Group :
With QGIS : 32203
"""

from qgis.core import QgsProcessing
from qgis.core import QgsProcessingAlgorithm
from qgis.core import QgsProcessingMultiStepFeedback
from qgis.core import QgsProcessingParameterMapLayer
from qgis.core import QgsProcessingParameterFeatureSink
import processing

class tianjiaziduan(QgsProcessingAlgorithm):

    def initAlgorithm(self, config=None):
        self.addParameter(QgsProcessingParameterMapLayer('parcel4326', 'parcel', defaultValue=None))
        self.addParameter(QgsProcessingParameterFeatureSink('Parcel_removepoint', 'parcel_removepoint', type=QgsProcessing.TypeVectorAnyGeometry, createByDefault=True, defaultValue=None))

    def processAlgorithm(self, parameters, context, model_feedback):
        # Use a multi-step feedback, so that individual child algorithm progress reports are adjusted for the
        # overall progress through the model
        feedback = QgsProcessingMultiStepFeedback(4, model_feedback)
        results = {}
        outputs = {}

        # 字段计算器
        alg_params = {
            'FIELD_LENGTH': 15,
            'FIELD_NAME': 'fclass',
            'FIELD_PRECISION': 2,
            'FIELD_TYPE': 2,  # 字符串
            'FORMULA': '@layer_name',
            'INPUT': parameters['parcel4326'],
            'OUTPUT': QgsProcessing.TEMPORARY_OUTPUT
        }
        outputs[''] = processing.run('native:fieldcalculator', alg_params, context=context, feedback=feedback, is_child_algorithm=True)

        feedback.setCurrentStep(1)
        if feedback.isCanceled():
            return {}

        # 字段计算器2
        alg_params = {
            'FIELD_LENGTH': 12,
            'FIELD_NAME': 'next_id',
            'FIELD_PRECISION': 0,
            'FIELD_TYPE': 1,  # 整型
            'FORMULA': '@row_number + 1190000001',
            'INPUT': outputs['']['OUTPUT'],
            'OUTPUT': QgsProcessing.TEMPORARY_OUTPUT
        }
        outputs[''] = processing.run('native:fieldcalculator', alg_params, context=context, feedback=feedback, is_child_algorithm=True)

        feedback.setCurrentStep(2)
        if feedback.isCanceled():
            return {}

        # 字段计算器3
        alg_params = {
            'FIELD_LENGTH': 10,
            'FIELD_NAME': 'sellparcel',
            'FIELD_PRECISION': 0,
            'FIELD_TYPE': 1,  # 整型
            'FORMULA': '1',
            'INPUT': outputs['']['OUTPUT'],
            'OUTPUT': QgsProcessing.TEMPORARY_OUTPUT
        }
        outputs[''] = processing.run('native:fieldcalculator', alg_params, context=context, feedback=feedback, is_child_algorithm=True)

        feedback.setCurrentStep(3)
        if feedback.isCanceled():
            return {}

        # 删减四边形多余顶点
        alg_params = {
            'INPUT': outputs['']['OUTPUT'],
            'angle': 5,
            'OUTPUT': parameters['Parcel_removepoint']
        }
        outputs[''] = processing.run('script:删减四边形多余顶点', alg_params, context=context, feedback=feedback, is_child_algorithm=True)
        results['Parcel_removepoint'] = outputs['']['OUTPUT']
        return results

    def name(self):
        return '6添加字段'

    def displayName(self):
        return '6添加字段'

    def group(self):
        return 'QGis-resources'

    def groupId(self):
        return 'QGis-resources'

    def createInstance(self):
        return tianjiaziduan()


# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

from odoo import api, fields, models
from datetime import datetime

class AccountPaymentOrder(models.Model):
    _inherit = 'account.payment.order'

    amount_untaxed_invoice = fields.Float(
        string='Amount untaxed invoice'
    )
    invoice_id = fields.Many2one(
        comodel_name='account.invoice',
        string='Invoice'
    )

    @api.multi
    def generated2uploaded(self):
        if self.payment_type == 'inbound' \
                and self.amount_untaxed_invoice > 0 \
                and not self.invoice_id.id:
            product_id = int(
                self.env['ir.config_parameter'].sudo().get_param(
                    'account_payment_order_invoice_product_id'
                )
            )
            if product_id > 0:
                product = self.env['product.product'].browse(product_id)
                vals = {
                    'partner_id': self.company_partner_bank_id.bank_id.partner_id.id,
                    'date': datetime.today(),
                    'date_invoice': datetime.today(),
                    'state': 'draft',
                    'type': 'in_invoice',
                    'comment': ' '
                }
                invoice_obj = self.env['account.invoice'].sudo().create(vals)
                self.invoice_id = invoice_obj.id
                vals = {
                    'invoice_id': self.invoice_id.id,
                    'product_id': product.id,
                    'name': product.name,
                    'quantity': 1,
                    'price_unit': self.amount_untaxed_invoice,
                    'account_id': product.property_account_expense_id.id,
                }
                invoice_line_obj = self.env['account.invoice.line'].sudo().create(vals)
                invoice_line_obj._onchange_product_id()
                invoice_line_obj.compute_taxes()
        # super
        return super(AccountPaymentOrder, self).generated2uploaded()


import numpy as np
import pandas as pd
from PIL import Image
import matplotlib.pyplot as plt

theta1 = pd.read_csv('theta1.csv')
theta2 = pd.read_csv('theta2.csv')
theta3 = pd.read_csv('theta3.csv')

theta1 = theta1.values[:, 1:]
theta2 = theta2.values[:, 1:]
theta3 = theta3.values[:, 1:]

#show picture
path = "dataset2/5/5_62057.jpg"
img = Image.open(path)
data = img.getdata()
data = np.matrix(data)
q = data.reshape(45,45)
print(q)
reim  = Image.fromarray(q)
plt.imshow(reim)
plt.show()

#recognize
def sigmoid(z):
    return 1 / (1 + np.exp(-z))
q = q.reshape(1, 2025)
q = np.insert(q, 0, values=np.ones(q.shape[0]), axis=1)
z2 = np.dot(q, theta1.T)
a2 = np.insert(sigmoid(z2), 0, values=np.ones(q.shape[0]), axis=1)
z3 = np.dot(a2, theta2.T)
a3 = np.insert(sigmoid(z3), 0, values=np.ones(q.shape[0]), axis=1)
z4 = np.dot(a3, theta3.T)
a4 = sigmoid(z4)
print(a4)
words = ["+", "-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "=", "div", "x"]
result1  = np.argmax(a4)
print("数字为："+words[result1])


import json

def search_staff(firstname, lastname, middlename):
	file = open('data/staff.json', 'r')
	staffs = json.load(file)['data']

	results = []
	for staff in staffs:
		if firstname  != None and firstname  != staff['firstname'].lower():  continue
		if lastname   != None and lastname   != staff['lastname'].lower():   continue
		if middlename != None and middlename != staff['middlename'].lower(): continue
		results.append(staff)
	return results


from __future__ import unicode_literals
from django.core.mail import send_mail, send_mass_mail
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.template import Context, loader
from django.utils.translation import override
import email
from rudi.models import Group
import socket

SUBJECT_CODE = "Anmeldung zum %s %s"
SUBJECT_ASSIGNMENT = "%s %s - Teamzuweisung"

def send_single(context, text_template, subject, sender, recipients):
    text = text_template.render(context).encode("utf-8")

    # force 7bit/8bit content-transfer-encoding for text instead of base64
    # this helps to avoid getting caught by spam filters
    email.charset.add_charset("utf-8", email.charset.SHORTEST, None, None)

    return send_mail(
        subject=subject.encode("utf-8"),
        message=text,
        from_email=sender,
        recipient_list=recipients,
    )

def send_code(request, team):
    text_template = loader.get_template(
        "rudi/mails/code.txt")

    context = Context({
        "team": team,
        "link": request.build_absolute_uri(reverse(
            "rudi:confirm",
            args=(
                team.code,
            ),
        )),
    })
    subject = SUBJECT_CODE % (team.event.name, team.event.semester)
    if team.participant_2_email:
        recipients = [
            team.participant_1_email,
            team.participant_2_email
        ]
    else:
        recipients = [team.participant_1_email]
    with override(team.language):
        send_single(
            context,
            text_template,
            subject,
            team.event.advisor.email,
            recipients,
        )

def send_assignment(groups):
    sent = 0
    skipped = 0
    for group in groups:
        if not group.chef.sent_assignment_mail:
            sent += _send_assignment_per_course(group)
            group.chef.sent_assignment_mail = True
            group.chef.save()
        else:
            skipped += 1
    return (sent, skipped)

def _send_assignment_per_course(group):
    if group.course.name == "a":
        text_template = loader.get_template(
            "rudi/mails/assignment_appetizer.txt")
        chef_main_course = Group.objects.get(
            Q(member1=group.chef) | Q(member2=group.chef),
            event=group.event,
            course__name="m",
        )
        chef_dessert = Group.objects.get(
            Q(member1=group.chef) | Q(member2=group.chef),
            event=group.event,
            course__name="d",
        )
        context = Context({
            "chef": group.chef,
            "member1": group.member1,
            "member2": group.member2,
            "chef_main_course": chef_main_course.chef,
            "chef_dessert": chef_dessert.chef,
        })
    elif group.course.name == "m":
        text_template = loader.get_template(
            "rudi/mails/assignment_main_course.txt")
        chef_appetizer = Group.objects.get(
            Q(member1=group.chef) | Q(member2=group.chef),
            event=group.event,
            course__name="a",
        )
        chef_dessert = Group.objects.get(
            Q(member1=group.chef) | Q(member2=group.chef),
            event=group.event,
            course__name="d",
        )
        context = Context({
            "chef": group.chef,
            "member1": group.member1,
            "member2": group.member2,
            "chef_appetizer": chef_appetizer.chef,
            "chef_dessert": chef_dessert.chef,
        })
    elif group.course.name == "d":
        text_template = loader.get_template(
            "rudi/mails/assignment_dessert.txt")
        chef_appetizer = Group.objects.get(
            Q(member1=group.chef) | Q(member2=group.chef),
            event=group.event,
            course__name="a",
        )
        chef_main_course = Group.objects.get(
            Q(member1=group.chef) | Q(member2=group.chef),
            event=group.event,
            course__name="m",
        )
        context = Context({
            "chef": group.chef,
            "member1": group.member1,
            "member2": group.member2,
            "chef_appetizer": chef_appetizer.chef,
            "chef_main_course": chef_main_course.chef,
        })
    subject = SUBJECT_ASSIGNMENT % (group.event.name, group.event.semester)
    if group.chef.participant_2_email:
        recipients = [
            group.chef.participant_1_email,
            group.chef.participant_2_email
        ]
    else:
        recipients = [group.chef.participant_1_email]
    with override(group.chef.language):
        return send_single(
            context,
            text_template,
            subject,
            group.event.advisor.email,
            recipients,
        )

def send_custom(sender, recipients, subject, body):
    datalist = []
    for recipient in recipients:
        datalist.append((subject, body, sender, [recipient]))

    # force 7bit/8bit content-transfer-encoding for text instead of base64
    # this helps to avoid getting caught by spam filters
    email.charset.add_charset("utf-8", email.charset.SHORTEST, None, None)

    sent_amount = send_mass_mail(tuple(datalist))

    # Send confirmation to advisor
    advisor_body = "This is a confirmation message. The users"
    addresses = "\n - ".join(recipients)
    advisor_body += addresses + "were sent the following message:\n\n" + body
    send_mail(
        "Confirmation: " + subject,
        advisor_body,
        "rudi@%s" % socket.gethostname(),
        [sender]
    )

    return sent_amount


"""add address column to devices

Revision ID: c906c4a3f534
Revises: 07f2680e4b64
Create Date: 2020-01-12 12:08:19.560564

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'c906c4a3f534'
down_revision = '07f2680e4b64'
branch_labels = None
depends_on = None

def upgrade():
    op.add_column('devices', sa.Column('address', sa.String(255), comment='Network address for management console (e.g. IP or Web Address)'))

def downgrade():
    # Use batch operation for drop column as SQLite does not natively support it
    # See https://alembic.sqlalchemy.org/en/latest/batch.html
    # op.drop_column('devices', 'address')
    with op.batch_alter_table('devices') as batch_op:
        batch_op.drop_column('address')


import math
import numpy as np

import matplotlib
from matplotlib.widgets import Slider

matplotlib.use('TkAgg')
from matplotlib import pyplot as plt
from matplotlib import gridspec

plt.rcParams['text.usetex'] = True

class NonLinearCurveFittingFromClosedFormSolution:
    def __init__(self):
        self.points = []
        self.current_line = None

        self.fig = plt.figure(constrained_layout=True)
        self.grid_spec = gridspec.GridSpec(43, 40, figure=self.fig)
        self.ax = self.fig.add_subplot(self.grid_spec[:40, :])
        slider_axis = self.fig.add_subplot(self.grid_spec[41:43, :])
        self.slider = Slider(slider_axis, "Polynomial order: ", 1, 10, valstep=1)
        self.slider.on_changed(self.on_slider_changed)

        self.fig.canvas.mpl_connect("button_press_event", self.on_click)
        self.ax.axis('off')
        self.ax.set_title('Non linear curve fitting')
        self.ax.set_xlim([-5, 5])
        self.ax.set_ylim([-5, 5])
        self.ax.set_aspect(1)
        self.fig.tight_layout()
        plt.show()

    def on_slider_changed(self, _):
        if len(self.points) > 1:
            order = int(self.slider.val)
            theta = np.asarray(self.get_curve_parameters(self.points, order)).reshape(-1)
            x = np.linspace(-10, 10, 100)
            y = np.asarray([sum(np.asarray([np.power(px, i) for i in range(order + 1)]) * theta) for px in x])
            if self.current_line is not None:
                self.current_line[0].remove()
            self.current_line = self.ax.plot(x, y, color='red', linewidth=2)
        self.fig.canvas.draw()

    def on_click(self, event):
        if event.inaxes is self.ax:
            current_point = [round(event.xdata, 2), round(event.ydata, 2)]
            self.points.append(current_point)
            self.ax.scatter(event.xdata, event.ydata, marker='x', color='green', linewidth=2)
            self.on_slider_changed(None)

    @staticmethod
    def get_curve_parameters(points, order):
        """
        @param points: list of points [[x1, y1], ... [xm, ym]]
        @param order: a number (int)
        @return: theta [theta1, theta2, ...] (length should be equal to polynomial-order + 1)
        Example: for order 2 --> y = [theta1, theta2, theta3] . [1, x, xx]
        """
        print('-----------------------------------------------')
        print("points: ", points)
        print("polynomial order: ", order)
        X_matrix = []
        Y_matrix = []

        for point in points:
            this_point_polynomial = []
            for i in range(order + 1):
                this_point_polynomial.append(np.power(point[0], i))

            X_matrix.append(this_point_polynomial)
            Y_matrix.append([point[1]])

        X_matrix = np.asarray(X_matrix)  # m x order+1
        Y_matrix = np.asarray(Y_matrix)  # m x 1
        theta = np.linalg.inv(X_matrix.T.dot(X_matrix)).dot(X_matrix.T.dot(Y_matrix))

        return theta.T[0]

NonLinearCurveFittingFromClosedFormSolution()


import os
import glob

#사용한 함수
#folder 생성
def createFolder(directory):
    try:
        if not os.path.exists(directory):
            os.makedirs(directory)
    except:
        print('Error')

def get_attr(name:str):
    if '.jpg' in name:
        return '.jpg'

    elif '.png' in name:
        return '.png'

    elif '.jpeg' in name:
        return '.jpeg'

    else:
        return None

##데이터 이름 바꾸기##
#새로운 파일 name:  label + artist_name + 걍 index + attr
#aritists 폴더 접근
main_path = 'C:\\Users\\User\\Desktop\\artists\\origin'
artist_lists = os.listdir(main_path)

#artist 폴더 접근
for label, artist in enumerate(artist_lists):

    artist_path = main_path + '\\' + artist
    artworks = os.listdir(artist_path)

    #files = [artworks.....]
    files = glob.glob(artist_path + '\*')
    for num, f in enumerate(files):
        try:
            attr = get_attr(f.lower())
            new_name = str(label) + artist + '_' + str(num) + attr
            os.rename(f, artist_path + '\\' + new_name)
        except:
            pass

##데이터 스플릿##
## train/ test로 8:2 split
def split(img_list, test_count, train_path, test_path):

  test_files=[]
  for i in random.sample( img_list, test_count ):
      if get_attr(i) is not None:
        test_files.append(i)

  # 차집합으로 train/test 리스트 생성하기
  train_files = [x for x in img_list if x not in test_files and get_attr(x) is not None]

  for k in train_files:
    shutil.copy(k, train_path)

  for c in test_files:
    shutil.copy(c, test_path)

#aritists 폴더 접근
origin_path = 'C:\\Users\\User\\Desktop\\artists\\origin'
artist_lists = os.listdir(origin_path)

#artist 폴더 접근
for artist in artist_lists:

    #경로 및 폴더 생성
    artist_train_path = 'C:\\Users\\User\\Desktop\\artists\\train'
    artist_test_path = 'C:\\Users\\User\\Desktop\\artists\\test'
    createFolder(artist_train_path)
    createFolder(artist_test_path)

    #작품집
    artworks = sorted(glob.glob(origin_path+ '\\' + artist + '\*'))

    #train과 test에 얼마나 옮길 지 count, 이거 숫자 정확하지 않다 ~ 중간 과정에서 tiff등은 제외함
    artworks_test_count = round(len(artworks)*0.2)
    print(f'{artist}의 그림 중 test에는 {artworks_test_count} 들어갑니다')

    #
    split(artworks, artworks_test_count, artist_train_path, artist_test_path)

##데이터 학습할 수 있는 형태로 변경 및 저장/불러오기##
#근데 이미지 크기가 각기 다르다 보니, batch_size = 1인 형태이다
#1.PIL.Image.open()을 통해 이미지를 불러오고
#2.tensor 형식으로 바꾸고, unsqueeze를 통해 4차원(N,C,H,W)형태로 변경

#데이터 zip 및 불러오기 ~ 각 이미지 크기가 다르다보니 dataloader 불가
#data 경로
train_path = 'C:\\Users\\User\\Desktop\\artists\\train'
test_path = 'C:\\Users\\User\\Desktop\\artists\\test'
train_artworks = glob.glob(train_path + '\*')
test_artworks = glob.glob(test_path + '\*')
pil2tensor = transforms.ToTensor()

#data 라벨링 및 xb,yb 묶기
xb = []
yb = []
i = 1

print(len(train_artworks), '이만큼 해야된다')
with open('C:\\Users\\User\\Desktop\\origin\\train_dl.pkl', 'wb') as fil_e:

    for f in random.sample(train_artworks, len(train_artworks)):
        try:
            img = Image.open(f)
            xb.append(torch.unsqueeze(pil2tensor(img), dim = 0))
            yb.append(torch.tensor([int(os.path.basename(f)[0])]))
            i+= 1

            if i % 100 == 0:
                print(f'{i}개 만큼 했다')
                train_dl = list(zip(xb,yb))
                pickle.dump(train_dl, fil_e)
                print(f'{i//100}번 째 dataset 만듦')
                xb = []
                yb = []

        except:
            print(f)
            pass

    print('train에서는 마지막 dataset 저장 간다')
    train_dl = list(zip(xb,yb))
    pickle.dump(train_dl, fil_e)

#데이터 저장 및 불러오기
print('train_dl 완성완료')
#train_path = 'C:\\Users\\User\\Desktop\\origin\\train_dl.pkl'
#with open('C:\\Users\\User\\Desktop\\origin\\train_dl.pkl', 'rb') as fs:
#    data = pickle.load(fs)

#해당 val_dl 마련 및 저장
xb = []
yb = []
i = 1
num = 0
print('--------')

print(len(test_artworks), '이만큼 해야된다')
with open('C:\\Users\\User\\Desktop\\origin\\val_dl.pkl', 'wb') as fil_e:
    for f in random.sample(test_artworks, len(test_artworks)):
        try:
            img = Image.open(f)
            xb.append(torch.unsqueeze(pil2tensor(img), dim = 0))
            yb.append(torch.tensor([int(os.path.basename(f)[0])]))
            i+= 1

            if i % 100 == 0:
                print(f'{i}개 만큼 했다')

                val_dl = list(zip(xb,yb))
                pickle.dump(train_dl, fil_e)
                print(f'{i//100}번 째 dataset 만듦')
                xb = []
                yb = []

        except:
            print(f)
            pass

    val_dl = list(zip(xb,yb))
    pickle.dump(val_dl, fil_e)

    print('--------------------------------')
    print('--------------------------------')
    print('--------------------------------')
    print('--------------------------------')

    print('all dataset 저장 완료')


diccionario = {}
palabras = input("Introduce palabras (español) y sus traducciones (inglés): ")
for i in palabras.split(','):
    clave, valor = i.split(':')
    diccionario[clave] = valor
frase = input('Introduce una frase en español: ')
for i in frase.split():
    if i in diccionario:
        print(diccionario[i], end=' ')
    else:
        print(i, end=' ')


import logging
from plexapi.server import PlexServer
from plexapi.exceptions import Unauthorized
import os
from dotenv import load_dotenv
import sys
import textwrap
from tmdbapis import TMDbAPIs
import requests
import pathlib
from timeit import default_timer as timer
from helpers import get_ids, get_plex, load_and_upgrade_env

# // TODO: improved error handling
# // TODO: TV Theme tunes
# // TODO: Process Music libraries
# // TODO: Process Photo libraries

# import tvdb_v4_official

start = timer()

import logging
from pathlib import Path
from datetime import datetime, timedelta
# current dateTime
now = datetime.now()

# convert to string
RUNTIME_STR = now.strftime("%Y-%m-%d %H:%M:%S")

SCRIPT_NAME = Path(__file__).stem

VERSION = "0.1.0"

env_file_path = Path(".env")

logging.basicConfig(
    filename=f"{SCRIPT_NAME}.log",
    filemode="w",
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
    level=logging.INFO,
)

logging.info(f"Starting {SCRIPT_NAME} {VERSION} at {RUNTIME_STR}", 'info', 'a')
print(f"Starting {SCRIPT_NAME} {VERSION} at {RUNTIME_STR}", 'info', 'a')

if load_and_upgrade_env(env_file_path) < 0:
    exit()

LIBRARY_NAME = os.getenv("LIBRARY_NAME")
TMDB_KEY = os.getenv("TMDB_KEY")
TVDB_KEY = os.getenv("TVDB_KEY")
REMOVE_LABELS = os.getenv("REMOVE_LABELS")

if REMOVE_LABELS:
    lbl_array = REMOVE_LABELS.split(",")

# Commented out until this doesn't throw a 400
# tvdb = tvdb_v4_official.TVDB(TVDB_KEY)

tmdb = TMDbAPIs(TMDB_KEY, language="en")

tmdb_str = "tmdb://"
tvdb_str = "tvdb://"

local_dir = f"{os.getcwd()}/posters"

os.makedirs(local_dir, exist_ok=True)

show_dir = f"{local_dir}/shows"
movie_dir = f"{local_dir}/movies"

os.makedirs(show_dir, exist_ok=True)
os.makedirs(movie_dir, exist_ok=True)

def progress(count, total, status=""):
    bar_len = 40
    filled_len = int(round(bar_len * count / float(total)))

    percents = round(100.0 * count / float(total), 1)
    bar = "=" * filled_len + "-" * (bar_len - filled_len)
    stat_str = textwrap.shorten(status, width=30)

    sys.stdout.write("[%s] %s%s ... %s\r" % (bar, percents, "%", stat_str.ljust(30)))
    sys.stdout.flush()

print("tmdb config...")

base_url = tmdb.configuration().secure_base_image_url
size_str = "original"

plex = get_plex()

logging.info("connection success")

print(f"getting items from [{LIBRARY_NAME}]...")
items = plex.library.section(LIBRARY_NAME).all()
item_total = len(items)
print(f"looping over {item_total} items...")
item_count = 1
for item in items:
    tmpDict = {}
    imdb_id, tmdb_id, tvdb_id = get_ids(item.guids, TMDB_KEY)
    item_count = item_count + 1
    try:
        progress(item_count, item_total, item.title)
        pp = None
        if item.TYPE == "show":
            try:
                pp = (
                    tmdb.tv_show(tmdb_id).poster_path
                    if tmdb_id
                    else tmdb.find_by_id(tvdb_id=tvdb_id).tv_results[0].poster_path
                )
            except:
                pp = "NONE"
            tgt_dir = show_dir
        else:
            pp = tmdb.movie(tmdb_id).poster_path
            tgt_dir = movie_dir

        if pp is not None:
            if pp == "NONE":
                posters = item.posters()
                item.setPoster(posters[0])
            else:
                ext = pathlib.Path(pp).suffix
                posterURL = f"{base_url}{size_str}{pp}"
                local_file = f"{tgt_dir}/{item.ratingKey}{ext}"
                if not os.path.exists(local_file):
                    r = requests.get(posterURL, allow_redirects=True)
                    open(f"{local_file}", "wb").write(r.content)
                item.uploadPoster(filepath=local_file)
        else:
            progress(item_count, item_total, "unknown type: " + item.title)

    except Exception as ex:
        progress(item_count, item_total, "EX: " + item.title)

end = timer()
elapsed = end - start
print(f"{os.linesep}{os.linesep}processed {item_count - 1} items in {elapsed} seconds.")


from django.http import HttpResponse
from django.http import Http404
from django.shortcuts import render, redirect
from django.views import generic
from django.views.generic.edit import CreateView, UpdateView
from .models import TwoWheeler

# Create your views here.
def bike(request):
    bike_all = TwoWheeler.objects.all()
    con = {
        'bike_all': bike_all,
    }
    return render(request, "bike/first.html", con)

def details(request, pk):
    try:
        x = TwoWheeler.objects.get(pk=pk)
    except TwoWheeler.DoesNotExist:
        raise Http404("Vehicle Not Found")
    con = {
        'x': x,
    }
    return render(request, "bike/details.html", con)

class BikeInsert(CreateView):
    model = TwoWheeler
    fields = ['Model_name', 'Company_name', 'Model_Info', 'Model_img', 'Price_before', 'Price_after']


""" for (test in data_frame):
        if(searchAsana(test) == true):
                updateAsana(test)
        if(searchAsana(test) == false:
                addToListOfNewTests(test)
        else:
            addtoListForHumanVerification(test)

        ##Afterwards....
        allCNumbersInAsana = getTestByFunctionalArea(area)
        for (CNumber in allCNumbersinAsana):
            if(not in data_frame['ID']):
                addtoListofMissingtestCases(CNumber)
"""

##MBTA Workspace GID: 15492006741476
##Test bed project GID : 1203186680032258

##Remember to add "Automatically Generated" tag GID = '1203191978331220'
##Remember to add "Automatically Updated" tag GID = ''
##Asana records consist of Test: [Area] and optionally,
##Create a new CSV containing all test cases received for which we received a C number, but there are no C Numbers in our Asana records. These are new test cases
##Create a CSV containing all test cases where we had an Asana record, but did not receive a C number. These are potentially missing test cases.

##Test Script Review Status GID = 1184137262520341
##Reference Review Required Task GID = 1202894410931244
##Reference Approved Task GID = 1202632050244277
##Custom Field 'ID Number' GID = 515737795293097

#Step 1 - Check to see if a task exists in Asana with the defined C number.

import sys
import requests
import os
import pandas
import numpy
import asana
import re ##Regex -- used for matching name string
import json
import datetime
import openpyxl
from openpyxl import Workbook, load_workbook
from datetime import date

##HELPERS

def AddTasksToWorksheet(tasks, sheet, workbook):
##QA COMPLETE TAG GID = 649069647070258
##HOLD TAG GID = 259956811260129
##STATUS COLUMN HEADER = OPYXL ROW 10, COLUMN 10
    qa_tag = {'gid': '649069647070258', 'resource_type': 'tag'}
    hold_tag =  {'gid': '259956811260129', 'resource_type': 'tag'}
    rownum = 11
    for row, task in enumerate(tasks):
        ##OPYXL is 1-indexed. This should be the first row of the output spreadsheet you want to fill. TODO: Find the actual header and drop down one instead of writing '11'

            if hold_tag not in task['tags']:
                if qa_tag in task['tags']:
                    for ind, field in enumerate(task['custom_fields']):
                        print(field['name'])
                        ##Add review status...
                        if (field['name'] == 'Test Script Review Status'):
                         ##   print(task['custom_fields'][ind]['display_value'])
                            sheet.cell(row = rownum, column = 10).value = task['custom_fields'][ind]['display_value']
                        if (field['name'] == 'ID Number'):
                            sheet.cell(row = rownum, column = 5).value = task['custom_fields'][ind]['display_value']
                        if (field['name'] == 'Test Type'):
                            sheet.cell(row = rownum, column = 2).value = task['custom_fields'][ind]['display_value']
                        if (field['name'] == 'Functional area'):
                            sheet.cell(row = rownum, column = 4).value = task['custom_fields'][ind]['display_value']
                        if (field['name'] == 'Comment Tracking'):
                            sheet.cell(row = rownum, column = 3).value = task['custom_fields'][ind]['display_value']
                        if (field['name'] == 'Test Type'):
                            sheet.cell(row = rownum, column = 2).value = task['custom_fields'][ind]['display_value']

                    sheet.cell(row = rownum, column = 7).value = task['notes']
                    sheet.cell(row = rownum, column = 6).value = task['name']
                    rownum = rownum + 1

            workbook.save('output.xlsm')

   ## sheet.cell(rownum, column=11).value = task

def get_tasks_by_section(section_gid):
    tasklist = []
    tasks = client.tasks.get_tasks({'section':section_gid, 'completed_since':date.today(), 'opt_fields':['name', 'notes', 'tags', 'custom_fields']}, )
    print(tasks)
    for task in tasks:
        tasklist.append(task)
    return tasklist

##Put your token in a 'credentials.py' in the same directory as this script
sys.path.append(os.path.relpath('.\credentials.py'))
from credentials import token

##SIMPLE ASANA AUTHENTICATION##
##Headers to log in as Robert

##Set token equal to a Personal Access Token

client = asana.Client.access_token(token)
workspace = '15492006741476'

##LOAD THE TEMPLATE

template_contents = os.listdir('template')
if(len(template_contents)>1):
    print('!ERROR! More than one file in template folder')
else:
    temp_filename = os.listdir('template')[0]
    temp_filepath = os.path.dirname(os.path.abspath(__file__)) + '/template/'+ temp_filename

temp_xlsx=pandas.ExcelFile(temp_filepath)
##Select the 'Document Comments' worksheet
df_temp = pandas.read_excel(temp_xlsx, "Document Comments")

##LOAD THE INPUT SHEET (WE DO THIS TO IMPORT ANY HEADERS SO CONTRACTS CAN CHANGE THEM...)

input_contents = os.listdir('input')
if(len(template_contents)>1):
    print('!ERROR! More than one file in input folder')
else:
    in_filename = os.listdir('input')[0]
    in_filepath = os.path.dirname(os.path.abspath(__file__)) + '/input/'+ in_filename

in_xlsx=pandas.ExcelFile(in_filepath)
##Select the 'Document Comments' worksheet
df_in = pandas.read_excel(in_xlsx, "Document Comments")

##Opyxl is for insertion
temp_opyxl_wb = openpyxl.load_workbook(in_filepath, keep_vba=True)
temp_opyxl_sheet = temp_opyxl_wb['Document Comments']
print(temp_opyxl_sheet.cell(row = 10, column = 10 ).value)

##Check row 8 of template + input. Are they the same? These are the headers for where we import content. If these don't match exactly, the program will not run.
df_temp_headers = df_temp.iloc[8]
df_in_headers = df_in.iloc[8]

if(df_in_headers.equals(df_temp_headers)):
    print("Matching!")
else:
    print("This program has determined that row 8 of the input and template are different. Does your input format match the template exactly?")
    quit()

print("This program has determined that row 8 of both the input and template match.")

test_project_id = input("Please enter the ID of the test project (E.G: 'Test: System Website' is `1202257161854797` as extracted from https://app.asana.com/0/1202257161854797/list)")

##Seek and register sections named 'General Commetns' and 'Test cases currently under review'. Case specific, etc.

general_gid = None
cases_gid = None

##Validate that there's only one section with the rquired name in the project. Get the GIDs.
test_project_sections = client.sections.find_by_project(test_project_id)
for section in test_project_sections:
    print(section)
    if(section['name'] == 'General comments'):
        if(general_gid != None):
            print("WARNING! Multiple sections found with the name, 'General comments'. Quitting.")
        general_gid = section['gid']
    if(section['name'] == 'Test cases currently under review'):
        if(cases_gid != None):
            print("Warning! Multiple sections labeled 'Test cases under review'. Quitting.")
            quit()
        cases_gid = section['gid']

gen_tasklist = get_tasks_by_section(general_gid)

cases_tasklist = get_tasks_by_section(cases_gid)

total_tasklist = gen_tasklist + cases_tasklist

AddTasksToWorksheet(total_tasklist, temp_opyxl_sheet, temp_opyxl_wb)


#! /usr/bin/env python
"""Steady-state lat-by-lat RCE throughout the annual cycle using climlab."""

from contextlib import contextmanager, redirect_stdout
import os

import climlab
from climlab.convection import ConvectiveAdjustment
from climlab.radiation import (
    AnnualMeanInsolation,
    DailyInsolation,
    FixedInsolation,
    RRTMG,
)
from climlab.radiation.water_vapor import (
    FixedRelativeHumidity,
    ManabeWaterVapor,
)
from climlab.solar.insolation import daily_insolation
import numpy as np
import xarray as xr

ALBEDO = 0.3
DAY_TYPE = 1  # 1: `day` is calendar day, starting January 1
DAY_OF_YEAR = 80.  # 80 = NH spring equinox
DAY_OF_YEAR_STR = 'day_of_year'
DIR_OUTPUT = 'output'
DIR_TMP = 'tmp'
DT_IN_DAYS = 20.
LAPSE_RATE = 6.5
LAPSE_RATE_DRY = 10.
LAT_STR = 'lat'
LEV_STR = 'lev'
MIXED_LAYER_DEPTH = 1.
NUM_VERT_LEVELS = 30
NUM_DAYS = 1000.
SOLAR_CONST = 1365.2
STEF_BOLTZ_CONST = 5.6704e-8
TEMP_MIN_VALID = 150.
TEMP_MAX_VALID = 340.
TEMP_SFC_INIT = None
TIME_STR = 'time'
WRITE_TO_DISK = False

@contextmanager
def suppress_stdout(yes=True):
    if yes:
        with open(os.devnull, 'w') as f, redirect_stdout(f):
            yield
    else:
        yield

def _coord_arr_1d(start, stop, spacing, dim):
    """Create xr.DataArray of an evenly spaced 1D coordinate ."""
    arr_np = np.arange(start, stop + 0.1*spacing, spacing)
    return xr.DataArray(arr_np, name=dim, dims=[dim],
                        coords={dim: arr_np})

def lat_arr(start=None, stop=None, spacing=1., dim=LAT_STR):
    """Convenience function to create an array of latitudes."""
    if start is None and stop is None:
        start = -90 + 0.5*spacing
        stop = 90 - 0.5*spacing
    return _coord_arr_1d(start, stop, spacing, dim)

def time_arr(start=0, stop=100, spacing=1., dim=TIME_STR):
    """Convenience function to create an array of times."""
    return _coord_arr_1d(start, stop, spacing, dim)

def days_of_year_arr(start=1, stop=365, spacing=1., dim=DAY_OF_YEAR_STR):
    """Convenience function to create an array of times."""
    return _coord_arr_1d(start, stop, spacing, dim)

def coszen_from_insol(lats, insol, quiet=True):
    """Get cosine of zenith angle."""
    state = climlab.column_state(num_lev=1, lat=lats)
    with suppress_stdout(yes=quiet):
        return RRTMG(state=state, insolation=insol).coszen

def time_avg_insol(lat, start_day, end_day, day_type=1):
    """Average insolation over the subset of the annual cycle."""
    days = np.arange(start_day, end_day+0.1, 1)
    insolation = [daily_insolation(lat=lat, day_type=day_type, day=day)
                  for day in days]
    return np.array(insolation).mean(axis=0)

def _gen_column_state(lat,
                      insol_type: str = 'fixed_day',
                      num_vert_levels: int = NUM_VERT_LEVELS,
                      mixed_layer_depth: float = MIXED_LAYER_DEPTH):
    """Generate column state for climlab."""
    # Hack to get climlab's DailyInsolation to work at a single latitude: just
    # create a len-2 array with the same latitude.  Later in the pipeline,
    # strip off that duplicate value.
    do_duplicate = (insol_type in ('ann_cycle', 'ann_mean') and
                    (np.isscalar(lat) or np.size(lat) == 1))
    if do_duplicate:
        state_lat = [lat, lat]
    else:
        state_lat = lat
    return climlab.column_state(lat=state_lat,
                                num_lev=num_vert_levels,
                                water_depth=mixed_layer_depth)

def _gen_h2o_proc(state,
                  dry_atmos: bool = False,
                  water_vapor=ManabeWaterVapor):
    """Generate the water vapor process for climlab."""
    if dry_atmos:
        return FixedRelativeHumidity(relative_humidity=0., qStrat=0.,
                                     state=state)
    return water_vapor(state=state)

def _gen_convec_proc(state, convec_model=ConvectiveAdjustment,
                     dry_atmos: bool = False,
                     lapse_rate: float = LAPSE_RATE):
    """Generate the convection process for climlab."""
    if dry_atmos:
        print("Dry atmosphere specified, so overriding given lapse rate of "
              "'{}' with dry adiabatic lapse rate.".format(lapse_rate))
        lapse_rate = LAPSE_RATE_DRY
    return convec_model(state=state, adj_lapse_rate=lapse_rate)

def _gen_insol_proc(lat, state,
                    insol_type: str = 'fixed_day',
                    day_of_year: int = DAY_OF_YEAR,
                    day_type: int = DAY_TYPE,
                    insol_avg_window: int = 1,
                    solar_const: float = SOLAR_CONST):
    """Generate insolation and corresponding zenith angle arrays."""
    if insol_type == 'ann_cycle':
        return DailyInsolation(state=state, domains=state.Ts.domain)
    if insol_type == 'ann_mean':
        return AnnualMeanInsolation(state=state, domains=state.Ts.domain)
    elif insol_type == 'fixed_day':
        if insol_avg_window > 1:
            dday = 0.5*insol_avg_window,
            insolation = time_avg_insol(lat, day_of_year - dday,
                                        day_of_year + dday,
                                        day_type=day_type)
        else:
            insolation = daily_insolation(lat=lat, day_type=day_type,
                                          day=day_of_year)
        coszen = coszen_from_insol(lat, insolation)
        return FixedInsolation(S0=solar_const, domains=state.Ts.domain,
                               coszen=coszen)
    VALID_INSOL_TYPES = ('fixed_day', 'ann_mean', 'ann_cycle')
    raise ValueError("insol type must be one of {0}.  "
                     "Got {1}".format(VALID_INSOL_TYPES, insol_type))

def _gen_rad_proc(state, insol_proc, h2o_proc, albedo,
                  rad_model=RRTMG,
                  quiet: bool = True):
    """Generate the radiation process for climlab."""
    with suppress_stdout(yes=quiet):
        return rad_model(state=state,
                         specific_humidity=h2o_proc.q,
                         albedo=albedo,
                         insolation=insol_proc.insolation,
                         coszen=insol_proc.coszen,
                         S0=insol_proc.S0)

def _gen_col_model(state, h2o_proc, convec_proc, insol_proc, rad_proc):
    """Generate a climlab model by combining the given subprocesses."""
    model = climlab.TimeDependentProcess(state=state)
    model.add_subprocess('WaterVapor', h2o_proc)
    model.add_subprocess('Convection', convec_proc)
    model.add_subprocess('Insolation', insol_proc)
    model.add_subprocess('Radiation', rad_proc)
    return model

def rad_equil(insol,
              albedo: float = ALBEDO,
              ghg_layers: int = 1,
              stef_boltz_const: float = STEF_BOLTZ_CONST):
    """Radiative equilibrium with optional single greenhouse layer.

    Standard planetary enerby balance model is solar constant divided by 4,
    which is the global mean insolation accounting for the diurnal cycle that
    generates an effective 0.25 cosine of zenith angle.  Here, that is replaced
    with whatever the actual local insolation value is that is specified, since
    we're considering individual latitudes and the zenith angle can differ.

    """
    temp = (insol*(1-albedo) / (STEF_BOLTZ_CONST))**0.25
    temp *= (2**0.25)**int(ghg_layers)
    return temp

def _init_temps(state, rad_proc, temp_sfc_init=TEMP_SFC_INIT):
    """Initialize surface and atmospheric temperatures for climlab."""
    if temp_sfc_init is None:
        temp_rad_eq = rad_equil(rad_proc.insolation, albedo=rad_proc.asdir)
        temp_rad_eq = np.maximum(temp_rad_eq, TEMP_MIN_VALID)
        temp_rad_eq = np.minimum(temp_rad_eq, TEMP_MAX_VALID)
        state['Ts'][:] = temp_rad_eq
    else:
        state['Ts'][:] = temp_sfc_init
    # TODO: initialize with adiabatic lapse rate.  Currently isothermal.
    # TODO: simpler intermediate solution: just drop temperature uniformly
    # across the levels over say 100 K from surface to TOA.
    state['Tatm'][:] = state['Ts'][:]
    return state

def create_rce_model(lat,
                     day_type=DAY_TYPE,
                     insol_type='fixed_day',
                     day_of_year=DAY_OF_YEAR,
                     insol_avg_window=1,
                     num_vert_levels=NUM_VERT_LEVELS,
                     albedo=ALBEDO,
                     mixed_layer_depth=MIXED_LAYER_DEPTH,
                     dry_atmos=False,
                     rad_model=RRTMG,
                     water_vapor=ManabeWaterVapor,
                     convec_model=ConvectiveAdjustment,
                     lapse_rate=LAPSE_RATE,
                     temp_sfc_init=TEMP_SFC_INIT,
                     quiet=True):
    """Create a column model for a single latitude."""
    state = _gen_column_state(lat,
                              insol_type=insol_type,
                              num_vert_levels=num_vert_levels,
                              mixed_layer_depth=mixed_layer_depth)
    h2o_proc = _gen_h2o_proc(state,
                             dry_atmos=dry_atmos,
                             water_vapor=water_vapor)
    convec_proc = _gen_convec_proc(state,
                                   convec_model=convec_model,
                                   lapse_rate=lapse_rate,
                                   dry_atmos=dry_atmos)
    insol_proc = _gen_insol_proc(lat, state,
                                 insol_type=insol_type,
                                 day_of_year=day_of_year,
                                 day_type=day_type,
                                 insol_avg_window=insol_avg_window)
    rad_proc = _gen_rad_proc(state, insol_proc, h2o_proc, albedo,
                             rad_model=rad_model,
                             quiet=quiet)
    state = _init_temps(state, rad_proc, temp_sfc_init=temp_sfc_init)
    return _gen_col_model(state, h2o_proc, convec_proc, insol_proc, rad_proc)

def _add_metadata(ds, model):
    """Drop unneeded mixed-layer depth bounds and include useful attributes."""
    ds = ds.squeeze('depth').drop('depth_bounds')
    ds.coords['coszen'] = model.subprocess['Radiation'].coszen
    ds.coords['S0'] = model.subprocess['Radiation'].S0
    ds.coords['albedo'] = model.subprocess['Radiation'].asdir
    return ds

def _advance_one_dt(rce_model, dt_in_days,
                    check_temps_valid: bool = True,
                    temp_min_valid: float = TEMP_MIN_VALID,
                    temp_max_valid: float = TEMP_MAX_VALID,
                    quiet: bool = False):
    """Advance the model by one timestep."""
    if check_temps_valid:
        # Prevent model from crashing in weakly insolated columns.
        outside_valid_range = ((np.any(rce_model.Ts > temp_max_valid)) or
                               (np.any(rce_model.Ts < temp_min_valid)))
    else:
        outside_valid_range = False
    if outside_valid_range:
        ds = xr.ones_like(rce_model.to_xarray().copy(deep=True))*np.nan
    else:
        with suppress_stdout(yes=quiet):
            rce_model.integrate_days(dt_in_days)
            ds = rce_model.to_xarray().copy(deep=True)
    # Hack to use time-varying daily insolation for single latitude: previously
    # duplicated the latitude; now remove the duplicate.
    if rce_model.Ts.domain.shape[0] == 2:
        ds = ds.isel(**{LAT_STR: 0})
    else:
        # TODO: fix hack logic re: annual cycle so that metadata gets appended
        # for annual cycle too.
        ds = _add_metadata(ds, rce_model)
    return ds

def run_rce_model(model,
                  num_days_run: int = NUM_DAYS,
                  dt_in_days: int = DT_IN_DAYS,
                  check_temps_valid: bool = True,
                  temp_min_valid: float = TEMP_MIN_VALID,
                  temp_max_valid: float = TEMP_MAX_VALID,
                  quiet: bool = False):
    """Integrate one RCE model in time, combining results into one Dataset."""
    times = time_arr(stop=num_days_run, spacing=dt_in_days)
    results = []
    for time in times:
        results.append(_advance_one_dt(model, dt_in_days,
                                       check_temps_valid=check_temps_valid,
                                       temp_min_valid=temp_min_valid,
                                       temp_max_valid=temp_max_valid))
    return xr.concat(results, dim=times)

def create_and_run_rce_model(lat,
                             insol_type='fixed_day',
                             day_type=DAY_TYPE,
                             day_of_year=DAY_OF_YEAR,
                             dry_atmos=False,
                             insol_avg_window=1,
                             mixed_layer_depth=MIXED_LAYER_DEPTH,
                             num_vert_levels=NUM_VERT_LEVELS,
                             albedo=ALBEDO,
                             rad_model=RRTMG,
                             water_vapor=ManabeWaterVapor,
                             convec_model=ConvectiveAdjustment,
                             lapse_rate=LAPSE_RATE,
                             temp_sfc_init=TEMP_SFC_INIT,
                             dt_in_days=DT_IN_DAYS,
                             num_days_run=NUM_DAYS,
                             check_temps_valid=True,
                             temp_min_valid=TEMP_MIN_VALID,
                             temp_max_valid=TEMP_MAX_VALID,
                             write_to_disk=WRITE_TO_DISK,
                             path_output=None,
                             quiet=True):
    """Create and run a column model for a single latitude."""
    print("lat value: {}".format(lat))
    model = create_rce_model(
        lat,
        insol_type=insol_type,
        day_type=day_type,
        day_of_year=day_of_year,
        insol_avg_window=insol_avg_window,
        num_vert_levels=num_vert_levels,
        albedo=albedo,
        dry_atmos=dry_atmos,
        rad_model=rad_model,
        water_vapor=water_vapor,
        convec_model=convec_model,
        lapse_rate=lapse_rate,
        temp_sfc_init=temp_sfc_init,
        quiet=quiet,
    )
    ds = run_rce_model(
        model,
        num_days_run=num_days_run,
        dt_in_days=dt_in_days,
        check_temps_valid=check_temps_valid,
        temp_min_valid=temp_min_valid,
        temp_max_valid=temp_max_valid
    )
    ds.coords[LAT_STR] = lat

    if write_to_disk:
        if path_output is None:
            path = os.path.join(DIR_TMP, 'rce_climlab_lat{}.nc'.format(lat))
        else:
            path = path_output

        print(path)
        ds.to_netcdf(path)

    return ds

if __name__ == '__main__':
    pass


from twistedbot.plugins.base import PluginChatBase

class Help(PluginChatBase):
    @property
    def command_verb(self):
        return "help"

    @property
    def help(self):
        return "without argument shows aviable commands or help for specfic command"

    def command(self, sender, command, args):
        if not args:
            msg = ["%s [COMMAND]" % self.command_verb]
            msg.append(" ".join(self.world.eventregister.chat_commands.keys()))
            self.send_chat_message(msg)
        else:
            cmd = args[0]
            if cmd in self.world.eventregister.chat_commands:
                self.send_chat_message(self.world.eventregister.chat_commands[cmd].help)
            else:
                self.send_chat_message("unknown comamnd %s" % cmd)

plugin = Help


import re
from typing import Dict
from typing import List
import json
import logging
import os
import random
# from preprocessing_prepeare_sentence import preprocessing
# from preprocessing_prepeare_sentence import head_start_token,head_end_token,tail_start_token,tail_end_token
import copy
from overrides import overrides
from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter
from allennlp.common.file_utils import cached_path
from allennlp.data.dataset_readers.dataset_reader import DatasetReader
from allennlp.data.fields import LabelField, TextField
from allennlp.data.instance import Instance
from allennlp.data.tokenizers import Tokenizer, WordTokenizer
from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer
from allennlp.data.fields import ListField, IndexField, MetadataField, Field

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

e1_start = '[unused1]'
e1_end = '[unused2]'
e2_start = '[unused3]'
e2_end = '[unused4]'
default_ner = '[unused5]'

UNUSED = '[unused{}]'
class NerDict():
    def __init__(self, ner_dict_path):
        self.path = ner_dict_path
        if os.path.isfile(ner_dict_path):
            with open(ner_dict_path) as f:
                data = json.load(f)
                self.d = data['d']
                self.counter = data['counter']
        else:
            self.d = {}
            self.counter = 6  # first unused

    def get_ner_token(self, ne):
        if isinstance(ne, list):
            if len(ne) > 1:
                print("size of list is{}".len(ne))
            ne = ne[0]
        if ne not in self.d:
            self.d[ne] = UNUSED.format(self.counter)
            self.counter += 1
            with open(self.path, 'w') as f:
                json.dump({'d': self.d, 'counter': self.counter}, f)

        return self.d[ne]

def add_word_markers(line):
    # line['token'] = [start_line] + line['token'] + [end_line]
    if line['subj_start'] < line['obj_start']:
        indexes = [line['subj_start'], line['subj_end'] + 2, line['obj_start'] + 2, line['obj_end'] + 4]
        line['token'].insert(indexes[0], e1_start)
        line['token'].insert(indexes[1], e1_end)
        line['token'].insert(indexes[2], e2_start)
        line['token'].insert(indexes[3], e2_end)
    else:
        indexes = [line['subj_start'] + 2, line['subj_end'] + 4, line['obj_start'], line['obj_end'] + 2]
        line['token'].insert(indexes[2], e2_start)
        line['token'].insert(indexes[3], e2_end)
        line['token'].insert(indexes[0], e1_start)
        line['token'].insert(indexes[1], e1_end)
    line['indexes'] = indexes
    return line

def add_mask_markers(line):
    indexes = [line['subj_start'], line['subj_end'] + 1, line['obj_start'], line['obj_end'] + 1]
    if line['subj_start'] < line['obj_start']:
        line['token'] = line['token'][:line['obj_start']] + [e2_start, default_ner, e2_end] + line['token'][
                                                                                line['obj_end'] + 1:]
        line['token'] = line['token'][:line['subj_start']] + [e1_start, default_ner, e1_end] + line['token'][
                                                                                line['subj_end'] + 1:]
    else:
        line['token'] = line['token'][:line['subj_start']] + [e1_start, default_ner, e1_end] + line['token'][line['subj_end'] + 1:]
        line['token'] = line['token'][:line['obj_start']] + [e2_start, default_ner, e2_end] + line['token'][line['obj_end'] + 1:]
    return line

def add_ner_markers(line, nd):
    # indexes = [line['subj_start'], line['subj_end'] + 1, line['obj_start'], line['obj_end'] + 1]
    if line['subj_start'] < line['obj_start']:
        line['token'] = line['token'][:line['obj_start']] + [e2_start, nd.get_ner_token(line['obj_type']), e2_end] + line['token'][
                                                                                line['obj_end'] + 1:]
        line['token'] = line['token'][:line['subj_start']] + [e1_start, nd.get_ner_token(line['subj_type']), e1_end] + line['token'][
                                                                                line['subj_end'] + 1:]
    else:
        line['token'] = line['token'][:line['subj_start']] + [e1_start, nd.get_ner_token(line['subj_type']), e1_end] + line['token'][line['subj_end'] + 1:]
        line['token'] = line['token'][:line['obj_start']] + [e2_start, nd.get_ner_token([line['obj_type']]), e2_end] + line['token'][line['obj_end'] + 1:]
    return line

# def get_ner_mask(ner):

def load_annotated_tokens(infile, debug=False):
    data = json.load(infile)
    if debug:
        data = data[:500]  # for debugging
    # adjusted_tokens = [add_new_word_markers(x) for x in data if x['relation'] == 'no_relation']
    # adjusted_tokens = [add_new_word_markers(x) for x in data]
    return data

@DatasetReader.register("dep_reader")
class DepReader(DatasetReader):
    def __init__(self,
                 lazy: bool = False,
                 tokenizer: Tokenizer = None,
                 token_indexers: Dict[str, TokenIndexer] = None, debug=False) -> None:
        super().__init__(lazy)
        self.debug = debug
        self._token_indexers = token_indexers or {"tokens": SingleIdTokenIndexer()}
    @overrides
    def _read(self, file_path):
        with open(cached_path(file_path), "r") as data_file:
            logger.info("Reading instances from json files at: %s", data_file)
            data = json.load(data_file)
            if self.debug:
                data = data[:500]  # for debugging
            for x in data:
                yield self.text_to_instance(x)
    @overrides
    def text_to_instance(self, data: dict) -> Instance:  # type: ignore
        field_of_tokens = TextField(data['sdp'], self._token_indexers)
        return Instance({'sen_dep':field_of_tokens, 'label': LabelField(data['relation'])})

@DatasetReader.register("tacred_reader")
class TacredReader(DatasetReader):
    def __init__(self,
                 bert_model: str,
                 lazy: bool = False,
                 tokenizer: Tokenizer = None,
                 token_indexers: Dict[str, TokenIndexer] = None, debug=False, mode='basic') -> None:
        super().__init__(lazy)
        self._tokenizer = tokenizer or WordTokenizer()
        self._token_indexers = token_indexers or {"tokens": SingleIdTokenIndexer()}
        self.spacy_splitter = SpacyWordSplitter(keep_spacy_tokens=True)
        self.TRAIN_DATA = "meta_train"
        self.TEST_DATA = "meta_test"
        self.tokens_with_markers = "tokens_with_markers"
        self.head_bert = "head_after_bert"
        self.tail_bert = "tail_after_bert"
        self.labels_to_id = {}
        self.max_id = 0
        self.debug = debug
        self.marker_tokens = self._tokenizer.tokenize('{} {}'.format(e1_start, e2_start))
        self.mode = mode
        if mode =='ner':
            self.ner_dict = NerDict('ner_dict.json')
    @overrides
    def _read(self, file_path):
        with open(cached_path(file_path), "r") as data_file:
            logger.info("Reading instances from json files at: %s", data_file)
            data = json.load(data_file)
            if self.debug:
                data = data[:500]  # for debugging
            for x in data:
                yield self.text_to_instance(x)

    def load_ner_dict(self, ner_dict_path):
        if os.path.isfile(ner_dict_path):
            with open(ner_dict_path) as f:
                return json.load(f)
        else:
            return {}
    @overrides
    def text_to_instance(self, data: dict) -> Instance:  # type: ignore
        if self.mode == 'basic':
            data = add_word_markers(data)
        elif self.mode == 'ner':
            data = add_ner_markers(data, self.ner_dict)
        elif self.mode == 'mask':
            data = add_mask_markers(data)

        tokenized_tokens = self._tokenizer.tokenize(' '.join(data['token']))
        field_of_tokens = TextField(tokenized_tokens, self._token_indexers)
        marker_loc = [tokenized_tokens.index(x)+1 for x in self.marker_tokens] # +1 for the ['cls'] token
        return Instance({'sen':field_of_tokens, 'loc': MetadataField(marker_loc), 'label': LabelField(data['relation'])})  # , 'sentence': TextField(' '.join(data['token']))})

    def proccess_json(self, json_data):
        return add_word_markers(json_data)

    def get_id(self, label):
        if label not in self.labels_to_id:
            self.labels_to_id[label] = self.max_id
            self.max_id += 1
        return self.labels_to_id[label]

@DatasetReader.register("siamese_reader")
class SiameseReader(TacredReader):
    def __init__(self,
                 bert_model: str,
                 lazy: bool = False,
                 tokenizer: Tokenizer = None,
                 token_indexers: Dict[str, TokenIndexer] = None, debug=False) -> None:
        super().__init__(bert_model, lazy, tokenizer, token_indexers, debug)

    def tokenize(self, d):
        tokenized_tokens = self._tokenizer.tokenize(' '.join(d['token']))
        return TextField(tokenized_tokens, self._token_indexers)

    @overrides
    def text_to_instance(self, data: dict) -> Instance:  # type: ignore
        s0 = data[0]
        s1 = data[1]
        if 'indexes' not in s0:
            s0 = add_word_markers(s0)
            s1 = add_word_markers(s1)
        sen1 = self._tokenizer.tokenize(' '.join(s0['token']))
        sen2 = self._tokenizer.tokenize(' '.join(s1['token']))
        l0 = [sen1.index(x) + 1 for x in self.marker_tokens]
        l1 = [sen2.index(x) + 1 for x in self.marker_tokens]
        return Instance({'sen1': TextField(sen1, self._token_indexers), 'sen2': TextField(sen2, self._token_indexers), 'loc': ListField([MetadataField(l0), MetadataField(l1)]), 'label': LabelField(data[2], skip_indexing=True)})  # , 'sentence': TextField(' '.join(data['token']))})}

@DatasetReader.register("siamese_ner_reader")
class SiameseNerReader(SiameseReader):
    def __init__(self,
                 bert_model: str,
                 lazy: bool = False,
                 tokenizer: Tokenizer = None,
                 token_indexers: Dict[str, TokenIndexer] = None, debug=False) -> None:
        super().__init__(bert_model, lazy, tokenizer, token_indexers, debug)
        self.ner_idx = create_ner_indx(NerDict('ner_dict.json').d)

    @overrides
    def text_to_instance(self, data: dict) -> Instance:  # type: ignore
        s0 = data[0]
        s1 = data[1]
        i = super().text_to_instance(data)
        return Instance({'sen1': i.fields['sen1'], 'sen2': i.fields['sen2'],
                         'loc': i.fields['loc'], 'label': i.fields['label'], 'ner': MetadataField([self.ner_idx[s0['subj_type']], self.ner_idx[s0['obj_type']], self.ner_idx[s1['subj_type']], self.ner_idx[s1['obj_type']]])})  # , 'sentence': TextField(' '.join(data['token']))})}

@DatasetReader.register("relation_ner_reader")
class RelationAndNERReader(TacredReader):
    def __init__(self,
                 bert_model: str,
                 lazy: bool = False,
                 tokenizer: Tokenizer = None,
                 token_indexers: Dict[str, TokenIndexer] = None, debug=False, mode='basic') -> None:
        super().__init__(bert_model, lazy, tokenizer, token_indexers, debug)

        self.ner_idx = create_ner_indx(NerDict('ner_dict.json').d)

    @overrides
    def text_to_instance(self, data: dict) -> Instance:  # type: ignore
        data = add_word_markers(data)
        tokenized_tokens = self._tokenizer.tokenize(' '.join(data['token']))
        field_of_tokens = TextField(tokenized_tokens, self._token_indexers)
        marker_loc = [tokenized_tokens.index(x) + 1 for x in self.marker_tokens]  # +1 for the ['cls'] token
        return Instance({'sen': field_of_tokens, 'loc': MetadataField(marker_loc),
                         'label': LabelField(data['relation']), 'ner': MetadataField([self.ner_idx[data['subj_type']], self.ner_idx[data['obj_type']]])})

@DatasetReader.register("multi_ner_reader")
class MultiNERReader(TacredReader):
    def __init__(self,
                 bert_model: str,
                 lazy: bool = False,
                 tokenizer: Tokenizer = None,
                 token_indexers: Dict[str, TokenIndexer] = None, debug=False) -> None:
        super().__init__(bert_model, lazy, tokenizer, token_indexers, debug)
        self.ner_dict = NerDict('ner_dict.json')

    @overrides
    def text_to_instance(self, data: dict) -> Instance:  # type: ignore
        sens = []
        base_tokens = data['token']
        sens.append(add_word_markers(data)['token'])
        data['token'] = base_tokens
        sens.append(add_ner_markers(data, self.ner_dict)['token'])
        data['token'] = base_tokens
        sens.append(add_mask_markers(data)['token'])

        markers = []
        texts = []
        for s in sens:
            tokenized_tokens = self._tokenizer.tokenize(' '.join(s))
            texts.append(TextField(tokenized_tokens, self._token_indexers))
            markers.append(MetadataField([tokenized_tokens.index(x) + 1 for x in self.marker_tokens]))  # +1 for the ['cls'] token

        return Instance({'sen0': texts[0], 'sen1': texts[1], 'sen2': texts[2], 'loc0': markers[0], 'loc1': markers[1], 'loc2': markers[2],
                         'label': LabelField(data['relation'])})  # , 'sentence': TextField(' '.join(data['token']))})

def create_ner_indx(d):
    return {r: int(re.findall(r'\d+', v)[0]) - 6 for r,v in d.items()}

# @DatasetReader.register("relation_ner_reader")
# class RelationAndNERReader(TacredReader):
#     def __init__(self,
#                  bert_model: str,
#                  lazy: bool = False,
#                  tokenizer: Tokenizer = None,
#                  token_indexers: Dict[str, TokenIndexer] = None, debug=False, mode='basic') -> None:
#         super().__init__(bert_model, lazy, tokenizer, token_indexers, debug)
#
#         self.ner_idx = create_ner_indx(NerDict('ner_dict.json').d)
#
#     @overrides
#     def text_to_instance(self, data: dict) -> Instance:  # type: ignore
#         data = add_word_markers(data)
#         tokenized_tokens = self._tokenizer.tokenize(' '.join(data['token']))
#         field_of_tokens = TextField(tokenized_tokens, self._token_indexers)
#         marker_loc = [tokenized_tokens.index(x) + 1 for x in self.marker_tokens]  # +1 for the ['cls'] token
#         return Instance({'sen': field_of_tokens, 'loc': MetadataField(marker_loc),
#                          'label': LabelField(data['relation']), 'ner': MetadataField([self.ner_idx[data['subj_type']], self.ner_idx[data['obj_type']]])})