from abc import ABC

from metadata_validation_conversion.celery import app
from metadata_validation_conversion.helpers import send_message
from .ElixirValidatorResults import ElixirValidatorResults
from .JoinedResults import JoinedResults
from .RelationshipsIssues import RelationshipsIssues
from .WarningsAndAdditionalChecks import WarningsAndAdditionalChecks
from .helpers import get_submission_status

from celery import Task
from metadata_validation_conversion.constants import SAMPLES_ALLOWED_SPECIAL_SHEET_NAMES
from .update_utils import check_biosampleid

class LogErrorsTask(Task, ABC):
    abstract = True

    def on_failure(self, exc, task_id, args, kwargs, einfo):
        send_message(room_id=kwargs['room_id'], validation_status='Error',
                     errors=f'There is a problem with the validation process. Error: {exc}')

@app.task(base=LogErrorsTask)
def verify_sample_ids(json_to_test, room_id):
    """
    This task verifies the biosample id provided for the update process
    :param room_id: room id to create ws url
    :param json_to_test: dictionary containing submitted worksheet data
    :return: valid and invalid sample ids
    """
    erroneous_sample_ids = list()
    reg_valid_sample_ids = list()
    for sheetname, submitted_data in json_to_test.items():
        if sheetname not in SAMPLES_ALLOWED_SPECIAL_SHEET_NAMES:
            for ele in submitted_data:
                # check biosample_id of entry
                try:
                    sample_id = ele['custom']['biosample_id']['value']
                    reg_valid_sample_ids, erroneous_sample_ids = check_biosampleid(sample_id.upper(),
                                                                                reg_valid_sample_ids,
                                                                                erroneous_sample_ids)
                except KeyError:
                    send_message(validation_status='Error', room_id=room_id,
                                 errors=f'There is a problem with the validation process. '
                                        f'sample_id missing in sheet {sheetname}')
                    return

                # check biosample_id of derived_from
                # derived_from is not a mandatory field, pass if key is missing
                try:
                    derived_from = ele['derived_from']

                    if isinstance(derived_from, list):
                        for entry in derived_from:
                            reg_valid_sample_ids, erroneous_sample_ids = check_biosampleid(entry['value'].upper(),
                                                                                reg_valid_sample_ids,
                                                                                erroneous_sample_ids)
                    if isinstance(derived_from, dict):
                        reg_valid_sample_ids, erroneous_sample_ids = check_biosampleid(derived_from['value'].upper(),
                                                                                reg_valid_sample_ids,
                                                                                erroneous_sample_ids)
                except KeyError:
                    pass

    return reg_valid_sample_ids, erroneous_sample_ids

@app.task(base=LogErrorsTask)
def validate_against_schema(json_to_test, rules_type, structure, room_id):
    """
    Task to send json data to elixir-validator
    :param json_to_test: json to test against schema
    :param rules_type: type of rules to validate
    :param structure: structure of original template
    :return: all issues in dict
    """
    elixir_validation_results = ElixirValidatorResults(json_to_test,
                                                       rules_type,
                                                       structure)
    return elixir_validation_results.run_validation()

@app.task(base=LogErrorsTask)
def collect_warnings_and_additional_checks(json_to_test, rules_type,
                                           structure, room_id):
    """
    Task to do additional checks inside python app
    :param json_to_test: json to test against additional checks
    :param rules_type: type of rules to validate
    :param structure: structure of original template
    :return: all issues in dict
    """
    additional_checks_object = WarningsAndAdditionalChecks(json_to_test,
                                                           rules_type,
                                                           structure)
    return additional_checks_object.collect_warnings_and_additional_checks()

@app.task(base=LogErrorsTask)
def collect_relationships_issues(json_to_test, validation_type, structure, action, room_id):
    """
    This task will do relationships check
    :param json_to_test: json to be tested
    :param structure: structure of original template
    :return: all issues in dict
    """
    relationships_issues_object = RelationshipsIssues(json_to_test, validation_type, structure, action)
    return relationships_issues_object.collect_relationships_issues()

@app.task(base=LogErrorsTask)
def join_validation_results(results, room_id):
    """
    This task will join results from previous two tasks
    :param room_id: room id to create ws url
    :param results: list with results of previous two tasks
    :return: joined issues in dict
    """
    joined_results_object = JoinedResults(results)
    results = joined_results_object.join_results()
    submission_status = get_submission_status(results)
    send_message(validation_status='Finished', room_id=room_id,
                 table_data=results, submission_status=submission_status)
    return results


import tensorflow as tf
from tensorflow import keras
import numpy as np

# Return Gram matrix (all possible dot products)
def gram_matrix(mat):
    channels = int(mat.shape[-1])
    a = tf.reshape(mat, [-1, channels])
    gram = tf.matmul(a, a, transpose_a=True)
    return gram

# Convert RGB to grayscale
def rgb2gray(rgb):
    return np.dot(rgb[..., :3], [0.2989, 0.5870, 0.1140])

# Get Watermark dataset from CIFAR10 dataset, shape=(number, 32, 32, 1)
def get_watermark_dataset(number):
    (x_train, y_train), (_, _) = keras.datasets.cifar10.load_data()

    dataset = x_train.astype('float32') / 255.
    dataset = rgb2gray(dataset)
    dataset = dataset[:-50000+number]  # to number images
    dataset = dataset.reshape((number, 32, 32, 1))
    return np.array(dataset)

# Get Coverimage dataset from ImageNET dataset, shape=(number, 128, 128, 3)
def get_coverimage_dataset(number, path_to_dataset):
    images = tf.keras.preprocessing.image_dataset_from_directory(path_to_dataset, image_size=(128, 128),
                                                                 shuffle=True, batch_size=1
                                                                 )
    dataset = []
    for element in images.as_numpy_iterator():
        dataset.append(element[0] / 255)

    dataset = np.array(dataset)
    dataset = dataset.reshape((len(images), 128, 128, 3))
    dataset = dataset[:-len(images)+number]
    return dataset


import tushare as ts

class DataRetriever:
    def __init__(self, token, code, start, end, file_name):
        ts.set_token(token=token)
        self.api = ts.pro_api()
        self.code = code
        self.start = start
        self.end = end
        self.file_name = file_name

    def retrieve_data(self):
        data = self.api.daily(ts_code=self.code, start_date=self.start, end_date=self.end)
        return data

    def create_json_file(self):
        json_str = self.retrieve_data().to_json(orient='index', force_ascii=False)
        with open(self.file_name, 'w') as json_file:
            json_file.write(json_str)


import plotly.offline as py
import plotly.graph_objs as go

# Create random data with numpy
Y = []

file = open('EnVStime.txt', 'r')
# file.readline()
# file.readline()
X = [int(i) for i in range(1, 7000)]
i = 0
for line in file:
    row = line.split()
    Y.append(row[0])
file.close()
# Create a trace
trace = go.Scattergl(
x = X,
y = Y,
mode = 'lines+markers',
marker = dict(size = 3)

)

data = [trace]

py.plot(data, filename='EnVStime')


input = __import__('sys').stdin.readline
if __name__ == "__main__":
    a = input().strip()
    b = input().strip()
    a = ' '+a
    b = ' '+b
    len_a, len_b = len(a), len(b)
    dp = [['']*(len_a) for _ in range(len_b)]
    max_len = 0
    max_str = ''
    for i in range(1, len_b):
        for j in range(1, len_a):
            if b[i] == a[j]:
                if len(dp[i-1][j-1])+1 > len(dp[i-1][j]) and len(dp[i-1][j-1])+1 > len(dp[i][j-1]):
                    dp[i][j] = dp[i-1][j-1] + a[j]
                else:
                    if len(dp[i - 1][j]) > len(dp[i][j - 1]):
                        dp[i][j] = dp[i - 1][j]
                    else:
                        dp[i][j] = dp[i][j - 1]
            else:
                if len(dp[i-1][j]) > len(dp[i][j-1]):
                    dp[i][j] = dp[i-1][j]
                else:
                    dp[i][j] = dp[i][j-1]
            if max_len < len(dp[i][j]):
                max_len = len(dp[i][j])
                max_str = dp[i][j]
    print(max_len)
    print(max_str)


from metaflow import FlowSpec,card,step,batch,current,Parameter

CHART_OPTIONS = [
    {
        "caption":"This is a first dummy chart", # Caption of the chart
        "x_key" : "x_1",  # The key to match in Task object
        "y_key" : "y_1",  # The key to match in Task object
        "xlabel": "some x label",
        "ylabel": "some y label",
        "chart_type":"line",
        "id" : "cid1",
    },
    {
        "caption":"This is a second dummy chart", # Caption of the chart
        "x_key" : "x_2",  # The key to match in Task object
        "y_key" : "y_2",  # The key to match in Task object
        "xlabel": "some x label",
        "ylabel": "some y label",
        "chart_type":"line",
        "id" : "cid2",
    },
]
TABLES = [
    {
        "heading":"Model Metadata",
        "cells" : [
            {
                "name":"Path of model : (S3 path to the model weights)", # Name is the name of the property to show in the table.
                "key" : "s3_url"  # The key to match in Task object
            },
            {
                "name":"Wandb dashboard URL",
                "key" :"wandb_url"
            },
            {
                "name":"Job Execution Meduim", #
                "key": "exec_medium"
            }
        ]
    },
]

IMAGES = [
   {
        "caption":"Image From Remote URL", # Caption for the image.
        # The "key" is the key to match in Task object to retrieve the image.
        "key" : "",
        # The artifact Actual path to the image.
        # This can be a remote path
        "path_key": "remote_image"
    },
    {
        "caption":"Image from a Metaflow artifact",
        "key" : "random_image",
        "path_key": ""
    },
]
class CardPipelineFlow(FlowSpec):
    num_rows = Parameter('num-rows',default = 1000000,type=int,help='The number of rows from the dataset to use for Training.')

    batch_size = Parameter('batch-size',default = 64,type=int,help='Batch size to use for training the model.')

    browsing_path_parquet = Parameter(
        'browsing-path-parquet',\
        envvar="BROWSING_PATH_PARQUET",\
        default=None,type=str,help='Path to the browsing.parquet file in S3. This file contains browsing related data in the Coveo Challenge'
    )

    sku_path_parquet = Parameter(
        'sku-path-parquet',\
        envvar="SKU_PATH_PARQUET",\
        default=None,type=str,help='Path to the sku_to_content.parquet files in S3; This file contains unique productids.'
    )

    max_epochs = Parameter(
        'max-epochs',\
        envvar="MAX_EPOCHS",\
        default=1,type=int,help='Maximum number of epochs to train model.'
    )

    num_gpus = Parameter(
        'num-gpus',
        envvar="NUM_GPUS",\
        default=0,type=int,help='Number of GPUs to use when training the model.'
    )

    @step
    def start(self):
        self.next(self.train)

    # @batch(cpu=4,memory=30000,gpu=1,image='valayob/coveo-challenge-flow-image:0.4')
    @card(type='coveo_data_card',\
        options={\
            "charts": CHART_OPTIONS,\
            "tables": TABLES,\
            "images":IMAGES,\
            "show_parameters":True
        },\
        id='testcard')
    @step
    def train(self):
        import random
        from metaflow import current
        import numpy as np
        self.params = current.parameter_names
        print(self.params)
        self.wandb_url= "<WANDBURL COMES HERE>"
        self.model_wieghts_path= "<MODEL WEIGHTS COMES HERE>"
        self.exec_medium = "local"
        self.random_image = np.random.randn(1024,768).tolist()
        self.remote_image = "https://picsum.photos/1024/768"
        self.y_1 = np.random.randn(10).tolist()
        self.x_1 = [i for i in range(1,10)]
        self.y_2 = [random.randint(0,10) for _ in range(10)]
        self.x_2 = [i for i in range(1,10)]

        self.next(self.end)

    @step
    def end(self):
        print("Done Computation")

if __name__ == "__main__":
    CardPipelineFlow()


import pandas as pd # 一般的にpandasはpdと名前を付けてimportされる
#まず2つのCSVを読み込む
df=pd.read_csv('student.csv')
df2=pd.read_csv('universities.csv')
#dfは学校IDをキーに，df2はIDをキーに内部結合
df_merged=pd.merge(df,df2,left_on='学校ID',right_on='ID')
#print(df_merged)
#how='left'を指定#左外部結合#左外部結合したやつをｃｓｖで保存
df_merged=pd.merge(df,df2,left_on='学校ID',right_on='ID',how='left')
#print(df_merged)
#print(df.iloc[:, :])
#print(df.iloc[:3, :2])
#print(df.iloc[1:3, 1:2])#1行以上、3行未満#1列以上、2列未満
df3=df2.iloc[2:10, 1:4]
print(df3)


import xlrd
import os
from twilio.rest import Client
from client import clientInfo

loc = "/Users/dylanduhamel/CODE/VSCodeProjects/Python/Book_Automator/src/3.xls"
clientList = []

wb = xlrd.open_workbook(loc)
sheet = wb.sheet_by_index(0)
sheet.cell_value(0, 0)

for i in range(2, sheet.nrows -4, 2):
     name = sheet.cell_value(i, 0).lower()
     bookID = sheet.cell_value(i + 1, 0).lower()
     balance = int(sheet.cell_value(i, 12))
     clientList.append(clientInfo(bookID, name, balance))

for obj in clientList:
     print(obj.messageFormat())

# client = Client("AC2a01d84e68aa71949fc83eccd0aaa546", "953fd2f2c3004dff70f128fc5ebab45f")

# message = client.messages \
#                 .create(
#                      body=clientList[13].messageFormat(),
#                      from_='+12546153182',
#                      to='+15719699225'
#                  )


import unittest
from etl.coinapi import CoinApiETL
from etl.initialize_database import CryptoDataDownloadETL
import pandas as pd

class CoinApiETLTest(unittest.TestCase):

    def setUp(self):
        self.capi = CoinApiETL()

    def test_endpoint(self):
        self.assertEqual(self.capi.endpoint,
                         "https://rest.coinapi.io/v1/ohlcv/GEMINI_SPOT_BTC_USD/history")

    def test_request_parameters(self):
        self.assertEqual(dict(
            period_id='1HRS',
            time_start='2019-07-01T01:00:00',
            time_end='2019-07-02T00:00:00'
        ), self.capi._create_request_parameters())

    def test_transform(self):
        data = [{
            "time_period_end": "2019-01-01T00:00:00",
            "price_open": 12.0,
            "price_high": 12.0,
            "price_low": 12.0,
            "price_close": 12.0,
            "volume_traded": 100.0}]
        expected_result = [{
            "measurement": "ohlcv",
            "tags": {
                "asset": "btc"
            },
            "time": "2019-01-01T00:00:00",
            "fields": {
                    "open": 12.0,
                    "high": 12.0,
                    "low": 12.0,
                    "close": 12.0,
                    "volume": 100.0
            }
        }]
        self.assertEqual(
            expected_result,
            self.capi.transform(data)
        )

class CryptoDataDownloadETLTest(unittest.TestCase):

    def setUp(self):
        self.cdd = CryptoDataDownloadETL()

    def test_transform(self):
        data = pd.DataFrame([{
            "Date": "2019-01-01T00:00:00",
            "Open": 12.0,
            "High": 12.0,
            "Low": 12.0,
            "Close": 12.0,
            "Volume": 100.0}])
        expected_result = [{
            "measurement": "ohlcv",
            "tags": {
                "asset": "btc"
            },
            "time": "2019-01-01T00:00:00",
            "fields": {
                    "open": 12.0,
                    "high": 12.0,
                    "low": 12.0,
                    "close": 12.0,
                    "volume": 100.0
            }
        }]
        self.assertEqual(
            expected_result,
            self.cdd.transform(data)
        )


import json
import sys,getopt

def main(argv):
    server=""
    portal="My default portal"
    razorize=""
    verbose=True

    def help():
        print('Usage: hey -s SERVER [-p PORTAL] -r [-v] [-h]')
        print('dkgjjgkjgkjg kfgjdkfjgk dfkgd kfkgjkgj')
        print('   -s, --server		Mandatory. my BIG server')
        print('   -p, --portal		my small portal')
        print('   -r, --razorize		Mandatory. whether razorizer or let it be')
        print('   -v, --verbose		verbosion mode god')
        print('   -h, --help		Display help and exit')
    try:
        opts, args = getopt.getopt(argv,"s:p:rv",["server=","portal="])
    except getopt.GetoptError:
        help()
        sys.exit(2)

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            help()
            sys.exit()
        elif opt in ("-s", "--server"):
         server = arg
        elif opt in ("-p", "--portal"):
         portal = arg
        elif opt in ("-r", "--razorize"):
         razorize = arg
        elif opt in ("-v", "--verbose"):
         verbose = arg

    if server == "":
        print("Missing parameter server")
        help()
        sys.exit(2)
    elif razorize == "":
        print("Missing parameter razorize")
        help()
        sys.exit(2)

if __name__ == "__main__":
    main(sys.argv[1:])


from tkinter import *
from tkinter import messagebox
from socket import *
from threading import Thread

player = 1
turn = 0

def win(winner):
    messagebox.showinfo('Tic Tac Toe', winner+" wins!!" )
    root.destroy()

def check():

    b1=bt1['text']
    b2=bt2['text']
    b3=bt3['text']
    b4=bt4['text']
    b5=bt5['text']
    b6=bt6['text']
    b7=bt7['text']
    b8=bt8['text']
    b9=bt9['text']

    if (b1==b2 and b2==b3 and b1 =='O') or (b1==b2 and b2==b3 and b1 =='X'):
        bt1.config(bg="green")
        bt2.config(bg="green")
        bt3.config(bg="green")
        win(b1)
    if (b4==b5 and b5==b6 and b4 =='O') or (b4==b5 and b5==b6 and b4 =='X'):
        bt4.config(bg="green")
        bt5.config(bg="green")
        bt6.config(bg="green")
        win(b4)
    if (b7==b8 and b8==b9 and b7 =='O') or (b7==b8 and b8==b9 and b7 =='X'):
        bt7.config(bg="green")
        bt8.config(bg="green")
        bt9.config(bg="green")
        win(b7)
    if (b1==b4 and b4==b7 and b1 =='O') or (b1==b4 and b4==b7 and b1 =='X'):
        bt1.config(bg="green")
        bt4.config(bg="green")
        bt7.config(bg="green")
        win(b1)
    if (b2==b5 and b5==b8 and b2 =='O') or (b2==b5 and b5==b8 and b2 =='X'):
        bt2.config(bg="green")
        bt5.config(bg="green")
        bt8.config(bg="green")
        win(b2)
    if (b3==b6 and b6==b9 and b3 =='O') or (b3==b6 and b6==b9 and b3 =='X'):
        bt3.config(bg="green")
        bt6.config(bg="green")
        bt9.config(bg="green")
        win(b3)
    if (b1==b5 and b5==b9 and b1 =='O') or (b1==b5 and b5==b9 and b1 =='X'):
        bt1.config(bg="green")
        bt5.config(bg="green")
        bt9.config(bg="green")
        win(b1)
    if (b3==b5 and b5==b7 and b3 =='O') or (b3==b5 and b5==b7 and b3 =='X'):
        bt3.config(bg="green")
        bt5.config(bg="green")
        bt7.config(bg="green")
        win(b3)
    elif turn ==5:
        messagebox.showinfo('Tic Tac Toe', "its a tie")

def clicked1():
    global player
    global turn
    if (bt1['text']==" "):
        if (player == 1):
            player=2
            turn += 1
            bt1['text']='O'
            send_play(1)
        check()

def clicked2():
    global player
    global turn
    if (bt2['text']==" "):
        if (player == 1):
            player=2
            turn += 1
            bt2['text']='O'
            send_play(2)
        check()

def clicked3():
    global player
    global turn
    if (bt3['text']==" "):
        if (player == 1):
            player=2
            turn += 1
            bt3['text']='O'
            send_play(3)
        check()

def clicked4():
    global player
    global turn
    if (bt4['text']==" "):
        if (player == 1):
            player=2
            turn += 1
            bt4['text']='O'
            send_play(4)
        check()

def clicked5():
    global player
    global turn
    if (bt5['text']==" "):
        if player == 1:
            player=2
            turn += 1
            bt5['text']='O'
            send_play(5)
        check()

def clicked6():
    global player
    global turn
    if (bt6['text']==" "):
        if( player == 1):
            player=2
            turn += 1
            bt6['text']='O'
            send_play(6)
        check()

def clicked7():
    global player
    global turn
    if (bt7['text']==" "):
        if (player == 1):
            player=2
            turn += 1
            bt7['text']='O'
            send_play(7)
        check()

def clicked8():
    global player
    global turn
    if (bt8['text']==" "):
        if (player == 1):
            player=2
            turn += 1
            bt8['text']='O'
            send_play(8)
        check()

def clicked9():
    global player
    global turn
    if (bt9['text']==" "):
        if (player == 1):
            player=2
            turn += 1
            bt9['text']='O'
            send_play(9)
        check()

def send_play(n):
    n = str(n)
    n = n.encode()
    c.send(n)

def handle_play(n):
    global player
    n = n-1
    button_list [n]["text"] = "X"
    player = 1

def apply_play(p):
    p = p.decode()
    p = int(p)
    handle_play(p)

root = Tk()

root.title('Server: tic tac toe')

lb1 = Label(root, text='player1: O', font=('Helvetica','15'))
lb1.grid(row=0, column=0)

button_list = list()

bt1=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked1)

bt2=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked2)

bt3=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked3)

bt4=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked4)

bt5=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked5)

bt6=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked6)

bt7=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked7)

bt8=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked8)

bt9=Button(root, text=" ",width = 6, height = 3,font=('Helvetica',20),command = clicked9)

bt1.grid(row = 0, column=1)
bt2.grid(row = 0, column=2)
bt3.grid(row = 0, column=3)

bt4.grid(row = 1, column=1)
bt5.grid(row = 1, column=2)
bt6.grid(row = 1, column=3)

bt7.grid(row = 2, column=1)
bt8.grid(row = 2, column=2)
bt9.grid(row = 2, column=3)

button_list.append(bt1)
button_list.append(bt2)
button_list.append(bt3)
button_list.append(bt4)
button_list.append(bt5)
button_list.append(bt6)
button_list.append(bt7)
button_list.append(bt8)
button_list.append(bt9)

s = socket(AF_INET,SOCK_STREAM)

s.bind(('192.168.87.1', 5050))
s.listen(5)
c=None

def handle_client():
    global player
    global c
    player = 1
    c,ad = s.accept()
    receive = Thread(target = receive_message, args = [c,])
    receive.start()

def receive_message(c):
    while True:
        p = c.recv(10)
        apply_play(p)

acc = Thread(target=handle_client)
acc.start()

root.mainloop()


from pprint import pprint
import numpy as np
import pandas as pd

import json
import sys
import os
import xml
import time

import pyspark
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, StringType, LongType, IntegerType, FloatType

import copy
import uuid

spark = (pyspark.sql.SparkSession.builder.getOrCreate())

userSchema = StructType([
    StructField("id", StringType(), False),
    StructField("name", StringType(), False),
])
user_data = [
    ("1", "alisa"),
    ("1", "doom"),
    ("2", "grande"),
    ("3", "alex"),
    ("4", "john"),
    ("5", "mia")
]
user_df = spark.createDataFrame(data=user_data, schema=userSchema)
user_df.createOrReplaceTempView("user")

userBuySchema = StructType([
    StructField("user_id", StringType(), False),
    StructField("item_id", StringType(), False),
])
userBuy_data = [
    ("1", "2"),
    ("1", "4"),
    ("3", "5"),
    ("6", "4")
]
userBuy_df = spark.createDataFrame(data=userBuy_data, schema=userBuySchema)
userBuy_df.createOrReplaceTempView("userBuy")

spark.sql("""
select *
from user as u left join userBuy as ub on ub.user_id = u.id
""").show()

spark.sql("""
select *
from user as u left join userBuy as ub on u.id = ub.user_id
""").toPandas().to_csv("./out1.csv", index=False)

spark.sql("""
select *
from user as u inner join userBuy as ub on u.id = ub.user_id
""").show()
# .toPandas().to_csv("./out2.csv", index=False)


import numpy as np
from joblib import dump
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import recall_score, precision_score, roc_auc_score
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB

from service.constants import RANDOM_STATE, TEST_SET_SIZE, MIN_TEST_ITEMS_COUNT, VECTORIZER_PATH_TEMPLATE, \
    VECTORIZER_PARAMS_PATH_TEMPLATE, MODEL_PATH_TEMPLATE

TEXT_COLUMN_KEY = 'text'
SCORE_COLUMN_KEY = 'score'

def train_model(df, working_directory):
    if TEXT_COLUMN_KEY not in df.columns or SCORE_COLUMN_KEY not in df.columns:
        raise Exception(f'Data should contain "{TEXT_COLUMN_KEY}" and "{SCORE_COLUMN_KEY}" columns')

    if len(df) == 0:
        raise Exception('Data should contain at least 1 entry')

    path_vectorizer = VECTORIZER_PATH_TEMPLATE.substitute(working_directory=working_directory)
    path_vectorizer_params = VECTORIZER_PARAMS_PATH_TEMPLATE.substitute(working_directory=working_directory)
    path_model = MODEL_PATH_TEMPLATE.substitute(working_directory=working_directory)

    labels = (df['score'] >= 3).astype(np.int32).values

    min_df = min(round(len(df) * 0.0001), 1000)  # not greater when 1000
    vectorizer = TfidfVectorizer(
        strip_accents='unicode',
        lowercase=True,
        stop_words='english',
        token_pattern=r'(?u)(\b[a-z]{2,}\b|[\u263a-\U0001f645])',
        ngram_range=(1, 2),
        min_df=min_df,
        binary=True
    )
    dump(vectorizer, path_vectorizer)

    text_vectorized = vectorizer.fit_transform(df['text'])

    vectorizer_params = {
        'vocabulary_': vectorizer.vocabulary_,
        'idf_': vectorizer.idf_,
    }
    dump(vectorizer_params, path_vectorizer_params)

    if len(df) * TEST_SET_SIZE >= MIN_TEST_ITEMS_COUNT:
        x_train, x_test, y_train, y_test = train_test_split(text_vectorized, labels, test_size=TEST_SET_SIZE,
                                                            random_state=RANDOM_STATE)
        test_dataset_type = 'test'
    else:
        x_train, x_test, y_train, y_test = text_vectorized, text_vectorized, labels, labels
        test_dataset_type = 'train'

    model = MultinomialNB()
    model.fit(x_train, y_train)
    dump(model, path_model)

    prediction_probas = model.predict_proba(x_test)
    predictions = np.argmax(prediction_probas, axis=1)
    recall = recall_score(y_test, predictions)
    precision = precision_score(y_test, predictions)
    roc_auc = roc_auc_score(y_test, prediction_probas[:, 1])
    result = f'Model has been trained successfully!\nTested on {test_dataset_type} dataset of size: {x_test.shape[0]}\nrecall: {recall}\nprecision: {precision}\nroc-auc: {roc_auc}'

    return result


"""
This is a class of a rect which is contained in the haar feature.

Contains the coordinates of the rect, the width and the height
"""

class Rect:
    def __init__(self, x, y, width, height, weight):
        self.x = x
        self.y = y
        self.width = width
        self.height = height
        self.weight = weight

    def calc_rect_value(self, mini_grid):
        #use integral image to calc the value
        #returns the value of the rect

        w = mini_grid[int(self.x) - 1][int(self.y) - 1] if (int(self.x) > 0 and int(self.y) > 0) else 0
        x = mini_grid[int(self.x) + int(self.width) - 1][int(self.y) - 1] if (int(self.y) > 0) else 0
        y = mini_grid[int(self.x) - 1][int(self.y) + int(self.height) - 1] if (int(self.x) > 0) else 0
        z = mini_grid[int(self.x) + int(self.width) - 1][int(self.y) + int(self.height) - 1]
        value = ( z - x - y + w) / (self.width * self.height)#returns the average sum of the rectangle
        return value


from django.urls import path

from .views import (CreateRateGenerics, DetailCarGenerics, ListCarGenerics,
                    PopularCarGenerics)

urlpatterns = [
    path("cars/", ListCarGenerics.as_view()),
    path("cars/<int:pk>/", DetailCarGenerics.as_view()),
    path("rate/", CreateRateGenerics.as_view()),
    path("popular/", PopularCarGenerics.as_view()),
]


"""
The filter() function returns an iterator were the items are filtered through a function to test if the item is accepted or not.
filter(function, iterable)
def abir(n):
    if n % 2 != 0:
        return n

num = [1, 2, 3]
num1 = []

for i in num:
    num1.append(abir(i))
print(num1)

"""

num = [1, 2, 3, 4]

g = lambda x: (x % 2) != 0

print(list(filter(g, num)))


import os

import fitz
import openpyxl

import config_module
import exit_module
import global_data_module

SOURCE_EXCEL = 'sourceExcel'
SOURCE_EXCEL_FILE_NAME_COLUMN = 'sourceExcelFileNameColumn'
SOURCE_EXCEL_START_ROW = 'sourceExcelStartRow'
SOURCE_PDF = 'sourcePdf'
IMAGE_SCALE = 'scale'
OUTPUT_DIR = 'outputDir'

def read_asin_data(excel_path: str) -> []:
    workbook = openpyxl.load_workbook(excel_path)
    row_index = 0
    name_list = []
    for row in workbook.worksheets[0].rows:
        if row[0].value is not None and row_index >= config_module.get_config_obj()[SOURCE_EXCEL_START_ROW]:
            if row[0].value.strip() != '':
                name_list.append(row[config_module.get_config_obj()[SOURCE_EXCEL_FILE_NAME_COLUMN]].value)
        row_index = row_index + 1
    return name_list

print('正在初始化程序数据...')
global_data_module.init()
print('正在解析配置文件并读取Excel...')
# 读取Excel
file_names = read_asin_data(config_module.get_config_obj()[SOURCE_EXCEL])
print('正在解码PDF...')
# 读取PDF
pdf_doc = fitz.open(config_module.get_config_obj()[SOURCE_PDF])
print('PDF页数：' + str(pdf_doc.pageCount) + ', EXCEL 记录数：' + str(len(file_names)))
if pdf_doc.pageCount != len(file_names):
    exit_module.tip_and_wait_then_exit('Excel的记录数量与PDF页数不匹配，无法继续操作')
mat = fitz.Matrix(config_module.get_config_obj()[IMAGE_SCALE], config_module.get_config_obj()[IMAGE_SCALE])
if not os.path.exists(config_module.get_config_obj()[OUTPUT_DIR]):
    os.makedirs(config_module.get_config_obj()[OUTPUT_DIR])
for page_index in range(pdf_doc.pageCount):
    pdf_page = pdf_doc[page_index]
    print('开始转换写出第 ' + str(page_index + 1) + ' / ' + str(len(file_names)) + ' 页图片数据')
    pdf_page.getPixmap(matrix=mat).writePNG(
        config_module.get_config_obj()[OUTPUT_DIR] + os.sep + file_names[page_index] + '.png')

print('任务执行完毕')


# Part 1 and Part 2
# Zheng Lin Lei

from pathlib import Path
import time

SCRIPT_DIR = Path(__file__).parent
INPUT_FILE = Path(SCRIPT_DIR, "../input.txt")

def main():
    with open(INPUT_FILE, mode="rt") as f:
        elf_meals = f.read().split("\n\n") # split on empty lines

    elf_calories = [] # store total calories for each elf
    for elf in elf_meals:
        calories = sum(map(int, elf.splitlines()))
        elf_calories.append(calories)

    print(f"Part 1: {max(elf_calories)}")

    elf_calories = sorted(elf_calories)
    print(f"Part 2: {elf_calories[-3:]}")
    print(f"Part 2: {sum(elf_calories[-3:])}")

if __name__ == "__main__":
    t1 = time.perf_counter()
    main()
    t2 = time.perf_counter()
    print(f"Execution time: {t2 - t1:0.4f} seconds")


import requests
import json
from stream import Stream

game_to_id = {}

def prepare_game_id_cache(games):
    for game in games:
        if game in game_to_id:
            continue

        payload = {
            'q': game,
            'liveonly': True
        }

        r = requests.get('https://www.hitbox.tv/api/games', params=payload)
        if r.status_code == 200:
            data = r.json()
            for category in data.get('categories', {}):
                if category.get('category_name', '').lower() == game.lower():
                    game_to_id[game] = category['category_id']
                    break

def get_streams(games):
    prepare_game_id_cache(games)
    streams = []
    for game in games:
        game_id = game_to_id.get(game)
        if game_id is None:
            continue

        params = {
            'game': game_id,
            'liveonly': True,
            'showHidden': False,
            'publicOnly': True
        }
        r = requests.get('https://www.hitbox.tv/api/media/live/list', params=params)
        print('The hitbox API returned {} for {}'.format(r.status_code, game))
        if r.status_code == 200:
            data = r.json()
            for stream in data.get('livestream', []):
                kwargs = {
                    'url': stream.get('channel', {}).get('channel_link', ''),
                    'viewers': int(stream.get('media_views', 0)),
                    'display_name': stream.get('media_user_name', ''),
                    'game': game,
                    'status': stream.get('media_status', '')
                }

                streams.append(Stream(**kwargs))

    return streams


# from the command line, run with
# mpiexec -n 4 python -m mpi4py mpitest.py

from mpi4py import MPI
import sys
import numpy as np

size = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
name = MPI.Get_processor_name()

#sys.stdout.write(
#    "Hello, World! I am process %d of %d on %s.\n" % (rank, size, name))

# geneate a list of n random numbers between 0 and 1
n = 2000
localArray = np.random.random(n)

# sort the local array
localArray.sort()

# find the start and end of the array based on the range of 0 to 1
# and the number of processor
arrayBoundaries = []

# loop over the processors
bufferSize = n/2

arecv = np.zeros([size, int(bufferSize)], dtype='f')
for i in range(0, size):
    #  find the elements with in range
    xmin = float(i)/size
    xmax = float(i+1)/size
    asend = localArray[localArray >= xmin]
    asend = asend[asend < xmax]

    # copy this data into the send buffer
    sendbuf = np.zeros(int(bufferSize), dtype='f')
    sendbuf[0:len(asend)] = asend
    MPI.COMM_WORLD.Gather(sendbuf, arecv, root=i )

# compress the data into a local array from the gather
# ignore the zeros
newLocal = []
for i in range(0, size):
    for j in range(0, int(bufferSize)):
        if arecv[i,j] != 0:
            newLocal.append(arecv[i,j])

newLocal.sort()
#print("rank = ", rank, "local size = ", len(newLocal), "localMin = ", newLocal[0], "localMax = ", newLocal[-1])
localData = np.array( [len(newLocal), newLocal[0], newLocal[-1]] , dtype='f')
globalData = np.zeros([size, 3], dtype='f')

MPI.COMM_WORLD.Gather(localData, globalData, root=0)
if rank == 0:
    #print(globalData)
    for i in range(0, size):
        print("node = ", i, " node data size = ", globalData[i,0], "node Min = ", globalData[i,1], "node Max = ", globalData[i,2])

MPI.COMM_WORLD.Barrier()


import os
import numpy as np
import pandas as pd
import anndata as ad

from typing import Optional, Union
from adpbulk import ADPBulk
from anndata import AnnData
from tqdm import tqdm
import pkg_resources

class SingleCellTensor:
    """
    Represent tensorized single cell data

    Parameters
    ----------
    tensor: np.ndarray
        Tensor data with shape (samples, celltypes, genes).
    gene_list: List[str]
        List of genes.
    sample_list: List[str], optional
        List of sample names, default is None.
    celltype_list: List[str], optional
        List of cell types, default is None.
    region_list: List[str], optional
        List of regions, default is None.
    normalized: bool, optional
        Boolean flag indicating if the tensor is normalized, default is False.
    sample_features: pd.DataFrame, optional
        DataFrame with sample features, default is None.

    """

    def __init__(
            self,
            tensor: np.ndarray,
            gene_list: list[str],
            sample_list: Optional[list[str]] = None,
            celltype_list: Optional[list[str]] = None,
            region_list: Optional[list[str]] = None,
            normalized: bool = False,
            sample_features: Optional[pd.DataFrame] = None
    ) -> None:

        self.gene_list = gene_list
        self.sample_list = sample_list
        self.celltype_list = celltype_list
        self.region_list = region_list

        self.tensor = tensor
        self.normalized = normalized
        self.sample_features = sample_features

    @staticmethod
    def from_anndata(
            adata: ad.AnnData,
            sample_label: str,
            celltype_label: str,
            normalize: bool = True,
            variance_scale: bool = False,
            sqrt_transform: bool = False,
            scale_to: int = 1e6,
            cell_types: Optional[list[str]] = None,
            enrich_db_genes_only: bool = False,
            hgnc_approved_genes_only: bool = False,
            custom_gene_set: Optional[list[str]] = None,
            filter_gene_count: int = 10,
            enrich_dbs: list[str] = ['GO_Biological_Process_2021']  # ,'KEGG_2021_Human', 'Reactome_2022']
    ) -> 'SingleCellTensor':
        """

        Compose tensor from single cell anndata.

        Parameters
        ----------
        adata: ad.AnnData
            Single cell Anndata object.
        sample_label: str
            Sample label.
        celltype_label: str
            Cell type label.
        normalize: bool, optional
            Boolean flag indicating if the tensor should be normalized, default is True.
        variance_scale: bool, optional
            Boolean flag indicating if variance scaling should be applied to the tensor, default is False.
        sqrt_transform: bool, optional
            Boolean flag indicating if square root transform should be applied to the tensor, default is False.
        scale_to: int, optional
            Value to scale tensor to, default is 1e6.
        cell_types: List[str], optional
            List of cell types, default is None.
        enrich_db_genes_only: bool, optional
            Boolean flag indicating if only enriched genes should be included, default is False.
        hgnc_approved_genes_only: bool, optional
            Boolean flag indicating if only HGNC approved genes should be included, default is False.
        custom_gene_set: List[str], optional
            List of genes to include, default is None.
        filter_gene_count: int, optional
            Minimum gene count for filtering, default is 10.
        enrich_dbs: List[str], optional
            List of enrichment databases to use, default is ['GO_Biological_Process_2021'].

        Returns
        -------
        SingleCellTensor
            Tensor object.

        """

        adata = adata.raw.to_adata()

        adata = SingleCellTensor.filter_adata(
            adata, cell_types, celltype_label, custom_gene_set, enrich_db_genes_only,
            enrich_dbs, filter_gene_count, hgnc_approved_genes_only
        )

        adpb = ADPBulk(adata, [sample_label, celltype_label])
        pseudobulk_matrix = adpb.fit_transform()
        sample_meta = adpb.get_meta()

        if normalize:
            row_sums = pseudobulk_matrix.sum(axis=1)
            pseudobulk_matrix = pseudobulk_matrix / row_sums[:, np.newaxis] * scale_to

        if variance_scale:
            std = pseudobulk_matrix.std()
            std[std == 0] = 1
            pseudobulk_matrix /= std

        if sqrt_transform:
            pseudobulk_matrix = pseudobulk_matrix.apply(np.sqrt)

        tensor_sample_list = sample_meta[sample_label].unique()
        tensor_celltype_list = sample_meta[celltype_label].unique()
        tensor_gene_list = pseudobulk_matrix.columns.tolist()

        sample_meta_dict = {(row[sample_label], row[celltype_label]): row['SampleName'] for idx, row in
                            sample_meta.iterrows()}

        tensor = np.zeros((len(tensor_sample_list), len(tensor_celltype_list), len(tensor_gene_list)))

        for i in tqdm(range(len(tensor_sample_list)), desc=f"Building tensor from matrix"):
            for j in range(len(tensor_celltype_list)):
                key = (tensor_sample_list[i], tensor_celltype_list[j])
                if key in sample_meta_dict:
                    tensor[i, j, :] = pseudobulk_matrix.loc[sample_meta_dict[key], tensor_gene_list]

        sample_features = SingleCellTensor.adata_obs_to_summary_df(adata, sample_label=sample_label)

        return SingleCellTensor(tensor,
                                celltype_list=tensor_celltype_list,
                                gene_list=tensor_gene_list,
                                sample_list=tensor_sample_list,
                                normalized=normalize,
                                sample_features=sample_features
                                )

    @staticmethod
    def filter_adata(
        adata: AnnData,
        cell_types: Optional[list] = None,
        celltype_label: Optional[str] = 'celltype',
        custom_gene_set: Optional[list] = None,
        enrich_db_genes_only: bool = False,
        enrich_dbs: Optional[list] = None,
        filter_gene_count: Optional[int] = None,
        hgnc_approved_genes_only: bool = False
    ) -> AnnData:
        """
        Filter AnnData object according to specified criteria.

        Parameters
        ----------
        adata : AnnData
            An annotated data matrix.
        cell_types : list or None, optional
            List of cell types to retain, by default None.
        celltype_label : str, optional
            Name of the column in `adata.obs` where cell types are stored, by default 'celltype'.
        custom_gene_set : list or None, optional
            List of genes to retain, by default None.
        enrich_db_genes_only : bool, optional
            Whether to keep only genes that are present in at least one of the enrich databases, by default False.
        enrich_dbs : list or None, optional
            List of enrich databases to use, by default None.
        filter_gene_count : int or None, optional
            Minimum total number of UMI counts per gene to keep the gene, by default None.
        hgnc_approved_genes_only : bool, optional
            Whether to keep only HGNC-approved protein-coding genes, by default False.

        Returns
        -------
        AnnData
            The filtered AnnData object.
        """

        if cell_types is not None:
            adata = adata[adata.obs[celltype_label].isin(cell_types)].copy()

        if enrich_db_genes_only:
            go_genes = set.union(
                *[set(g for gset in SingleCellTensor.parse_gmt(db).values() for g in gset) for db in enrich_dbs])
            adata = adata[:, adata.var_names.isin(go_genes)].copy()

        if hgnc_approved_genes_only:
            hgnc_path = pkg_resources.resource_filename("scBTF", "resources/gene_sets/hgnc_complete_set_2020-07-01.txt")
            hgnc_full = pd.read_csv(hgnc_path, sep='\t', low_memory=False)
            selected_genes = hgnc_full[
                (hgnc_full.status == 'Approved') & (hgnc_full.locus_group == 'protein-coding gene')]
            adata = adata[:, adata.var_names.str.split('.').str[0].isin(selected_genes['symbol'])].copy()

        if custom_gene_set is not None:
            adata = adata[:, adata.var_names.isin(custom_gene_set)].copy()

        if filter_gene_count is not None:
            adata = adata[:, adata.X.sum(axis=0) > filter_gene_count].copy()

        return adata

    @staticmethod
    def from_anndata_with_regions(
            adata: ad.AnnData,
            sample_label: str,
            celltype_label: str,
            region_label: str,
            normalize: bool = True,
            variance_scale: bool = False,
            sqrt_transform: bool = False,
            scale_to: int = 1e6,
            cell_types: list[str] = None,
            enrich_db_genes_only: bool = False,
            hgnc_approved_genes_only: bool = False,
            custom_gene_set: list[str] = None,
            filter_gene_count: int = 10,
            enrich_dbs: list[str] = ['GO_Biological_Process_2021']  # ,'KEGG_2021_Human', 'Reactome_2022']
    ):
        """
        Compose tensor from single cell anndata

        Returns
        -------
        tensors
            dictionary with two entries raw and normalized
            each is a sample x celltype x gene tensor
        """

        adata = adata.raw.to_adata()

        adata = SingleCellTensor.filter_adata(
            adata, cell_types, celltype_label, custom_gene_set, enrich_db_genes_only,
            enrich_dbs, filter_gene_count, hgnc_approved_genes_only
        )

        adpb = ADPBulk(adata, [sample_label, celltype_label, region_label])
        pseudobulk_matrix = adpb.fit_transform()
        sample_meta = adpb.get_meta()

        if normalize:
            row_sums = pseudobulk_matrix.sum(axis=1)
            pseudobulk_matrix = pseudobulk_matrix / row_sums[:, np.newaxis] * scale_to

        if variance_scale:
            std = pseudobulk_matrix.std()
            std[std == 0] = 1
            pseudobulk_matrix /= std

        if sqrt_transform:
            pseudobulk_matrix = pseudobulk_matrix.apply(np.sqrt)

        tensor_sample_list = sample_meta[sample_label].unique()
        tensor_celltype_list = sample_meta[celltype_label].unique()
        tensor_region_list = sample_meta[region_label].unique()
        tensor_gene_list = pseudobulk_matrix.columns.tolist()

        sample_meta_dict = {(row[sample_label], row[celltype_label], row[region_label]): row['SampleName']
                            for idx, row in sample_meta.iterrows()}

        tensor = np.zeros(
            (len(tensor_sample_list), len(tensor_celltype_list), len(tensor_region_list), len(tensor_gene_list)))
        for i in tqdm(range(len(tensor_sample_list)), desc=f"Building tensor from pseudobulk matrix"):
            for j in range(len(tensor_celltype_list)):
                for k in range(len(tensor_region_list)):
                    key = (tensor_sample_list[i], tensor_celltype_list[j], tensor_region_list[k])
                    if key in sample_meta_dict:
                        tensor[i, j, k, :] = pseudobulk_matrix.loc[sample_meta_dict[key], tensor_gene_list]

        sample_features = SingleCellTensor.adata_obs_to_summary_df(adata, sample_label=sample_label)

        return SingleCellTensor(tensor,
                                celltype_list=tensor_celltype_list,
                                gene_list=tensor_gene_list,
                                sample_list=tensor_sample_list,
                                region_list=tensor_region_list,
                                normalized=normalize,
                                sample_features=sample_features
                                )

    @staticmethod
    def from_anndata_ligand_receptor(
            adata: ad.AnnData,
            sample_label: str,
            celltype_label: str,
            normalize: bool = True,
            variance_scale: bool = False,
            sqrt_transform: bool = False,
            scale_to: int = 1e6,
            cell_types: list[str] = None,
            enrich_db_genes_only: bool = False,
            custom_gene_set: list[str] = None,
            filter_gene_count: int = 10,
            enrich_dbs: list[str] = ['GO_Biological_Process_2021'],  # ,'KEGG_2021_Human', 'Reactome_2022']
            communication_score: str = 'expression_mean'
    ):
        """
        Compose tensor from single cell anndata

        Returns
        -------
        tensors
            dictionary with two entries raw and normalized
            each is a sample x celltype x gene tensor
        """
        sc_tensor = SingleCellTensor.from_anndata(
            adata=adata,
            sample_label=sample_label,
            celltype_label=celltype_label,
            normalize=normalize,
            variance_scale=variance_scale,
            sqrt_transform=sqrt_transform,
            scale_to=scale_to,
            cell_types=cell_types,
            enrich_db_genes_only=enrich_db_genes_only,
            custom_gene_set=custom_gene_set,
            filter_gene_count=filter_gene_count,
            enrich_dbs=enrich_dbs
        )

        lr_table = pd.read_csv('data/liana_consensus_LR.csv')
        lr_table = list(set([(v['source_genesymbol'], v['target_genesymbol']) for v in lr_table.T.to_dict().values()]))
        lr_table = [lr for lr in lr_table if lr[0] in sc_tensor.gene_list and lr[1] in sc_tensor.gene_list]

        tensor = np.zeros(
            (len(sc_tensor.sample_list), len(sc_tensor.celltype_list), len(sc_tensor.celltype_list), len(lr_table)))

        for d in tqdm(range(len(sc_tensor.sample_list))):
            for lr in range(len(lr_table)):
                l = sc_tensor.tensor[d, :, sc_tensor.gene_list.index(lr_table[lr][0])]
                r = sc_tensor.tensor[d, :, sc_tensor.gene_list.index(lr_table[lr][1])]
                tensor[d, :, :, lr] = SingleCellTensor.compute_ccc_matrix(l, r, communication_score)

        sc_tensor.tensor = tensor
        sc_tensor.region_list = sc_tensor.celltype_list
        sc_tensor.gene_list = [lr[0] + '_' + lr[1] for lr in lr_table]

        return sc_tensor

    @staticmethod
    def parse_gmt(gmt):
        """
        Retrieve genes that make up gene set enrichment libraries
        """
        BASE_ENRICHR_URL = "http://maayanlab.cloud"
        DEFAULT_CACHE_PATH = "resources/gene_sets"
        os.makedirs(DEFAULT_CACHE_PATH, exist_ok=True)

        def _download_libraries(libname: str):
            import requests
            ENRICHR_URL = BASE_ENRICHR_URL + "/Enrichr/geneSetLibrary"
            query_string = "?mode=text&libraryName=%s"
            response = requests.get(ENRICHR_URL + query_string % libname, timeout=None)
            if not response.ok:
                raise Exception("Error fetching gene set library, input name is correct for the organism you've set?.")
            genesets_dict = {}
            outname = "Enrichr.%s.gmt" % libname  # pattern: database.library.gmt
            gmtout = open(os.path.join(DEFAULT_CACHE_PATH, outname), "w")
            for line in response.iter_lines(chunk_size=1024, decode_unicode="utf-8"):
                line = line.strip().split("\t")
                k = line[0]
                v = map(lambda x: x.split(",")[0], line[2:])
                v = list(filter(lambda x: True if len(x) else False, v))
                genesets_dict[k] = v
                outline = "%s\t%s\t%s\n" % (k, line[1], "\t".join(v))
                gmtout.write(outline)
            gmtout.close()

            return genesets_dict

        tmpname = "Enrichr." + gmt + ".gmt"
        tempath = os.path.join(DEFAULT_CACHE_PATH, tmpname)
        if os.path.isfile(tempath):
            with open(tempath) as genesets:
                genesets_dict = {
                    line.strip().split("\t")[0]: line.strip().split("\t")[2:]
                    for line in genesets.readlines()
                }
            return genesets_dict
        else:
            return _download_libraries(gmt)

    @staticmethod
    def adata_obs_to_summary_df(adata, sample_label):
        """
        Convert adata obs into a summary dataframe with one row for each unique `sample_label`
        """
        idx = [np.where(adata.obs[sample_label] == sample)[0][0] for sample in
               adata.obs[sample_label].value_counts().index]

        summary_df = adata.obs.iloc[idx, :].sort_values(by=sample_label)
        summary_df.index = summary_df[sample_label]

        return summary_df

    @staticmethod
    def compute_ccc_matrix(prot_a_exp, prot_b_exp, communication_score='expression_product'):
        """
        adapted from Liana https://github.com/saezlab/liana
        """
        if communication_score == 'expression_product':
            communication_scores = np.outer(prot_a_exp, prot_b_exp)
        elif communication_score == 'expression_mean':
            communication_scores = (np.outer(prot_a_exp, np.ones(prot_b_exp.shape)) + np.outer(
                np.ones(prot_a_exp.shape), prot_b_exp)) / 2.
        elif communication_score == 'expression_gmean':
            communication_scores = np.sqrt(np.outer(prot_a_exp, prot_b_exp))
        else:
            raise ValueError("Not a valid communication_score")
        return communication_scores


from spinn_front_end_common.utility_models import MultiCastCommand
from spinn_front_end_common.utilities.connections import LiveEventConnection

class EthernetControlConnection(LiveEventConnection):
    """
    A connection that can translate Ethernet control messages received
    from a Population.
    """
    __slots__ = ["__translators"]

    def __init__(
            self, translator, label, live_packet_gather_label, local_host=None,
            local_port=None):
        """
        :param AbstractEthernetTranslator translator:
            The translator of multicast to control commands
        :param str label: The label of the vertex to attach the translator to
        :param str live_packet_gather_label: The label of the LPG vertex that
            this control connection will listen to.
        :param str local_host: The optional host to listen on
        :param int local_port: The optional port to listen on
        """
        super().__init__(
            live_packet_gather_label, receive_labels=[label],
            local_host=local_host, local_port=local_port)
        self.__translators = dict()
        self.__translators[label] = translator
        self.add_receive_no_time_callback(
            label, self._translate, translate_key=False)

    def add_translator(self, label, translator):
        """
        Add another translator that routes via the LPG.

        :param str label: The label of the vertex to attach the translator to
        :param AbstractEthernetTranslator translator:
            The translator of multicast to control commands
        """
        super().add_receive_label(label)
        self.__translators[label] = translator
        self.add_receive_no_time_callback(
            label, self._translate, translate_key=False)

    def _translate(self, label, key, payload=None):
        translator = self.__translators[label]
        if payload is None:
            translator.translate_control_packet(MultiCastCommand(key))
        else:
            translator.translate_control_packet(MultiCastCommand(key, payload))


import os
import re
import sys
import time
import pickle
import numpy as np
from tqdm import trange
from pypinyin import pinyin, lazy_pinyin, Style

curdir = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, curdir)
with open(os.path.join(curdir, 'input_dict_nt_max.pickle'), 'rb') as handle:
    input_dict_nt_max = pickle.load(handle)

def remove_digits(x):
    return re.sub('[0-9]', '', x)

def pinyin_augment(_pinyin_list, hasTone):

    def iterate(prior, remains):
        if len(remains)==0:
            return [prior]
        else:
            result = []
            for name in remains[0]:
                new_prior = tuple([x for x in prior]+[name])
                result+=iterate(new_prior, remains[1:])
            return result

    def add_tones(r):
        if len(r)==0:
            return []
        if len(r)==1:
            result = []
            for tone in range(1,6):
                result.append(tuple([r[0]+str(tone)]))
            return result
        result = []
        for l in add_tones(r[1:]):
            for tone in range(1,6):
                result.append(tuple([r[0]+str(tone)]+list(l)))
        return list(set(result))

    pinyin_list = remove_digits(" ".join(_pinyin_list)).split()
    pys = []
    for py in pinyin_list:
        aug = [py]
        if py[:2]=='le' and len(py)==2:
            aug.append('re')
        if py[:1]=='n' and len(py)>1:
            aug.append('l'+py[1:])
        if py[:1]=='l' and len(py)>1:
            aug.append('n'+py[1:])
        if py[:1]=='c' and len(py)>1 and py[1:2]!='h':
            aug.append('ch'+py[1:])
        if py[:1]=='z' and len(py)>1 and py[1:2]!='h':
            aug.append('zh'+py[1:])
        if py[:1]=='s' and len(py)>1 and py[1:2]!='h':
            aug.append('sh'+py[1:])
        if py[:2]=='ch' and len(py)>2:
            aug.append('c'+py[2:])
        if py[:2]=='zh' and len(py)>2:
            aug.append('z'+py[2:])
        if py[:2]=='sh' and len(py)>2:
            aug.append('s'+py[2:])
        if py[-2:] in ['in', 'en', 'on'] and len(py)>1:
            aug.append(py[:-1]+"ng")
        if py[-3:] in ['ing', 'eng', 'ong'] and len(py)>2:
            aug.append(py[:-2]+"n")
        pys.append(aug)

    result_tmp = iterate((""), pys)
    result = []
    for item in result_tmp:
        result.append(item)
    result = list(set(result))
    if hasTone:
        result_ = []
        for r in result:
            result_ = result_+add_tones(list(r))
        return list(set(result_))
    else:
        return result

class Languagle_Model:

    def __init__(self):
        self.longest = 0
        self.pinyin_nt_set = set([])
        self.longest = max([len(x) for x in input_dict_nt_max])
        for key in input_dict_nt_max:
            if len(key)==1: self.pinyin_nt_set.add(key[0])

    def add_words(self, words, vague_add=False):
        for word in words:
            _py_nt = tuple([remove_digits(x) for x in self.translate_c2p(word)])
            input_dict_nt_max[_py_nt] = (word, 1)

    def translate_c2p(self, _str, hasTone=True):
        if hasTone:
            return lazy_pinyin(_str, style=Style.TONE3)
        return lazy_pinyin(_str)

    def translate_p2c(self, _pinyin, max_length=10, stride=1):

        def helper(_pinyin, _dict):
            if len(_pinyin) == 0:
                return ["", 0]

            rs = [(" ".join(_pinyin), 0)]
            for l in range(1,min(self.longest,len(_pinyin))+1):
                if l<len(_pinyin):
                    if tuple(_pinyin[:l]) in _dict:
                        r_prev = _dict[tuple(_pinyin[:l])]
                        r_late = helper(_pinyin[l:], _dict)
                        r = (r_prev[0]+r_late[0], r_prev[1]*r_late[1])
                        rs.append(r)
                elif l==len(_pinyin):
                    if tuple(_pinyin) in _dict:
                        r = _dict[tuple(_pinyin)]
                        rs.append(r)
            return sorted(rs, key=lambda kv: kv[1], reverse=True)[0]

        _pinyin = remove_digits(" ".join(_pinyin)).split()
        _input = [x for x in _pinyin if x in self.pinyin_nt_set]

        if len(_input)<(max_length+1):
            return helper(_input, input_dict_nt_max)[0]
        fixed = []
        last = "",-1,stride
        for start in range(0, len(_input)-max_length+stride, stride):
            text, prob = helper(_input[start:start+max_length], input_dict_nt_max)
            if prob>last[1] or start>=(len(fixed)+(len(last[0])-last[2])//stride*stride):
                fixed.append(text[:stride])
                last = text,prob,stride
            else:
                fixed.append(last[0][last[2]:last[2]+stride])
                last = last[0],last[1],last[2]+stride
        fixed.append(text[stride:])
        return "".join(fixed)

if __name__=="__main__":
    lm = Languagle_Model()

    print(lm.translate_c2p("我们都有光明的未来", hasTone=True))
    print(lm.translate_c2p("我们都有光明的未来", hasTone=False))
    print("----------------------------------------")

    print(lm.translate_p2c("jin1 tian1 de tian1 qi4 zhen1 bu4 cuo4".split()))
    print(lm.translate_p2c("shuang ye hong yu er yue hua".split()))
    print(lm.translate_p2c("yu shi wo ti chu le xin de jie jue fang an".split()))


"""Methods and utilities for sampling experiences"""
from collections import defaultdict
import numpy as np

import torch

class MultiBatch:
    """Represents a multi-agent experience batch"""

    OBS = "obs"
    ACTION = "actions"
    REWARD = "rewards"
    DONE = "dones"

    def __init__(self):
        self._policy_batches = defaultdict(list)
        self._agent_episodes = None
        self._policy_map = None

    def _store_episode(self):
        for agent_id, episode in self._agent_episodes.items():
            for key in episode.keys():
                episode[key] = np.array(episode[key])

            if self._policy_map is not None:
                batch = self._policy_batches[self._policy_map[agent_id]]
            else:
                batch = self._policy_batches[agent_id]

            batch.append(episode)

    def start_episode(self, policy_map=None):
        if self._agent_episodes is not None:
            self._store_episode()

        self._agent_episodes = defaultdict(lambda: defaultdict(list))
        self._policy_map = policy_map

    def end_episode(self):
        if self._agent_episodes is not None:
            self._store_episode()

        self._agent_episodes = None
        self._policy_map = None

    def append(self, field, values):
        for agent_id, value in values.items():
            self._agent_episodes[agent_id][field].append(value)

    def step(self, obs, actions, rewards, dones, fetches):
        for agent_id in obs.keys():
            episode = self._agent_episodes[agent_id]

            episode[MultiBatch.OBS].append(obs[agent_id])
            episode[MultiBatch.ACTION].append(actions[agent_id])
            episode[MultiBatch.REWARD].append(rewards[agent_id])
            episode[MultiBatch.DONE].append(dones[agent_id])

            for key, value in fetches[agent_id].items():
                episode[key].append(value)

    def policy_batch(self, policy_id):
        return self._policy_batches[policy_id]

    def items(self):
        return self._policy_batches.items()

class FrozenPolicy:
    """Wrapper for frozen Torch policies"""

    def __init__(self, model):
        self._model = model

    def make_agent(self):

        class Agent:

            def __init__(self, model):
                self._state = model.initial_state(batch_size=1)
                self._model = model

            def act(self, obs):
                obs = torch.as_tensor(obs, dtype=torch.float32)
                obs = obs.unsqueeze(0)  # NOTE: Add batch dimension
                obs = obs.unsqueeze(0)  # NOTE: Add time dimension (for RNNs)

                action, self._state = self._model(obs, self._state)
                action.squeeze(0)  # NOTE: Remove time dimension
                action.squeeze(0)  # NOTE: Remove batch dimension

                return action.numpy(), {}

        return Agent(self._model)

    def update(self, update):
        """Dummy to make sure we are not trying to update frozen polices"""
        raise NotImplementedError("Frozen policies cannot be updated")

    @property
    def model(self):
        return self._model

def sample(env, policies, num_episodes, max_steps, policy_fn=None):
    """Generates a batch of episodes"""
    batch = MultiBatch()
    total_steps = 0

    for episode in range(num_episodes):
        obs = env.reset()
        step = 0

        agents = {}
        map = {}
        dones = {}
        for agent_id in obs.keys():
            if policy_fn is not None:
                policy_id = policy_fn(agent_id)
            else:
                policy_id = agent_id

            agents[agent_id] = policies[policy_id].make_agent()
            map[agent_id] = policy_id
            dones[agent_id] = False

        batch.start_episode(map)  # NOTE: Why do we need the policy map
        batch.append(MultiBatch.OBS, obs)

        while step < max_steps and not all(dones.values()):
            actions = {}
            fetches = {}
            for agent_id, ob in obs.items():
                actions[agent_id], fetches[agent_id] = agents[agent_id].act(ob)

            obs, rewards, dones, _ = env.step(actions)

            batch.step(obs, actions, rewards, dones, fetches)
            step += 1

        batch.end_episode()
        total_steps += step

    stats = {"mean_reward": 0}
    for policy_id, agent_batch in batch.items():
        mean_reward = 0
        for episode in agent_batch:
            mean_reward += np.sum(episode[MultiBatch.REWARD])

        mean_reward /= len(agent_batch)
        stats[str(policy_id) + "/mean_reward"] = mean_reward
        stats["mean_reward"] += mean_reward

    stats["episodes"] = num_episodes
    stats["samples"] = total_steps

    return batch, stats


# 与えられた文字列の各文字を，以下の仕様で変換する関数cipherを実装せよ．
#  英小文字ならば(219 - 文字コード)の文字に置換
#  その他の文字はそのまま出力
# この関数を用い，英語のメッセージを暗号化・復号化せよ．
import string

def cipher(sentence):
    sentence = list(sentence)
    alp = string.ascii_lowercase
    for i, letter in enumerate(sentence):
        if letter in alp:
            sentence[i] = chr(219 - ord(letter))
    return "".join(sentence)

sentence = " Float like a butterfly, die like a bee !!  --Enrico Maxwell--"
print("sentence = \n", sentence)
print("ciphered = \n", cipher(sentence))


# traffic.py
import time

class LightState:
    @staticmethod
    def handle_clock_tick(light):
        pass

    @staticmethod
    def handle_button_press(light):
        pass

class Green1(LightState):
    @staticmethod
    def handle_clock_tick(light):
        if light.clock == 30:
            light.clock = 0
            light.state = Yellow1

class Yellow1(LightState):
    @staticmethod
    def handle_clock_tick(light):
        if light.clock == 5:
            light.clock = 0
            light.state = Green2

class Green2(LightState):
    @staticmethod
    def handle_clock_tick(light):
        if light.clock == 60 or (light.walk and light.clock >= 30):
            light.clock = 0
            light.walk = False
            light.state = Yellow2

    @staticmethod
    def handle_button_press(light):
        light.walk = True

class Yellow2(LightState):
    @staticmethod
    def handle_clock_tick(light):
        if light.clock == 5:
            light.clock = 0
            light.state = Green1

class TrafficLight:
    def __init__(self):
        self.state = Green1
        self.clock = 0
        self.walk = False

    def __repr__(self):
        return f'TrafficLight(state={self.state}, clock={self.clock}, walk={self.walk})'

    def handle_clock_tick(self):
        self.clock += 1
        self.state.handle_clock_tick(self)

    def handle_button_press(self):
        self.state.handle_button_press(self)

# This class is responsible for driving the traffic light.
# This is separate from the state machine

import threading
import queue

class LightController:
    def __init__(self, light):
        self.light = light
        self.event_queue = queue.Queue()
        self.observers = set()

    def subscribe(self, callback):
        self.observers.add(callback)

    def run(self):
        self.running = True
        threading.Thread(target=self.run_clock, daemon=True).start()
        threading.Thread(target=self.run_button, daemon=True).start()
        while True:
            evt = self.event_queue.get()
            if evt is None:
                break
            if evt == 'tick':
                self.light.handle_clock_tick()
            elif evt == 'button':
                self.light.handle_button_press()
            else:
                print("BAD EVENT:", evt)
            # Notify all observers watching the light
            for observer in self.observers:
                observer(self.light)
        self.running = False

    def run_clock(self):
        while self.running:
            time.sleep(0.25)
            self.event_queue.put('tick')

    def run_button(self):
        import sys
        while self.running:
            sys.stdin.readline()
            self.event_queue.put('button')

light = TrafficLight()
control = LightController(light)
control.subscribe(print)
control.run()


# -*- coding: utf-8 -*-
'''
@Time    : 18-11-8 下午4:56
@Author  : qinpengzhi
@File    : util.py
@Software: PyCharm
@Contact : qinpzhi@163.com
'''
import tensorflow as tf
from tensorflow.contrib import *
##线性函数
def linear(input,output_size,scope=None,stddev=0.02,bias_start=0.0):
    shape= input.get_shape().as_list()
    ##tf.random_normal_initializer() is the same as tf.RandomNormal()
    ##tf.constant_initializer() is the same as tf.Constant()
    with tf.variable_scope(scope or "Linear"):
        matrix=tf.get_variable(
            "Matrix",[shape[1],output_size],tf.float32,
            tf.random_normal_initializer(stddev=stddev))
        bias = tf.get_variable("bias",[output_size],initializer=tf.constant_initializer(bias_start))
        ##tf.multiply是点乘.tf.matmul是矩阵乘法
        return tf.matmul(input,matrix)+bias

##batch_norm
def batch_norm(input,epsilon=1e-5,momentum=0.9,scope=None,train=True):
    # is_training:图层是否处于训练模式。在训练模式下，它将积累转入的统计量moving_mean并
    # moving_variance使用给定的指数移动平均值
    # decay。当它不是在训练模式，那么它将使用的数值moving_mean和moving_variance。
    return tf.contrib.layers.batch_norm(
            input,decay=momentum, updates_collections=None,
            epsilon=epsilon,scale=True,is_training=train)

##deconv2d
def deconv2d(input,output_shape,
             k_h=5,k_w=5,d_h=2,d_w=2,stddev=0.02,name="deconv2d"):
    with tf.variable_scope(name):
        w = tf.get_variable('w',[k_h,k_w,output_shape[-1],input.get_shape()[-1]],
                            initializer=tf.random_normal_initializer(stddev=stddev))
        deconv=tf.nn.conv2d_transpose(input,w,output_shape=output_shape,
                                      strides=[1,d_h,d_w,1])
        biases=tf.get_variable('biases',[output_shape[-1]],initializer=tf.constant_initializer(0.0))
        ##tf.nn.bias_add是将偏差项加到value上，是tf.add的一个特例，其中bias必须是一维的
        deconv=tf.reshape(tf.nn.bias_add(deconv,biases),deconv.get_shape())
        return deconv

##leaky relu。leaky relu 的 α的取值为０.2。
def lrelu(x,leak=0.2,name="lrelu"):
    return tf.maximum(x,leak*x)

##conv2d将pool层融合在了stride中
def conv2d(input,output_dim,k_h=5,k_w=5,d_h=2,d_w=2,stddev=0.02,name="conv2d"):
    with tf.variable_scope(name):
        w=tf.get_variable('w',[k_h,k_w,input.get_shape()[-1],output_dim],
                          initializer=tf.truncated_normal_initializer(stddev=stddev))
        conv=tf.nn.conv2d(input,w,strides=[1,d_h,d_w,1],padding="SAME")
        biases=tf.get_variable('biases',[output_dim],initializer=tf.constant_initializer(0.0))
        conv=tf.reshape(tf.nn.bias_add(conv,biases),conv.get_shape())
        return conv


from torch.utils import data
import h5py
# X is the 2D input and Y is the classification result
# This func just append them together into a dataset.
def make_dataset(file_path, train=True):

  datasets = []
  dataset = []

  if train:
    fhd5 = h5py.File(file_path,'r')
    for gname, group in fhd5.items():
      datasets.append(group)
    idx = 0
    for things in datasets[0]:
      # dataset.append(datasets[0][idx],datasets[1][idx])
      dataset.append([idx,idx])
      idx = idx +1

  else:
    fhd5 = h5py.File(file_path,'r')
    for gname, group in fhd5.items():
      datasets.append(group)
    idx = 0
    for things in datasets[0]:
      dataset.append([idx,idx])
      # dataset.append(datasets[0][idx],datasets[1][idx])
      idx = idx +1
# #
  return dataset

class MyDataSet(data.Dataset):
  # The "train" parameter would tell which dataset to use.
  def __init__(self, file_path, transform = None, train = True):
    self.train = train
    if self.train:
      traindatasets = []
      fhd5 = h5py.File(file_path,'r')
      for gname, group in fhd5.items():
          # print(group)
        traindatasets.append(group)
      self.train_set_path = make_dataset(file_path, train)
    else:
      testdatasets = []
      fhd5 = h5py.File(file_path,'r')
      for gname, group in fhd5.items():
          # print(group)
        testdatasets.append(group)
      self.test_set_path = make_dataset(file_path, train)

  def __getitem__(self, index):
    if self.train:
      img_path, label_path = self.train_set_path[index]
      img = self.traindatasets[0][img_path]
      label = self.traindatasets[1][label_path]
      if self.transform is not None:
        img = self.transform(img)
      return img,label
    else:
      img_path, label_path = self.test_set_path[index]
      img = self.testdatasets[0][img_path]
      label = self.testdatasets[1][label_path]
      if self.transform is not None:
        img = self.transform(img)
      return img,label

  def __len__(self):
    if self.train:
      return len(self.train_set_path)

    else:
      return len(self.test_set_path)


# FAÇA UM PROGAA QE LEIA A TABUADA DE VÁRIOS NÚMEROS, UM DE CADA VEZ, PARA CADA VALOR DIGITADO PELO #
# USUÁRIO. O PROGRAMA SERÁ ENCERRADO QUANDO UM VALOR NEGATIVO FOR SOLICITADO. #

from time import sleep
while True:

    num = int(input('Digite um número para ver a tabuada:'))
    print('-'*35)
    if num < 0:
        break
    for c in range(1, 11):
        multi = num * c
        print(f'{num} x {c} = {multi}')
    print('-'*35)

print('ENCERRANDO PROGRAMA..')
sleep(2)
print('PROGRAMA ENCERRADO. VOLTE SEMPRE!!')


import re

from django import template

register = template.Library()

@register.simple_tag(takes_context=True)
def line_break(context, value):
    if type(value) == str:
        value = re.sub(r'^\s+|\n|\s+$', '<br>', value)
    return value


"""
Given an array of non-empty positive ints representing the amount of time specific queries take to execute.
Only query can be executed at a time but in any order.
A query's WAITING TIME is the amt of time it must wait before execution starts. IF a query is executed 2nd, its WT is the duration of 1st. IF 3rd, its WT is the sum of 1st and 2nd.
Write function that returns min WT of all query's WT.
For ex queries: [1, 4, 5] executed in order [5, 1, 4] -- (0) + (5) + (5 + 1) = 11WT

Input:
    q = [3, 1, 2, 2, 6]
Output:
    17

**this one makes no sense and appears to use an imaginary number
**I'm think of it a queue and tallying exec times, yes the WT grows deeper into Q, but Duh
**formula is at idx
newWT = WT + prevExec (0+0)
newWT = WT + prevExec (0 + 5)
newWT = WT + prevExec (5 + 1)
"""
import time

def minimumWaitingTimeW(queries):
    # Write your code here. (don't get this one)
    # wait time is an imaginary #? isn't this like a Q?
    # is the hint "can mutate" a directive to "minumum"
    queries.sort()

    wait_times = [0 for x in queries]
    for idx in range(len(queries)):
        if idx == 0:
            wait_times[idx] = 0
        else:
            wait_times[idx] = wait_times[idx-1] + queries[idx-1]

    return sum(wait_times)

# Algo solution
def minimumWaitingTime(q):
    # ex: [1, 2, 2, 3, 6]
    q.sort()
    total_WT = 0
    for idx, runtime in enumerate(q):
        q_remaining = len(q) - (idx + 1)
        total_WT += runtime * q_remaining
        # R1 5-(0+1) = 4
        # R1 0 += 1 * 4 = 4
        # R2 5-(1+1) = 3
        # R2 4 += 2 * 3 = 10
        # R3 5-(2+1) = 2
        # R3 10 += 2 * 2 = 14
        # R4 5-(3+1) = 1
        # R4 14 += 3 * 1 = 17
        # R5 5-(4+1) = 0
        # R5 17 += 6 * 0 = 17
    return total_WT

q = [3, 2, 1, 2, 6]
print(minimumWaitingTimeW(q))


from scipy import stats
import numpy as np

distributions = [stats.chi2(6), stats.chi2(7)]
n_dim = len(distributions)
num_mixtures = 15
rho = .1
n_rows = 100
p_mcar = 0.1
shift = 0
regression_parameter = 2
cov = np.array([[1, rho], [rho, 1]])
method = "shrinkage"

eps = 0.003
M = np.concatenate((np.array(20 * [50]), np.array(5 * [1000])))
n_sample = 10000


import logging, re

from django.contrib.auth.decorators import login_required
from django.views.generic.simple import direct_to_template

from intensity.models import AssetInfo, Activity, ServerInstance
from intensity.views import getpost_form
from intensity.components.map_wizard__forms import MapWizardForm
from intensity.utility import DummyRequest
from intensity.tracker.views import asset_clone, activity_new, activity_requisition
from intensity.components.instance_selector__views import select

## Do the actual work
def wizard(account, location, original, requisition):
    dummy_request = DummyRequest(account)

    # Clone the original asset

    response = asset_clone(dummy_request, original.uuid)
    url = response['Location']
    m = re.match('/tracker/asset/view/(\w+)/$', url)
    asset_id = m.group(1)
    assert(asset_id != original.uuid)
    asset = AssetInfo.objects.get(uuid=asset_id)
    asset.location = location
    asset.save()

    # Create an activity

    dummy_request.POST['asset_id'] = asset_id
    response = activity_new(dummy_request)
    url = response['Location']
    m = re.match('/tracker/activity/view/(\w+)/$', url)
    activity_id = m.group(1)
    activity = Activity.objects.get(uuid=activity_id)
    activity.name = location.replace('base/', '').replace('.tar.gz', '') + ' v1.0'
    activity.save()

    # If requested, requisition and select a server
    if requisition:
        # Requisition
        activity_requisition(dummy_request, activity_id)
        message = dummy_request.session['message'].lower()
        success = 'success' in message
        if not success:
            return 'Map creation wizard made your map asset and activity, but could not find a server. Try to requisition one later for your new activity.'

        # Select
        instances = ServerInstance.objects.filter(activity = activity).filter(requisitioner = account)
        assert(len(instances) == 1)
        select(dummy_request, instances[0].uuid)

    return True

@login_required
def view(request):
    def on_valid(form):
        logging.info('Map wizard - view')

        result = wizard(request.account, form.cleaned_data['location'], form.cleaned_data['original'], form.cleaned_data['requisition'])

        if type(result) is str:
            request.session['message'] = result
        else:
            if form.cleaned_data['requisition']:
                request.session['message'] = 'Map creation wizard finished successfully. To start editing your map, open the client program, log in and then select "connect to selected."'
            else:
                request.session['message'] = 'Map creation wizard finished successfully. To start editing your map, run it on a server and connect to that server.'

    return getpost_form(request, 'map_wizard.html', MapWizardForm, on_valid, '/tracker/account/')


import torch
from torch.autograd import Variable
from skimage import io, transform
from torch import nn, optim
from torch.nn import functional as F
from ResNet import ResNet, BasicBlock, Bottleneck
import math
import numpy as np
import torch.utils.model_zoo as model_zoo
import torchvision

def Variable(data, *args, **kwargs):
    if True:
        data = data.cuda().float()
    return torch.autograd.Variable(data,*args, **kwargs)

def unit_prefix(x, n=1):
    for i in range(n): x = x.unsqueeze(0)
    return x

def align(x, y, start_dim=0):
    xd, yd = x.dim(), y.dim()
    if xd > yd:
        y = unit_prefix(y, xd - yd)
    elif yd > xd:
        x = unit_prefix(x, yd - xd)

    xs, ys = list(x.size()), list(y.size())
    nd = len(ys)
    for i in range(start_dim, nd):
        td = nd - i - 1
        if ys[td] == 1:
            ys[td] = xs[td]
        elif xs[td] == 1:
            xs[td] = ys[td]
    return x.expand(*xs), y.expand(*ys)

def matmul(X, Y):
    results = []
    for i in range(X.size(0)):
        result = torch.mm(X[i], Y[i])
        results.append(result.unsqueeze(0))
    return torch.cat(results)

def xrecons_grid(X, B, A):
    """
    plots canvas for single time step
    X is x_recons, (batch_size x img_size)
    assumes features = BxA images
    batch is assumed to be a square number
    """
    padsize = 1
    padval = .5
    ph = B + 2 * padsize
    pw = A + 2 * padsize
    batch_size = X.shape[0]
    N = int(np.sqrt(batch_size))
    X = X.reshape((N, N, B, A))
    img = np.ones((N * ph, N * pw)) * padval
    for i in range(N):
        for j in range(N):
            startr = i * ph + padsize
            endr = startr + B
            startc = j * pw + padsize
            endc = startc + A
            img[startr:endr, startc:endc] = X[i, j, :, :]
    return img

class DrawModel(nn.Module):
    def __init__(self, T, A, B, z_size, N, dec_size, enc_size):
        super(DrawModel, self).__init__()
        self.T = T
        # self.batch_size = batch_size
        self.A = A
        self.B = B
        self.z_size = z_size
        self.N = N
        self.dec_size = dec_size
        self.enc_size = enc_size
        self.cs = [0] * T
        self.logsigmas, self.sigmas, self.mus = [0] * T, [0] * T, [0] * T

        self.encoder = nn.LSTMCell(2 * N * N + dec_size, enc_size)
        self.encoder_gru = nn.GRUCell(2 * N * N + dec_size, enc_size)
        self.mu_linear = nn.Linear(dec_size, z_size)
        self.sigma_linear = nn.Linear(dec_size, z_size)

        self.decoder = nn.LSTMCell(z_size, dec_size)
        self.decoder_gru = nn.GRUCell(z_size, dec_size)
        self.dec_linear = nn.Linear(dec_size, 5)
        self.dec_w_linear = nn.Linear(dec_size, N * N)

        self.sigmoid = nn.Sigmoid()

    def normalSample(self):
        return Variable(torch.randn(self.batch_size, self.z_size))

    # correct
    def compute_mu(self, g, rng, delta):
        rng_t, delta_t = align(rng, delta)
        tmp = (rng_t - self.N / 2 - 0.5) * delta_t
        tmp_t, g_t = align(tmp, g)
        mu = tmp_t + g_t
        return mu

    # correct
    def filterbank(self, gx, gy, sigma2, delta):
        rng = Variable(torch.arange(0, self.N).view(1, -1))
        mu_x = self.compute_mu(gx, rng, delta)
        mu_y = self.compute_mu(gy, rng, delta)

        a = Variable(torch.arange(0, self.A).view(1, 1, -1))
        b = Variable(torch.arange(0, self.B).view(1, 1, -1))

        mu_x = mu_x.view(-1, self.N, 1)
        mu_y = mu_y.view(-1, self.N, 1)
        sigma2 = sigma2.view(-1, 1, 1)

        Fx = self.filterbank_matrices(a, mu_x, sigma2)
        Fy = self.filterbank_matrices(b, mu_y, sigma2)

        return Fx, Fy

    def forward(self, x):
        self.batch_size = x.size()[0]
        h_dec_prev = Variable(torch.zeros(self.batch_size, self.dec_size))
        h_enc_prev = Variable(torch.zeros(self.batch_size, self.enc_size))

        enc_state = Variable(torch.zeros(self.batch_size, self.enc_size))
        dec_state = Variable(torch.zeros(self.batch_size, self.dec_size))
        for t in range(self.T):
            c_prev = Variable(torch.zeros(self.batch_size, self.A * self.B)) if t == 0 else self.cs[t - 1]
            x_hat = x - self.sigmoid(c_prev)  # 3
            r_t = self.read(x, x_hat, h_dec_prev)
            h_enc_prev, enc_state = self.encoder(torch.cat((r_t, h_dec_prev), 1), (h_enc_prev, enc_state))
            # h_enc = self.encoder_gru(torch.cat((r_t,h_dec_prev),1),h_enc_prev)
            z, self.mus[t], self.logsigmas[t], self.sigmas[t] = self.sampleQ(h_enc_prev)
            h_dec, dec_state = self.decoder(z, (h_dec_prev, dec_state))
            # h_dec = self.decoder_gru(z, h_dec_prev)
            self.cs[t] = c_prev + self.write(h_dec)
            h_dec_prev = h_dec
        return self.sigmoid(self.cs[-1])

    def get_output(self):
        return self.sigmoid(self.cs[-1])

    def loss(self, x):
        x = x.view(x.shape[0], -1)
        self.forward(x)
        criterion = nn.BCELoss()
        x_recons = self.sigmoid(self.cs[-1])
        Lx = criterion(x_recons, x) * self.A * self.B
        Lz = 0
        kl_terms = [0] * self.T
        for t in range(self.T):
            mu_2 = self.mus[t] * self.mus[t]
            sigma_2 = self.sigmas[t] * self.sigmas[t]
            logsigma = self.logsigmas[t]
            # Lz += (0.5 * (mu_2 + sigma_2 - 2 * logsigma))    # 11
            kl_terms[t] = 0.5 * torch.sum(mu_2 + sigma_2 - 2 * logsigma, 1) - self.T * 0.5
            Lz += kl_terms[t]
        # Lz -= self.T / 2
        Lz = torch.mean(Lz)  ####################################################
        loss = Lz + Lx  # 12
        return loss

    # correct
    def filterbank_matrices(self, a, mu_x, sigma2, epsilon=1e-9):
        t_a, t_mu_x = align(a, mu_x)
        temp = t_a - t_mu_x
        temp, t_sigma = align(temp, sigma2)
        temp = temp / (t_sigma * 2)
        F = torch.exp(-torch.pow(temp, 2))
        F = F / (F.sum(2, True).expand_as(F) + epsilon)
        return F

    # correct
    def attn_window(self, h_dec):
        params = self.dec_linear(h_dec)
        gx_, gy_, log_sigma_2, log_delta, log_gamma = params.split(1, 1)  # 21

        # gx_ = Variable(torch.ones(4,1))
        # gy_ = Variable(torch.ones(4, 1) * 2)
        # log_sigma_2 = Variable(torch.ones(4, 1) * 3)
        # log_delta = Variable(torch.ones(4, 1) * 4)
        # log_gamma = Variable(torch.ones(4, 1) * 5)

        gx = (self.A + 1) / 2 * (gx_ + 1)  # 22
        gy = (self.B + 1) / 2 * (gy_ + 1)  # 23
        delta = (max(self.A, self.B) - 1) / (self.N - 1) * torch.exp(log_delta)  # 24
        sigma2 = torch.exp(log_sigma_2)
        gamma = torch.exp(log_gamma)

        return self.filterbank(gx, gy, sigma2, delta), gamma

    # correct
    def read(self, x, x_hat, h_dec_prev):
        (Fx, Fy), gamma = self.attn_window(h_dec_prev)

        def filter_img(img, Fx, Fy, gamma, A, B, N):
            Fxt = Fx.transpose(2, 1)
            img = img.view(-1, B, A)
            # img = img.transpose(2,1)
            # glimpse = matmul(Fy,matmul(img,Fxt))
            glimpse = Fy.bmm(img.bmm(Fxt))
            glimpse = glimpse.view(-1, N * N)
            return glimpse * gamma.view(-1, 1).expand_as(glimpse)

        x = filter_img(x, Fx, Fy, gamma, self.A, self.B, self.N)
        x_hat = filter_img(x_hat, Fx, Fy, gamma, self.A, self.B, self.N)
        return torch.cat((x, x_hat), 1)

    # correct
    def write(self, h_dec=0):
        w = self.dec_w_linear(h_dec)
        w = w.view(self.batch_size, self.N, self.N)
        # w = Variable(torch.ones(4,5,5) * 3)
        # self.batch_size = 4
        (Fx, Fy), gamma = self.attn_window(h_dec)
        Fyt = Fy.transpose(2, 1)
        # wr = matmul(Fyt,matmul(w,Fx))
        wr = Fyt.bmm(w.bmm(Fx))
        wr = wr.view(self.batch_size, self.A * self.B)
        return wr / gamma.view(-1, 1).expand_as(wr)

    def sampleQ(self, h_enc):
        e = self.normalSample()
        # mu_sigma = self.mu_sigma_linear(h_enc)
        # mu = mu_sigma[:, :self.z_size]
        # log_sigma = mu_sigma[:, self.z_size:]
        mu = self.mu_linear(h_enc)  # 1
        log_sigma = self.sigma_linear(h_enc)  # 2
        sigma = torch.exp(log_sigma)

        return mu + sigma * e, mu, log_sigma, sigma

    def generate(self, batch_size=64):
        self.batch_size = batch_size
        h_dec_prev = Variable(torch.zeros(self.batch_size, self.dec_size), volatile=True)
        dec_state = Variable(torch.zeros(self.batch_size, self.dec_size), volatile=True)

        for t in range(self.T):
            c_prev = Variable(torch.zeros(self.batch_size, self.A * self.B)) if t == 0 else self.cs[t - 1]
            z = self.normalSample()
            h_dec, dec_state = self.decoder(z, (h_dec_prev, dec_state))
            self.cs[t] = c_prev + self.write(h_dec)
            h_dec_prev = h_dec
        imgs = []
        for img in self.cs:
            imgs.append(self.sigmoid(img).cpu().data.numpy())
        return imgs


## Prof.: Nei commenti non rusare variabile, metti cosa rappresenta quella variabile
x_Circle=0 #variabile coordinata x della pallina
y_Circle=0 #variabile coordinata y della pallina
circleWidth=40 #variabile larghezza pallina
circleHeight=40 #variabile altezza pallina
totalWidth=700 #variabile larghezza totale del campo di gioco
totalHeigth=500 #variabile altezza totale del campo di gioco
x_circleSpeed=3 #variabile velocita pallina coordinata x
y_circleSpeed=3 #variabile velocita pallina coordinata y
x_paddleLeft=25 #variabile coordinata x racchetta sinistra, il meno 25 lo si mette per distaccare di 25 la racchetta dal campo di gioco
y_paddleLeft=totalHeigth/2 #variabile coordinata y racchetta sinistra, in questo caso basta fare il quoziente tra l'altezza totale del campo di gioco e 2, poichè la racchetta si muove sull'asse y
paddleWidth=25 #variabile larghezza delle due racchette, sia sinistra che destra
paddleHeigth=100 #variabile altezza delle due racchette, sia destra che destra
paddleSpace = 5 #variabile racchetta che quando tocca il bordo del campo di gioco torna indietro di 5
x_paddleRight=totalWidth-25 #variabile coordinata x racchetta destra, in questo caso si fa la differenza tra la larghezza totale del campo di gioco e 25 cosi da posizionarla a specchio rispetto alla racchetta sinistra
y_paddleRight=totalHeigth/2 #variabile coordinata y racchetta destra, in questo caso basta fare il quoziente tra l'altezza totale del campo di gioco e 2, poichè la racchetta si muove sull'asse y
scoreLeft=0 #variabile inizializzo il punteggio del giocatore sinistra a 0
scoreRight=0 #variabile inizializzo il punteggio del giocatore destro a 0
winScore=5 #variabile punteggio massimo, il primo giocatore che riesce ad arrivare a 5 ha vinto
x_paddleSpacepeed=8 #variabile velocita racchetta coordinata x
y_paddleSpacepeed=8 #variabile velocita racchetta coordinata y

def setup():
    global x_Circle,y_Circle #variabili globali
    size(700,500); #dimensioni campo di gioco
    x_Circle= totalWidth/2 #posizionare la pallina al centro nella coordinata x
    y_Circle= totalHeigth/2 #posizionare la pallina al centro nella coordinata y
    rectMode(CENTER) #metodo per mettere le due racchette al centro del campo di gioco
## Prof.: RectMode non serve per posizionare ma per riferire i rettangolo utilizzando il punto centrale
    textSize(25)#grandezza testo

def draw():
    global x_Circle,y_Circle,circleWidth,circleHeight,totalWidth,totalHeigth,x_circleSpeed, y_circleSpeed,x_paddleLeft, y_paddleLeft, paddleWidth, paddleHeigth, y_paddleLeft, paddleWidth
    background(0,0,0); #sfondo campo gioco
    drawLinea() #prototipo funzione che crea la linea in mezzo
    drawCircle() #prototipo funzione che disegna la pallina
    drawPaddle() #prototipo funzione che disegna le due racchette
    moveCircle() #prototipo funzione che fa muovere la pallina
    bounceOff() #prototipo funzione  che fa rimbalzzare la pallina sui bordi del campo
    contactPaddle() #prototipo funzione che fa rimbalzare la pallina quando tocca la racchetta
    restrictPaddle() #prototipo funzione che risolve i problemi di contatto tra pallina e a racchetta
    scores() #prototipo funzione che conta i punti ai 2 giocatori
    gameOver() #prototipo funzione che fa fermare il gioco, quando uno dei due giocatori riesce a vincere

def drawLinea():
    fill(255) #colore bianco per la linea in mezzo
    rect(width/2,height/2, 2, 700); #creazione linea in mezzo

def drawCircle():
    global x_Circle,y_Circle,circleWidth,circleHeight
    fill(random(0,255),random(0,255),random(0,255)); #colora la pallina in modo randomico
    ellipse(x_Circle, y_Circle, circleWidth, circleHeight); #creazione pallina

def drawPaddle():
    global x_paddleLeft, y_paddleLeft, paddleWidth, paddleHeigth,x_paddleRight, y_paddleRight
    fill(255,0,0); #colore rosso per la racchetta sinistra
    rect(x_paddleLeft, y_paddleLeft, paddleWidth, paddleHeigth); #creazione racchetta sinistra
    fill(0,0,255); #colore blu per la racchetta destra
    rect(x_paddleRight, y_paddleRight, paddleWidth, paddleHeigth); #creazione racchetta destra

def moveCircle():
    global x_Circle,y_Circle,x_circleSpeed, y_circleSpeed
    x_Circle=x_Circle+x_circleSpeed; #far muovere la pallina attraverso la sua coordinata x
    y_Circle=y_Circle+y_circleSpeed; #far muovere la pallina attraverso la sua coordinata y

def bounceOff():
    global x_Circle,y_Circle,circleWidth, circleHeight,x_circleSpeed, y_circleSpeed, scoreLeft, scoreRight, totalHeigth, totalWidth
    if ( x_Circle > totalWidth - circleWidth/2 ):#se la x della pallina supera la larghezza totale
        setup()#richiamo la funzione
        x_circleSpeed = -x_circleSpeed #rimbalzo della pallina
        scoreLeft = scoreLeft+1 #aumento il punteggio del giocatore sinistro

    if ( x_Circle < 0 + circleWidth/2):#se la x della pallina supera la larghezza totale
        setup()#richiamo la funzione
        x_circleSpeed = -x_circleSpeed #rimbalzo della pallina
        scoreRight = scoreRight+1 #aumento il punteggio del giocatore destro

    if ( y_Circle > totalHeigth - circleHeight/2 ):#se la y della pallina supera la altezza totale
        y_circleSpeed = -y_circleSpeed #rimbalzo della pallina

    if ( y_Circle < 0 + circleHeight/2):#se la y della pallina supera la altezza totale
        y_circleSpeed = -y_circleSpeed #rimbalzo della pallina

def keyPressed():
    global y_paddleLeft, y_paddleRight, y_paddleSpacepeed
    if (key== "w" or key=="W"): #premendo w o W la racchetta sinistra si muoverà verso l'alto
        y_paddleLeft -=y_paddleSpacepeed #modifico la y diminuendola , cosi da far muovere la racchetta verso l'alto

    if (key == "z" or key =="Z"): #premendo z o Z la racchetta sinistra si muoverà verso il basso
        y_paddleLeft +=y_paddleSpacepeed #modifico la y aumentandola , cosi da far muovere la racchetta verso il basso

    if (keyCode == DOWN): #premendo la freccetta in basso la racchetta destra va in basso
        y_paddleRight +=y_paddleSpacepeed #modifico la y aumentandola , cosi da far muovere la racchetta verso il basso

    if (keyCode== UP): #premendo la freccetta in alto la racchetta destra va in alto
        y_paddleRight -=y_paddleSpacepeed # modifico la y diminiuendola, cosi da far muovere la racchetta verso l'alto

def contactPaddle():
    global x_Circle,x_paddleRight,paddleWidth,x_circleSpeed,y_circleSpeed,y_paddleLeft, circleHeight,circleWidth,y_Circle,y_paddleRight,x_paddleLeft
    if(x_Circle - circleWidth/2 < x_paddleLeft + paddleWidth/2 and y_Circle - circleHeight/2 < y_paddleLeft +paddleHeigth/2 and  y_Circle + circleHeight/2 > y_paddleLeft-paddleHeigth/2):#controllo per far rimbalzare la pallina al contatto con la racchetta sinistra
        if(x_circleSpeed < 0): # se variabile velocita pallina coordinata x < 0
            x_circleSpeed = -x_circleSpeed #rimbalzo della pallina al contatto con la racchetta

    if(x_Circle + circleWidth/2 > x_paddleRight - paddleWidth/2 and y_Circle - circleHeight/2 < y_paddleRight +paddleHeigth/2 and  y_Circle + circleHeight/2 > y_paddleRight-paddleHeigth/2):#controllo per far rimbalzare la pallina al contatto con la racchetta destra
        if(x_circleSpeed > 0): # se variabile velocita pallina coordinata x > 0
            x_circleSpeed = -x_circleSpeed #rimbalzo della pallina al contatto con la racchetta

def restrictPaddle():
## Prof.: non belo il ribalzo della racchetta sui bordi: meglio fermarsi sul bordo
    global y_paddleLeft, paddleHeigth, paddleSpace, y_paddleRight
    if(y_paddleLeft - paddleHeigth/2 < 0): #controllo quando la pallina tocca lo spigolo della racchetta sinistra
        y_paddleLeft = y_paddleLeft + paddleSpace

    if(y_paddleLeft + paddleHeigth/2 > height):
        y_paddleLeft = y_paddleLeft - paddleSpace

    if(y_paddleRight - paddleHeigth/2 < 0): #controllo quando la pallina tocca lo spigolo della racchetta destra
        y_paddleRight = y_paddleRight + paddleSpace

    if(y_paddleRight + paddleHeigth/2 > height):
        y_paddleRight = y_paddleRight - paddleSpace

def scores():
    global scoreLeft, scoreRight, winScore
    fill(255); #colore bianco per i punti
    text(scoreLeft, 320, 30) #viene scritto e posizionato il punteggio del giocatore sinistro
    text(scoreRight, 367, 30) #viene scritto e posizionato il punteggio del giocatore destro

def gameOver():
    global scoreLeft, scoreRight, winScore, colorL, colorR, x_circleSpeed, y_circleSpeed
    if(scoreLeft==winScore): #se il punteggio del giocatore sinistro è uguale alla variabile che conta il punteggio massimo che si puo raggiungere
        fill(255,0,0) #colore rosso
        text("RED WIN", 110, 110) #viene scritta e posizionata la frase RED WIN
        x_circleSpeed=0 #far fermare la pallina poiche il gioco è terminato, quindi riportare la pallina a coordinata x=0
        y_circleSpeed=0 #far fermare la pallina poiche il gioco è terminato, quindi riportare la pallina a coordinata y=0

    if(scoreRight==winScore): #se il punteggio del giocatore destro è uguale alla variabile che conta il punteggio massimo che si puo raggiungere
        fill(0,0,255) #colore blu
        text("BLU WIN", 495 , 390) #viene scritta e posizionata la frase BLU WIN
        x_circleSpeed=0 #far fermare la pallina poiche il gioco è terminato, quindi riportare la pallina a coordinata x=0
        y_circleSpeed=0 #far fermare la pallina poiche il gioco è terminato, quindi riportare la pallina a coordinata y=0