from __future__ import print_function
from keras.models import Model
from keras.callbacks import ModelCheckpoint
from keras.preprocessing.image import ImageDataGenerator
from keras.backend.tensorflow_backend import set_session
from keras.models import model_from_json
import keras.backend as K
import tensorflow as tf
import keras
import cv2
import numpy as np
import os
from glob import glob
import argparse
import pandas as pd
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt

import models
from utils import centering, un_centering
from utils import get_classmap_keras, get_classmap_numpy
from utils import image_read
from utils import plot_confusion_matrix
import utils

import face_recognition

from six import next

class predictor:
    def __init__(self, flag):
        self.flag = flag
        self.pd_label = pd.read_csv('./label.csv')

    def name2label(self, name):
        y_label = self.pd_label[self.pd_label['Filename']==name]
        y_label = y_label.values[0][1:]
        return y_label

    def user_generation(self, train_generator):
        # cv2.namedWindow('show', 0)
        # cv2.resizeWindow('show',500, 500)
        for total_iter, images in enumerate(train_generator):
            batch_idx = train_generator.batch_index
            batch_size = images.shape[0]

            if train_generator.batch_index != 0:
                batch_idx = batch_idx - 1
            else:
                batch_idx = train_generator.n//train_generator.batch_size

            idx_list = train_generator.index_array[batch_idx*train_generator.batch_size:batch_idx*train_generator.batch_size+batch_size]
            # print (batch_idx, idx_list.shape, train_generator.n)

            list_filenames = np.array(train_generator.filenames)[idx_list]

            list_filenames = [os.path.basename(path) for path in list_filenames]
            np_labels = np.array([self.name2label(name) for name in list_filenames], dtype=np.float32)

            np_labels /= 5

            yield (images, np_labels)

    def cam(self):
        img_size = self.flag.image_size
        batch_size = self.flag.batch_size
        epochs = self.flag.total_epoch

        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        set_session(tf.Session(config=config))

        with open(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, 'model.json'), 'r') as json_file:
            loaded_model_json = json_file.read()

        #model = model_from_json(loaded_model_json)
        model = models.vgg_like(self.flag)

        weight_list = sorted(glob(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, 'weight*')))
        model.load_weights(weight_list[-1])
        print ('[*] model load : %s'%weight_list[-1])

        label_list = [os.path.basename(path) for path
                    in sorted(glob(os.path.join(self.flag.data_path, '*')))]
        dict_name_list = dict()
        for name in label_list:
            dict_name_list[name] = [path for path
                    in sorted(glob(os.path.join(self.flag.data_path, name, '*')))]
            dict_name_list[name] = dict_name_list[name][:50]

        data_list = []
        [data_list.append(image_read(name, color_mode=1, target_size=flag.image_size))
                for name in dict_name_list['dog']]
        # print len(data_list)
        np_Inference_data_list = np.array(data_list)[:,0,:,:,:]
        np_original_data_list = np.array(data_list)[:,1,:,:,:].astype(np.uint8)
        # print np_Inference_data_list.shape

        result = model.predict(np_Inference_data_list, self.flag.batch_size)
        # print result.shape
        prediction_labels = np.argmax(result, axis=1)

        classmap = get_classmap_keras(self.flag, model, np_Inference_data_list)

        assert classmap.shape[0] == np_original_data_list.shape[0] == prediction_labels.shape[0]

        cv2.namedWindow("show", 0)
        cv2.resizeWindow("show", 500, 500)

        for idx in range(classmap.shape[0]):
            predicted_label = prediction_labels[idx]
            print ("[*] %d's label : %s"%(idx, label_list[predicted_label]))
            # print(np_original_data_list.shape)
            img_original = np_original_data_list[idx,:,:,:]
            img_classmap = classmap[idx,:,:,predicted_label]
            color_classmap = cv2.applyColorMap(img_classmap, cv2.COLORMAP_JET)
            img_show = cv2.addWeighted(img_original, 0.8, color_classmap, 0.5, 0.)
            # cv2.imshow("show", img_show)
            # cv2.imwrite("./result/dog_%d.png"%idx, img_show)
            # if cv2.waitKey(1) == 27:
            #     break
        print ("[*] done")

    def evaluate(self):
        img_size = self.flag.image_size
        batch_size = self.flag.batch_size
        epochs = self.flag.total_epoch

        t_start = cv2.getTickCount()
        config = tf.ConfigProto()
        # config.gpu_options.per_process_gpu_memory_fraction = 0.9
        config.gpu_options.allow_growth = True
        set_session(tf.Session(config=config))

        with open(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name.split('/')[0], 'model.json'), 'r') as json_file:
            loaded_model_json = json_file.read()
        model = model_from_json(loaded_model_json)
        # model = models.vgg_like(self.flag)
        if os.path.splitext(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name))[-1] == '.h5':
            weight_file_path = os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name)
        else:
            weight_list = sorted(glob(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, "weight*")))
            weight_file_path = weight_list[-1]
        model.load_weights(weight_file_path)
        model.compile(optimizer='Adam', loss='mse', metrics=['mae'])
        print ("[*] model load : %s"%weight_file_path) #weight_list[-1])
        t_total = (cv2.getTickCount() - t_start) / cv2.getTickFrequency() * 1000
        print ("[*] model loading Time: %.3f ms"%t_total)

        test_datagen = ImageDataGenerator(
            preprocessing_function=centering,
            rescale=1./255)

        test_generator = test_datagen.flow_from_directory(
                os.path.join(self.flag.data_path, 'val'),
                target_size=(img_size, img_size),
                batch_size=batch_size,
                shuffle=False,
                color_mode='rgb',
                class_mode=None)

        user_test_gen = self.user_generation(test_generator)

        t_start = cv2.getTickCount()
        loss, acc = model.evaluate_generator(user_test_gen, test_generator.n // self.flag.batch_size)
        t_total = (cv2.getTickCount() - t_start) / cv2.getTickFrequency() * 1000
        print ('[*] test loss : %.4f'%loss)
        print ('[*] test acc  : %.4f'%acc)
        print ("[*] evaluation Time: %.3f ms"%t_total)

        ### confusion matrix
        # pred_generator = test_datagen.flow_from_directory(
        #         os.path.join(self.flag.data_path, 'val'),
        #         target_size=(img_size, img_size),
        #         batch_size=1,
        #         shuffle=False,
        #         #color_mode='grayscale',
        #         class_mode='categorical')

        # pred = model.predict_generator(pred_generator, test_generator.n)
        # y_pred = np.argmax(pred, axis=1)
        # y_true = pred_generator.classes

        # cnf_matrix = confusion_matrix(y_true, y_pred)
        # np.set_printoptions(precision=2)
        # Plot non-normalized confusion matrix
        # plt.figure()
        # class_names = ['neutral', 'happiness', 'surprise', 'sadness', 'anger', 'disgust', 'fear', 'contempt']
        # plot_confusion_matrix(cnf_matrix, classes=class_names, title='Confusion matrix, without normalization')
        # # Plot normalized confusion matrix
        # plt.figure()
        # plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,title='Normalized confusion matrix')

        # plt.show()

    def inference_temp(self):
        img_size = (self.flag.image_width, self.flag.image_height)
        batch_size = self.flag.batch_size
        epochs = self.flag.total_epoch

        ### initialize
        t_start = cv2.getTickCount()
        config = tf.ConfigProto()
        # config.gpu_options.per_process_gpu_memory_fraction = 0.9
        config.gpu_options.allow_growth = True
        set_session(tf.Session(config=config))

        with open(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, 'model.json'), 'r') as json_file:
            loaded_model_json = json_file.read()
        model = model_from_json(loaded_model_json)
        # model = models.vgg_like(self.flag)
        weight_list = sorted(glob(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, "weight*")))
        model.load_weights(weight_list[-1])
        print ("[*] model load: %s"%weight_list[-1])
        t_total = (cv2.getTickCount() - t_start) / cv2.getTickFrequency() * 1000
        print ("[*] model loading Time: %.3f ms"%t_total)

        ### dataset
        # image_name_list = ['./data/g1.png','./data/iu1.jpg', './data/dh3.jpg', './data/tkwoo2.jpg']
        # image_name_list = ['./data/g1.png', 'data/shlee.jpg', './data/g1.png', './data/g2.png', './data/g3.png', './data/g4.png']
        # image_name_list = ['./data/temp0.png', './data/temp1.png', './data/temp2.png', './data/temp3.png']
        # image_name_list = ['./data/iusuji.jpg', './data/sejung.jpg', './data/sj3.jpg']
        image_name_list = ['./data/yui2.png', './data/yui3.png', './data/yui4.png', './data/yui5.png']
        # gap = 40
        for img_name in image_name_list:
            img, show = image_read(img_name, 1, target_size=None)
            rgb_ori = cv2.cvtColor(show, cv2.COLOR_BGR2RGB)
            face_location = face_recognition.face_locations(rgb_ori,0,'cnn')
            # face_location = (np.array(face_location) - common_margin).tolist()
            t,r,b,l = face_location[0]

            for i in range(1):
                gap_y = int(round((r-l)*0.5))
                gap_x = int(round(((r-l+2*gap_y)*4/3-(r-l))/2))
                t,r,b,l = (t-gap_y,r+gap_x,b+gap_y,l-gap_x)
                print ((b-t)/(r-l))
                t = max([t,0])
                r = min([r,img.shape[1]])
                b = min([b,img.shape[0]])
                l = max([l,0])
                crop_show = show[t:b,l:r]
                crop_img = img[t:b,l:r]
                crop_img = cv2.resize(crop_img, (160,120))
                img_input = np.expand_dims(crop_img, 0)

                t_start = cv2.getTickCount()
                result = model.predict(img_input, 1)
                t_total = (cv2.getTickCount() - t_start) / cv2.getTickFrequency() * 1000
                print ("[*] Processing Time: %.3f ms"%t_total)
                print (os.path.basename(img_name), result[0]*5)

            cv2.rectangle(show, (l,t), (r,b), (0,0,0), 2)
            cv2.imshow('show', show)
            key = cv2.waitKey()
            if key == 27:
                exit()
            continue

            # cv2.imshow("show", show)
            # if cv2.waitKey(0) == 27:
            #     break
        # data_test.to_csv('./result.csv')

    def inference(self):
        img_size = self.flag.image_size
        batch_size = self.flag.batch_size
        epochs = self.flag.total_epoch

        t_start = cv2.getTickCount()
        config = tf.ConfigProto()
        # config.gpu_options.per_process_gpu_memory_fraction = 0.9
        config.gpu_options.allow_growth = True
        set_session(tf.Session(config=config))

        with open(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, 'model.json'), 'r') as json_file:
            loaded_model_json = json_file.read()
        model = model_from_json(loaded_model_json)
        # model = models.vgg_like(self.flag)
        weight_list = sorted(glob(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, "weight*")))
        model.load_weights(weight_list[-1])
        print ("[*] model load: %s"%weight_list[-1])
        t_total = (cv2.getTickCount() - t_start) / cv2.getTickFrequency() * 1000
        print ("[*] model loading Time: %.3f ms"%t_total)

        if os.path.isdir(self.flag.test_image_path):
            image_name_list = utils.get_image_name_list(self.flag.test_image_path)
            # print os.path.splitext(self.flag.test_image_path)
        elif os.path.splitext(self.flag.test_image_path)[-1] == '.png':
            image_name_list = [self.flag.test_image_path]
        elif os.path.splitext(self.flag.test_image_path)[-1] == '.jpg':
            image_name_list = [self.flag.test_image_path]
        # print "[*] Inference data path:", self.flag.test_image_path
        # print "[*] # of data:", len(image_name_list)
        # print image_name_list

        # list_column = ['img_name', 'predicted_label', 'softmax']
        # if os.path.exists('./result.csv'):
        #     data_test = pd.read_csv('./result.csv', index_col=0)
        # else:
        #     fCsvOut = open('./result.csv', 'w')
        #     fCsvOut.write('img_name,predicted_label,softmax\n')
        #     fCsvOut.close()
        #     data_test = pd.read_csv('./result.csv', index_col=0)
        # np_data_test = data_test.values

        for img_name in image_name_list:
            img, show = image_read(img_name, 1, target_size=img_size)
            img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
            img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
            # img = np.expand_dims(img, 2)
            img = np.expand_dims(img, 0)

            t_start = cv2.getTickCount()
            result = model.predict(img, 1)
            t_total = (cv2.getTickCount() - t_start) / cv2.getTickFrequency() * 1000
            print ("[*] Processing Time: %.3f ms"%t_total)

            print (os.path.basename(img_name),)
            # print result

            predict_label = np.argmax(result)
            print (predict_label, )
            # print predict_label.dtype
            # print predict_label.astype(np.str)
            # exit()
            sm_str = ''.join('%0.3f '%e for e in result[0])[:-1]
            sm_stdprint = ''.join('%0.3f,'%e for e in result[0])[:-1]
            print (sm_stdprint)

            # inserting_data = [predict_label.astype(np.str), sm_str]
            # data_test.ix[os.path.basename(img_name)] = inserting_data

            # cv2.imshow("show", show)
            # if cv2.waitKey(0) == 27:
            #     break
        # data_test.to_csv('./result.csv')

    def inference_MIT(self):
        img_size = (self.flag.image_width, self.flag.image_height)
        batch_size = self.flag.batch_size
        epochs = self.flag.total_epoch

        ### initialize
        t_start = cv2.getTickCount()
        config = tf.ConfigProto()
        # config.gpu_options.per_process_gpu_memory_fraction = 0.9
        config.gpu_options.allow_growth = True
        set_session(tf.Session(config=config))

        with open(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, 'model.json'), 'r') as json_file:
            loaded_model_json = json_file.read()
        model = model_from_json(loaded_model_json)
        # model = models.vgg_like(self.flag)
        weight_list = sorted(glob(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, "weight*")))
        model.load_weights(weight_list[-1])
        print ("[*] model load: %s"%weight_list[-1])
        t_total = (cv2.getTickCount() - t_start) / cv2.getTickFrequency() * 1000
        print ("[*] model loading Time: %.3f ms"%t_total)

        ### dataset
        image_name_list = sorted(glob('/run/media/tkwoo/myWorkspace/workspace/01.dataset/04.facedataset/MIT_LK_SELF_images/*.png'))

        # fOut = open('./MIT_LK_SELF_pred.csv', 'w')
        pd_data = pd.read_csv('./temp.csv', names=['name', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'])
        np_data = pd_data.values

        enter_idx = 0
        for idx, img_name in enumerate(image_name_list):
            if idx%10 == 0 and idx != 0:
                enter_idx += 1
            # if idx < 4080:
                # continue
            img, show = image_read(img_name, 1, target_size=None)
            rgb_ori = cv2.cvtColor(show, cv2.COLOR_BGR2RGB)
            face_location = face_recognition.face_locations(rgb_ori, 0, 'cnn')
            # face_location = (np.array(face_location) - common_margin).tolist()
            if face_location is None or len(face_location) == 0:
                continue
            t,r,b,l = face_location[0]

            gap = 0
            t,r,b,l = (t-gap,r+gap,b+gap,l-gap)
            t = max([t,0])
            r = min([r,img.shape[1]])
            b = min([b,img.shape[0]])
            l = max([l,0])
            crop_show = show[t:b,l:r]
            crop_img = img[t:b,l:r]
            crop_img = cv2.resize(crop_img, (96,96))
            img_input = np.expand_dims(crop_img, 0)

            t_start = cv2.getTickCount()
            result = model.predict(img_input, 1)
            t_total = (cv2.getTickCount() - t_start) / cv2.getTickFrequency() * 1000
            # print ("[*] Processing Time: %.3f ms"%t_total)
            print (idx, os.path.basename(img_name), result[0]*5)
            # print (type('%.3f'%float(result[0][0]*5)))

            parsed_name = os.path.basename(img_name).split('_')
            human_id = parsed_name[1]
            human_idx = parsed_name[2].split('.')[0]
            # print (human_idx)
            # print (type(np_data[idx][0]), type(human_id))
            if int(np_data[enter_idx][0]) != int(human_id):
                print ('wrong')
                print (int(np_data[idx][0]), int(human_id))
                exit()
            np_data[enter_idx][int(human_idx)+1] = '%.3f'%float(result[0][0]*5)

            pd_out = pd.DataFrame(data=np_data, columns=['name', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'])
            pd_out.to_csv('result_%02d.csv'%gap)

            cv2.rectangle(show, (l,t), (r,b), (0,0,0), 2)
            cv2.imshow('show', show)
            key = cv2.waitKey(1)
            if key == 27:
                exit()
            continue


# https://stackoverflow.com/questions/34581255/python-flask-socketio-send-message-from-thread-not-always-working
# Set this variable to "threading", "eventlet" or "gevent" to test the
# different async modes, or leave it set to None for the application to choose
# the best option based on available packages.
async_mode = None

if async_mode is None:
    try:
        import eventlet
        async_mode = 'eventlet'
    except ImportError:
        pass

    if async_mode is None:
        try:
            from gevent import monkey
            async_mode = 'gevent'
        except ImportError:
            pass

    if async_mode is None:
        async_mode = 'threading'

    print('async_mode is ' + async_mode)

# monkey patching is necessary because this application uses a background
# thread
if async_mode == 'eventlet':
    import eventlet
    eventlet.monkey_patch()
elif async_mode == 'gevent':
    from gevent import monkey
    monkey.patch_all()

import json
from flask import Flask, render_template
from flask_socketio import SocketIO
from kafka import KafkaConsumer
#因为第一步骤安装好了flask，所以这里可以引用

app = Flask(import_name=__name__,
            static_url_path='/static', # 配置静态文件的访问 url 前缀
            static_folder='./vueproj/dist/static',    # 配置静态文件的文件夹
            template_folder='./vueproj/dist/') # 配置模板文件的文件夹

app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app, async_mode=async_mode)
thread = None
# 实例化一个consumer，接收topic为wordStats的消息
consumer = KafkaConsumer('wordStats')

# 一个后台线程，持续接收Kafka消息，并发送给客户端浏览器
def background_thread():
    print("-- now run back ground thread ----")
    for msg in consumer:
        wordStats = msg.value.decode('utf8')
        print("-------------- stat data ------------")
        print(wordStats)
        socketio.emit('wordStats_msg',{'data':wordStats})

# 客户端发送connect事件时的处理函数
@socketio.on('test_connect')
def connect(message):
    print(message)
    global thread
    if thread is None:
        print("now start back ground task")
        # 单独开启一个线程给客户端发送数据
        thread = socketio.start_background_task(target=background_thread)
    socketio.emit('connected', {'data': 'Connected'})

# 通过访问http://127.0.0.1:5000/访问index.html
@app.route("/")
def handle_mes():
    return render_template("index.html")

# main函数
if __name__ == '__main__':
    socketio.run(app,debug=True)


from bisect import bisect_left
def fibSearch(arr, x, n):
    fib2 = 0 # (m-2)'th Fibonacci No.
    fib1 = 1 # (m-1)'th Fibonacci No.
    fibM = fib2 + fib1 # m'th Fibonacci
    while (fibM < n):
        fib2 = fib1
        fib1 = fibM
        fibM = fib2 + fib1
    offset = -1;
    while (fibM > 1):
        i = min(offset+fib2, n-1)
        if (arr[i] < x):
            fibM = fib1
            fib1 = fib2
            fib2 = fibM - fib1
            offset = i
        elif (arr[i] > x):
            fibM = fib2
            fib1 = fib1 - fib2
            fib2 = fibM - fib1
        else :
            return i
    if(fib1 and arr[offset+1] == x):
        return offset+1;
    return -1


from airflow import DAG
from airflow.utils.dates import days_ago
from airflow.operators.python import PythonOperator

# import tasks
from m1_script import task1
from m2_script import task2
from postgres_script import task3
from dash_script import task4

default_args = {
    "owner": "airflow",
    "depends_on_past": False,
    'start_date': days_ago(2),
    "retries": 1,
}

dag = DAG(
    'accidents_etl_pipeline',
    default_args=default_args,
    description='accidents etl pipeline',
)
with DAG(
        dag_id='accidents_etl_pipeline',
        schedule_interval='@once',
        default_args=default_args,
        tags=['accidents-pipeline'],
) as dag:
    cleaning = PythonOperator(
        task_id='cleaning',
        python_callable=task1,
    )
    add_feature = PythonOperator(
        task_id='add_feature',
        python_callable=task2,
    )
    load_to_postgres_task = PythonOperator(
        task_id='load_to_postgres',
        python_callable=task3,
    )
    create_dashboard_task = PythonOperator(
        task_id='create_dashboard_task',
        python_callable=task4,

    )

    cleaning >> add_feature >> create_dashboard_task
    add_feature >> load_to_postgres_task


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from targets.firefox.fx_testcase import *

class Test(FirefoxTest):

    @pytest.mark.details(
        description='Bug 1325651 - Safe browsing is working in safe mode',
        test_case_id='215325',
        test_suite_id='69',
        locale=['en-US'],
        profile=Profiles.BRAND_NEW
    )
    def run(self, firefox):
        start_in_safe_mode_button_pattern = Pattern('start_in_safe_mode_button.png')
        about_support_title_pattern = Pattern('about_support_title.png')
        restart_button_pattern = Pattern('restart_button.png')

        navigate(LocalWeb.FIREFOX_TEST_SITE)

        test_site_opened = exists(LocalWeb.FIREFOX_IMAGE, FirefoxSettings.SITE_LOAD_TIMEOUT)
        assert test_site_opened, 'Firefox test site opened'

        next_tab()

        navigate(LocalWeb.POCKET_TEST_SITE)

        test_site_opened = exists(LocalWeb.POCKET_IMAGE, FirefoxSettings.SITE_LOAD_TIMEOUT)
        assert test_site_opened, 'Pocket test site opened'

        new_tab()

        navigate('about:support')

        about_support_page_opened = exists(about_support_title_pattern, FirefoxSettings.FIREFOX_TIMEOUT)
        assert about_support_page_opened, 'About Support page opened'

        about_support_title_loc = find(about_support_title_pattern)
        width, height = about_support_title_pattern.get_size()
        buttons_region = Region(about_support_title_loc.x, about_support_title_loc.y, width*2, height*5)

        copy_text_button_exists = exists('Copy raw data to clipboard', FirefoxSettings.FIREFOX_TIMEOUT,
                                         region=buttons_region)
        assert copy_text_button_exists, 'Copy raw data to clipboard button exists'

        click('Copy raw data to clipboard', region=buttons_region)

        time.sleep(FirefoxSettings.SHORT_FIREFOX_TIMEOUT)

        clipboard_before_restart = get_clipboard().partition('safeMode')

        restart_button_exists = exists('Restart with Add-ons Disabled', FirefoxSettings.FIREFOX_TIMEOUT)
        assert restart_button_exists, 'Restart with Add-ons Disabled button displayed'

        click('Restart with Add-ons Disabled')

        restart_button_displayed = exists(restart_button_pattern, FirefoxSettings.FIREFOX_TIMEOUT)
        assert restart_button_displayed, 'Restart button displayed'

        click(restart_button_pattern)

        start_in_safe_mode_button_displayed = exists(start_in_safe_mode_button_pattern, FirefoxSettings.FIREFOX_TIMEOUT)
        assert start_in_safe_mode_button_displayed, 'Start in safe mode button displayed'

        click(start_in_safe_mode_button_pattern)

        about_support_page_opened = exists(about_support_title_pattern, FirefoxSettings.SITE_LOAD_TIMEOUT)
        assert about_support_page_opened, 'About Support page opened'

        copy_text_button_exists = exists('Copy raw data to clipboard', FirefoxSettings.FIREFOX_TIMEOUT,
                                         region=buttons_region)
        assert copy_text_button_exists, 'Copy raw data to clipboard button exists'

        click('Copy raw data to clipboard', region=buttons_region)

        time.sleep(FirefoxSettings.SHORT_FIREFOX_TIMEOUT)

        clipboard_after_restart = get_clipboard().partition('safeMode')
        assert clipboard_after_restart[0] in clipboard_before_restart[0], 'about:support should have all information ' \
                                                                          'filled in'

        select_tab('1')

        test_site_opened = exists(LocalWeb.FIREFOX_IMAGE, FirefoxSettings.SITE_LOAD_TIMEOUT)
        assert test_site_opened, 'Firefox test site opened'

        select_tab('2')

        test_site_opened = exists(LocalWeb.POCKET_IMAGE, FirefoxSettings.SITE_LOAD_TIMEOUT)
        assert test_site_opened, 'Pocket test site opened'


#mcb.py - Saves and loads pieces of text to the clipboard.
#Usage: mcb.py save <keyword> - Saves clipboard to keyword.
#       mcb.py <keyword> - Loads keyword to clipboard.
#       mcb.py list - Loads all keywods to clipboard.

import shelve, pyperclip, sys

mcbShelf = shelve.open('mcb')

#Save clipboard content.
if len(sys.argv) == 3 and sys.argv[1].lower() == 'save':
	mcbShelf[sys.argv[2]] = pyperclip.paste()
elif len(sys.argv) == 2:
	#List keywords and load content.
	if sys.argv[1].lower() == 'list':
		pyperclip.copy(str(list(mcbShelf.keys())))
	elif sys.argv[1] in mcbShelf:
		pyperclip.copy(mcbShelf[sys.argv[1]])

mcbShelf.close()


'''
Root task (Crunchbase)
========================

Luigi routine to collect all data from the Crunchbase data dump and load it to MySQL.
'''

import luigi
import datetime
import logging

from nesta.core.routines.datasets.crunchbase.crunchbase_parent_id_collect_task import ParentIdCollectTask
from nesta.core.routines.datasets.crunchbase.crunchbase_geocode_task import CBGeocodeBatchTask
from nesta.core.luigihacks.misctools import find_filepath_from_pathstub as f3p
from nesta.core.orms.crunchbase_orm import Base
from nesta.core.orms.orm_utils import get_class_by_tablename

class RootTask(luigi.WrapperTask):
    '''A dummy root task, which collects the database configurations
    and executes the central task.

    Args:
        date (datetime): Date used to label the outputs
        db_config_path (str): Path to the MySQL database configuration
        production (bool): Flag indicating whether running in testing
                           mode (False, default), or production mode (True).
    '''
    date = luigi.DateParameter(default=datetime.date.today())
    production = luigi.BoolParameter(default=False)
    insert_batch_size = luigi.IntParameter(default=500)
    db_config_path = luigi.Parameter(default=f3p("mysqldb.config"))
    db_config_env = luigi.Parameter(default="MYSQLDB")

    def requires(self):
        '''Collects the database configurations and executes the central task.'''
        _routine_id = "{}-{}".format(self.date, self.production)

        logging.getLogger().setLevel(logging.INFO)
        yield ParentIdCollectTask(date=self.date,
                                  _routine_id=_routine_id,
                                  test=not self.production,
                                  insert_batch_size=self.insert_batch_size,
                                  db_config_path=self.db_config_path,
                                  db_config_env=self.db_config_env)

        geocode_kwargs = dict(date=self.date,
                              _routine_id=_routine_id,
                              test=not self.production,
                              db_config_env="MYSQLDB",
                              insert_batch_size=self.insert_batch_size,
                              env_files=[f3p("nesta"),
                                         f3p("config/mysqldb.config"),
                                         f3p("config/crunchbase.config")],
                              job_def="py37_amzn2",
                              job_queue="HighPriority",
                              region_name="eu-west-2",
                              poll_time=10,
                              memory=4096,
                              max_live_jobs=2)

        for tablename in ['organizations', 'funding_rounds', 'investors', 'people', 'ipos']:
            _class = get_class_by_tablename(Base, f'crunchbase_{tablename}')
            yield CBGeocodeBatchTask(city_col=_class.city,
                                     country_col=_class.country,
                                     location_key_col=_class.location_id,
                                     job_name=f"Crunchbase-{tablename}-{_routine_id}",
                                     **geocode_kwargs)


from typing import List
import torch
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F

class RewriteDecider(nn.Module):
    def __init__(self, hps, device=torch.device("cpu")):
        super(RewriteDecider, self).__init__()
        self._input_size = hps["encoder"]["output_size"]
        self.layers = nn.Sequential(
            nn.Linear(self._input_size, hps["rewriter_hidden_size"]),
            nn.Sigmoid(),
            nn.Linear(hps["rewriter_hidden_size"], 1),
            nn.Sigmoid()
        )
        self.to(device)
        self.optim = optim.Adam(self.parameters(), lr=hps["learning_rate"])

    def forward(self, rnn_outputs: List[torch.Tensor]) -> torch.Tensor:
        """
        :param rnn_outputs: list of [1, complexity_feature_hidden_state]
        :return:
        """
        outputs = list()
        for encoding in rnn_outputs:
            outputs.append(self.layers(encoding))
        outputs = torch.cat(outputs, dim=1)
        return outputs

if __name__ == "__main__":
    import numpy as np

    hps = dict(complexity_split=10, encoder=dict(output_size=36), rewriter_hidden_size=32)
    rnn_output = [torch.tensor(np.random.uniform(-50, 50, [1, 1, 36]), dtype=torch.float)
                  for _ in range(10)]
    classifier = RewriteDecider(hps)
    print(classifier(rnn_output).shape)


from datetime import datetime, date

import graphene
from graphene import relay

from authentication.types import UserType
from class_payment.fields import MyClassPaymentMasterField
from class_payment.fields.is_approved_class_payment_master_field import IsApprovedClassPaymentMasterField
from class_payment.fields.my_gym_class_payment_master_field import MyGymClassPaymentMasterField
from class_payment.models import ClassPaymentMaster
from class_payment.mutations.class_cancel_payment import ClassCancelPayment
from class_payment.mutations.class_deposit_without_account import ClassDepositWithoutAccount
from class_payment.mutations.hello_world import HelloWorld
from class_payment.mutations.is_approved import IsApproved
from class_payment.mutations.paid_check import PaidCheck
from class_payment.mutations.send_class_payment_alarm import SendClassPaymentAlarm
from class_payment.mutations.update_class_payment_master import UpdateClassPaymentMaster
from class_payment.types.class_payment_master.class_payment_master_node import ClassPaymentMasterNode
from class_payment.types.gym_bank_account_type import GymBankAccountType
from class_payment.types.payment_per_month_type import PaymentPerMonthType
from gym_class.models import ClassMaster, AttendanceDetail
from gym_class.types.attendance.attendance_type import AttendanceType

class Query(graphene.ObjectType):
    my_class_payment_masters = MyClassPaymentMasterField(ClassPaymentMasterNode)
    class_payment_master = relay.Node.Field(ClassPaymentMasterNode)
    my_gym_class_payment_masters = MyGymClassPaymentMasterField(ClassPaymentMasterNode, filtering_name=graphene.String(default_value=''), filtering_days=graphene.Int(default_value=3000))
    absent_history = graphene.List(AttendanceType, class_payment_master_id=graphene.Int())
    gym_account_info = graphene.Field(GymBankAccountType)
    payment_per_month = graphene.List(PaymentPerMonthType, year=graphene.Int())
    is_approved_class_payment_masters = IsApprovedClassPaymentMasterField(ClassPaymentMasterNode)
    @staticmethod
    def resolve_payment_per_month(_, info, year):
        gym = info.context.user.gym
        class_masters = gym.class_masters.all()
        gym_created = gym.date_created
        gym_created_year = int(gym_created.strftime('%Y'))
        gym_created_month = int(gym_created.strftime('%m'))
        if year == datetime.today().year:
            month = datetime.today().month
        else:
            month = 12
        payment_list = []
        for i in range(month):
            if date(gym_created_year, gym_created_month, 1) <= date(year, i+1, 1):
                class_payment_masters = ClassPaymentMaster.objects.filter(class_master__in=class_masters,
                                                                          date_paid__year=year,
                                                                          date_paid__month=i+1,
                                                                          payment_status="납부완료")
                amount = 0
                for class_payment_master in class_payment_masters:
                    amount += class_payment_master.price_to_pay
                payment_dic = {
                    'month': i+1,
                    'amount': amount
                }
                payment_list.append(payment_dic)
        return payment_list

    @staticmethod
    def resolve_gym_account_info(_, info):
        parent = info.context.user.parent
        student = parent.students.first()
        gym = student.class_master.gym
        info_dic = {
            'bank_name': gym.refund_bank_name,
            'bank_owner_name': gym.refund_bank_owner_name,
            'bank_account_no': gym.refund_bank_account_no
        }
        return info_dic

    @staticmethod
    def resolve_class_payment_masters(_, info, class_master_id=None, did_paid=None, ):
        gym = info.context.user.gym
        if class_master_id:
            class_masters = ClassMaster.objects.filter(pk=class_master_id, gym=gym)
        else:
            class_masters = gym.class_masters.all()

        if did_paid:
            return ClassPaymentMaster.objects.filter(class_master__in=class_masters,
                                                     did_paid=did_paid)
        else:
            return ClassPaymentMaster.objects.filter(class_master__in=class_masters)

    @staticmethod
    def resolve_class_payment_master(_, __, class_payment_master_id):
        return ClassPaymentMaster.objects.get(pk=class_payment_master_id)

    @staticmethod
    def resolve_absent_history(_, __, class_payment_master_id):
        class_payment_master = ClassPaymentMaster.objects.get(pk=class_payment_master_id)
        student = class_payment_master.student
        return AttendanceDetail.objects.filter(student=student, type='결석',
                                               attendance_master__date__range
                                               =[class_payment_master.date_from, class_payment_master.date_to]
                                               ).exclude(absent_reason='무단결석')

    @staticmethod
    def resolve_payment_by_parent(_, info):
        parent = info.context.user.parent.get()
        return ClassPaymentMaster.objects.filter(student__in=parent.students.all(),
                                                 payment_method="미납")

    @staticmethod
    def resolve_payment_completion_by_parent(_, info, year, month):
        parent = info.context.user.parent.get()
        new_month = 0
        new_year = 0
        if month+1 == 13:
            new_month = 1
            new_year = year+1
        return ClassPaymentMaster.objects.filter(student__in=parent.students.all(),
                                                 payment_status="완납",
                                                 date_paid__lt=datetime(new_year, new_month, 1),
                                                 date_paid__gte=datetime(year, month, 1))
    """
    @staticmethod
    def resolve_class_payment_master_by_parent(_, info):
        parent = info.context.user.parent.get()
        return ClassPaymentMaster.objects.filter(student__parent=parent,
                                                 is_approved=True)
    """

class Mutation(graphene.ObjectType):
    paid_check = PaidCheck.Field()
    class_cancel_payment = ClassCancelPayment.Field()
    class_deposit_without_account = ClassDepositWithoutAccount.Field()
    is_approved = IsApproved.Field()
    update_class_payment_master = UpdateClassPaymentMaster.Field()
    send_class_payment_alarm = SendClassPaymentAlarm.Field()
    hello_world = HelloWorld.Field()
schema = graphene.Schema(query=Query, mutation=Mutation)


import boto3
from dotenv import load_dotenv
import os

# Load environment variables from .env file
load_dotenv()

access_key = os.getenv("AWS_ACCESS_KEY_ID")
secret_key = os.getenv("AWS_SECRET_ACCESS_KEY")
region = 'us-west-2'  # Replace with your desired region

# Create a Boto3 client for EC2 with explicit credentials
ec2_client = boto3.client('ec2', region_name=region, aws_access_key_id=access_key, aws_secret_access_key=secret_key)

response = ec2_client.describe_images(
    Filters=[
        {'Name': 'name', 'Values': ['ubuntu/images/hvm-ssd/ubuntu-focal-22.04-amd64-server-*']},
        {'Name': 'architecture', 'Values': ['x86_64']},
        {'Name': 'state', 'Values': ['available']},
    ],
    Owners=['099720109477']  # Canonical owner ID
)

images = response['Images']
images.sort(key=lambda x: x['CreationDate'], reverse=True)

if images:
    latest_ubuntu_2204_ami_id = images[0]['ImageId']
    print("Latest Ubuntu Server 22.04 LTS AMI ID:", latest_ubuntu_2204_ami_id)
else:
    print("No matching AMI found.")


"Load data from file for pipeline/transport/injection costs"

import logging
from pathlib import Path
from typing import Dict, List

import pandas as pd

from powergenome.params import DATA_PATHS
from powergenome.price_adjustment import inflation_price_adjustment
from powergenome.util import snake_case_col, snake_case_str

logger = logging.getLogger(__name__)

def merge_co2_pipeline_costs(
    df: pd.DataFrame,
    co2_data_path: Path,
    co2_pipeline_filters: List[dict],
    region_aggregations: Dict[str, List[str]] = None,
    fuel_emission_factors: Dict[str, float] = None,
    target_usd_year: int = None,
) -> pd.DataFrame:
    """Merge columns with CO2 pipeline costs to a dataframe.

    Data come from an external file. Filters from the settings are used to select the
    data that will be merged.

    An example of the YAML settings parameter would be:

    co2_pipeline_filters:
      - technology: NaturalGas
        tech_detail: CCCCSAvgCF
        with_backbone: false
        percentile: 25

    Parameters
    ----------
    df : pd.DataFrame
        Generators data. Should include the rows "technology" and "region", along with a
        heat rate column that has "heat_rate" and "per_mwh" if fuel emission factors
        will be used to convert costs from per tonne to per MWh.
    co2_data_path : Path
        Path to the file with CO2 pipeline cost data
    co2_pipeline_filters : List[dict]
        Filters to apply to the CO2 cost data file
    region_aggregations : Dict[str, List[str]], optional
        Aggregations of multiple base regions to model regions, by default None
    fuel_emission_factors : Dict[str, float], optional
        CO2 emissions per unit of energy for different fuel types, by default None
    target_usd_year : int, optional
        The target year for cost inflation adjustment, by default None

    Returns
    -------
    pd.DataFrame
        The input dataframe (one row per resource) plus columns with cost data for CO2
        pipeline constuction/operation and CO2 disposal
    """
    co2_df = pd.read_csv(co2_data_path)
    co2_df = co2_df.loc[co2_df["parameter"] != "capacity_mw", :]
    if target_usd_year:
        for dollar_year in co2_df["dollar_year"].unique():
            co2_df.loc[
                (co2_df["dollar_year"] == dollar_year), "parameter_value"
            ] = inflation_price_adjustment(
                co2_df.loc[co2_df["dollar_year"] == dollar_year, "parameter_value"],
                dollar_year,
                target_usd_year,
            )
    for k, v in (region_aggregations or {}).items():
        co2_df.loc[co2_df["region"].isin(v), "region"] = k

    # When regions are aggregated, use the lowest-cost ipm region
    co2_df = co2_df.sort_values("parameter_value").drop_duplicates(
        [
            "region",
            "technology",
            "tech_detail",
            "with_backbone",
            "percentile",
            "parameter",
        ],
        keep="first",
    )

    # co2_df["resource"] = (co2_df["technology"] + "_" + co2_df["tech_detail"]).lower()
    filter_techs = [
        filt["technology"] + "_" + filt["tech_detail"] for filt in co2_pipeline_filters
    ]
    co2_resources = {}
    for co2_r in filter_techs:
        for model_r in df["technology"].unique():
            if snake_case_str(co2_r) in snake_case_str(model_r):
                co2_resources[co2_r] = model_r

    if not co2_resources:
        logger.warning(
            "You have specified technologies for CO2 pipeline costs in the parameter "
            "'co2_pipeline_filters' but none of the technologies are being used in "
            "your system."
        )
        return df

    df_list = []
    for filt in co2_pipeline_filters:
        # tech = filt["technology"].lower() + "_" + filt["tech_detail"].lower()
        query_str = f"technology == '{filt['technology']}'"
        for k, v in filt.items():
            if k != "technology":
                if isinstance(v, str):
                    query_str += f" and {k} == '{v}'"
                else:
                    query_str += f" and {k} == {v}"

        co2_costs = co2_df.query(query_str)
        co2_costs.loc[:, "technology"] = (
            co2_costs["technology"] + "_" + co2_costs["tech_detail"]
        ).map(co2_resources)
        co2_costs_wide = co2_costs.pivot(
            index=["region", "technology"],
            columns="parameter",
            values="parameter_value",
        ).reset_index()
        co2_costs_wide["capture_rate"] = filt.get("capture_rate", 90)
        df_list.append(co2_costs_wide)

    full_co2_costs_wide = pd.concat(df_list)
    # Split incoming df into techs with and without pipeline cost. Remove pipeline cost
    # columns from the "with" techs to avoid duplicates, merge in co2 costs, then concat
    # with the "without" techs
    df = df.reset_index()
    drop_cols = [
        c for c in full_co2_costs_wide.columns if c not in ["region", "technology"]
    ]
    df_co2_techs = df.loc[df["technology"].isin(co2_resources.values()), :]
    df_co2_techs = df_co2_techs.drop(columns=drop_cols, errors="ignore")
    df_co2_techs = pd.merge(
        df_co2_techs, full_co2_costs_wide, on=["region", "technology"], how="left"
    )
    df_non_co2_techs = df.loc[~df["technology"].isin(co2_resources.values()), :]

    df_co2_costs = (
        pd.concat([df_co2_techs, df_non_co2_techs]).set_index("index").sort_index()
    )

    drop_idx = df_co2_costs.loc[
        (df_co2_costs["technology"].isin(co2_resources.values()))
        & (df_co2_costs[co2_costs_wide.columns].isna().any(axis=1))
    ].index

    if not drop_idx.empty:
        for tech, _df in df_co2_costs.loc[drop_idx, :].groupby("technology"):
            regs = _df["region"].to_list()
            f"The CCS resource {tech} is being removed from regions {regs} because cost "
            "data was not included in your CO2 pipeline cost file."
    df_co2_costs = df_co2_costs.drop(index=drop_idx)
    df_co2_costs.loc[:, co2_costs_wide.columns] = df_co2_costs[
        co2_costs_wide.columns
    ].fillna(0)

    mass_cols = [c for c in co2_costs_wide.columns if "tonne" in c]
    if mass_cols and fuel_emission_factors:
        df_co2_costs = mass_to_energy_costs(
            df_co2_costs, mass_cols, fuel_emission_factors
        )

    return df_co2_costs

def mass_to_energy_costs(
    df: pd.DataFrame,
    mass_cols: List[str],
    fuel_emission_factors: Dict[str, float],
    heat_rate_col: str = None,
) -> pd.DataFrame:
    """Convert costs per tonne of CO2 to a MWh basis

    Parameters
    ----------
    df : pd.DataFrame
        Generators dataframe with a fuel column, a heat rate column, and the columns
        provided in the list `mass_cols`
    mass_cols : List[str]
        Names of columns with cost per tonne of CO2 disposal
    fuel_emission_factors : Dict[str, float]
        Emission rate of tonne per unit of energy for fuels used by CCS generators
    heat_rate_col : str, optional
        Name of the column with generator heat rates, by default None

    Returns
    -------
    pd.DataFrame
        Modified copy of the input, with the new column "co2_cost_mwh"
    """
    if not heat_rate_col:
        heat_rate_cols = [
            c for c in df.columns if "heat_rate" in c.lower() and "per_mwh" in c.lower()
        ]
        if len(heat_rate_cols) > 1:
            logger.warning(
                "More than one column with 'heat_rate' and 'per_mwh' was found when "
                "converting CO2 transport costs from per tonne to per MWh. Values are "
                "not being converted."
            )
            return df

        elif not heat_rate_cols:
            logger.warning(
                "Unable to identify a heat rate column when converting CO2 transport costs "
                "from per tonne to per MWh. A heat rate column should contain 'heat_rate' and "
                "'per_mwh' (capitalization does not matter). Values are not being converted."
            )
            return df
        else:
            heat_rate_col = heat_rate_cols[0]

    fuel_col = [c for c in df.columns if c.lower() == "fuel"]
    fuel_col = fuel_col[0]
    for fuel, ef in fuel_emission_factors.items():
        df.loc[
            df[fuel_col].str.contains(fuel, case=False), "tonne_co2_captured_mwh"
        ] = (
            df.loc[df[fuel_col].str.contains(fuel, case=False), heat_rate_col]
            * ef
            * (
                df.loc[df[fuel_col].str.contains(fuel, case=False), "capture_rate"]
                / 100
            )
        )

    df.loc[:, "co2_cost_mwh"] = df[mass_cols].sum(axis=1) * df["tonne_co2_captured_mwh"]
    df.loc[:, "co2_cost_mwh"] = df.loc[:, "co2_cost_mwh"].fillna(0)

    return df


import json
from pathlib import Path
import datetime

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

###############
# Open browser #

def open_browser():
	# Open browser
	global driver
	driver = webdriver.Firefox()
	print("Browser opened")

###############
# Close browser #

def close_browser():
	#close browser
	driver.quit()
	print("Browser closed")

###############
# Loads a .json file from the same directory as this script file #
# returns the result  #

def return_load_json_file(filename):

	print("loading file... ", filename)

	try:
		f = open(filename)
		json_data = json.load(f)
		f.close()

	except:
		print("Error: Failed to load file")
	else:
		print("Success!")
		return json_data

###############
# write data to json file
def write_to_json_file(filename, input_data):
    with open(filename + '.json', 'w') as json_file:
        json.dump(input_data, json_file, sort_keys=True, indent=4, separators=(',', ': '))

##############
# Searches 'player_data' in database #
# Creates a list of all 'fpl_code' values #
# Returns the result #

def return_manager_codes_as_list():

	manager_codes = []

	#create a list contaiing the FPL code of every manager in the database
	for key, val in database['player_data'].items():

		manager_fpl_code = val['manager_info']['fpl_code']
		manager_codes.append(manager_fpl_code)

	return manager_codes

##############
# Scrapes pervious season scores #
# stores them to player_data > fpl_history #
def scrape_and_return_fpl_history(manager_code):

	manager_URL = 'https://fantasy.premierleague.com/entry/'+manager_code+'/history'

	try:
		#open the webpage
		driver.get(manager_URL)

		#wait for the league table to appear in DOM
		element = WebDriverWait(driver, 10).until(
			EC.presence_of_element_located((By.CSS_SELECTOR, "table.Table-ziussd-1.fVnGhl"))
		)

	except:
		#if the table is not found, display an error message
		print("Error: Table element not found :(")

	else:
		#if the table is found, display successs message
		print("Success: Table element found :D")

		#create soup of page DOM
		soup = BeautifulSoup(driver.page_source, 'lxml')

		#locate 'Previous seasons' table
		element = soup.find("h3", text="Previous Seasons")

		#move up the soup DOM until the table is found
		while len(element.select('.Table-ziussd-1.fVnGhl')) == 0:
			element = element.parent
		else:
			prev_seasons_rows = element.select('.Table-ziussd-1.fVnGhl tbody tr')

			fpl_history = {}

			for row in prev_seasons_rows:
				row_contents_array = row.contents

				season = row_contents_array[0].get_text().replace("/", "_")
				score = row_contents_array[1].get_text()

				fpl_history[season] = score

			return fpl_history

def return_create_date_stamp():
	now = datetime.datetime.now()

	year = now.strftime("%Y")
	month = now.strftime("%m")
	day = now.strftime("%d")
	hour = now.strftime("%H")
	minute = now.strftime("%M")

	datestamp = year + month + day + "_" + hour + minute
	return datestamp

def execute():

	global database
	database = return_load_json_file('chumpionship_2021_database - new.json')

	open_browser()

	for manager_code in return_manager_codes_as_list():
		database['player_data'][manager_code]['manager_info']['fpl_history'] = scrape_and_return_fpl_history(manager_code)

	close_browser()

	write_to_json_file('chumpionship_2021_database---history', database)

execute()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# coding: utf-8
import numpy as np
from scipy.optimize import fmin_bfgs

def sigmoid(z):
    g = 1.0/(1.0+np.exp(-z))
    return g

def lrGD(theta, *args):
    theta = np.matrix(theta).transpose()
    X, y, the_lambda = args
    m = y.shape[0]
    grad = np.zeros(theta.shape)

    htheta = sigmoid(X * theta)
    grad[0] = 1.0/m * np.sum(np.array((htheta-y))*np.array(X[:, 0:1]))
    for i in range(1, theta.shape[0]):
        grad[i] = 1.0/m * np.sum(np.array((htheta-y)) *
                                 np.array(X[:, i:i+1])) + the_lambda/m*theta[i]
    return grad.flatten()

def lrCostFunction(theta, *args):
    theta = np.matrix(theta).transpose()
    X, y, the_lambda = args
    m = y.shape[0]
    htheta = sigmoid(X*theta)
    J = -np.sum(np.array(y)*np.array(np.log(htheta)) + np.array((1-y))
                * np.array(np.log(1-htheta)))/m + the_lambda*np.sum(
        np.array(theta[1:]) * np.array(theta[1:]))/(2*m)
    return J

def oneVsAll(X, y, num_labels, the_lambda):
    m, n = np.shape(X)
    all_theta = np.matrix(np.zeros((num_labels, n+1)))
    X = np.hstack((np.ones((m, 1)), X))
    initial_theta = np.zeros((n+1, 1))
    for c in range(num_labels):
        print ('Training for %d/%d' % (c+1, num_labels))
        args = (X, (y == c), the_lambda)
        theta = fmin_bfgs(lrCostFunction, initial_theta,
                          fprime=lrGD, args=args, maxiter=50)
        all_theta[c, :] = theta.transpose()

    return all_theta

def train():
    '''开始训练'''
    num_labels = 36
    the_lambda = 0.1
    data = np.matrix(np.loadtxt('data.dat'))
    y = data[:, 336]
    X = data[:, :336]
    all_theta = oneVsAll(X, y, num_labels, the_lambda)
    ''' 生成训练后的特征文本 '''
    np.savetxt('training_model/scipy_model/theta.dat', all_theta)

train()


from selenium import webdriver
import time
import os
from openpyxl import Workbook
import pandas as pd
from openpyxl.utils.dataframe import dataframe_to_rows

from links import links, country_name, zipcode, radius
from parsers.parsers import PageParser

'''
Files takes in list object from links.py
will take list object and then run chromedriver on each link in list
take data [list object] from each individual page and add to dictionary
take dictionary and write to excel file
'''

data=[]
chrome = webdriver.Chrome(executable_path = 'PASTE YOUR PATH HERE -- ex: /usr/local/bin/chromedriver.exec')

#loop iterates through each link.. goes to page.. and extracts info
for i in links:
    chrome.get(i)
    time.sleep(2)
    parser=PageParser(chrome)
    dealer = parser.extract_dealer_info
    data.append(dealer)
    time.sleep(1)

#converting data to dataframe
dataframe= pd.DataFrame(data=data, columns=['Dealer Name', 'Address', 'City', 'Website'])

#writing to excel file
i=0
file = f'cedia_dealers_{country_name}_{zipcode}_{radius.strip()}.xlsx'
while os.path.exists(file):
    i+=1
    file= f'cedia_dealers{i}_{country_name}_{zipcode}_{radius.strip()}.xlsx'

wb = Workbook()

ws = wb.active
ws.column_dimensions['A'].width = 40
ws.column_dimensions['B'].width = 40
ws.column_dimensions['C'].width = 40
ws.column_dimensions['D'].width = 40
ws.column_dimensions['E'].width = 40

for r in dataframe_to_rows(dataframe):
    ws.append(r)

wb.save(filename=file)
chrome.close()
print(f'File Saved.. {file}')


from django.urls import path, include

from .views import (
	ArticuloListView,
	ArticuloCreateView,
	ArticuloUpdateView,
	ArticuloDeleteView,
	Ingreso_DetalleCreateView,
	Salida_DetalleCreateView,
	GeneratePDF
	)

urlpatterns = [
	path("listarArticulo", ArticuloListView.as_view(), name="Listar_Articulo"),
	path("crearArticulo", ArticuloCreateView.as_view(), name="Crear_Articulo"),
	path("<pk>/editar", ArticuloUpdateView.as_view(), name="Editar_Articulo"),
	path("<pk>/borrar/", ArticuloDeleteView.as_view(), name="Eliminar_Articulo"),

	path('ingresar_articulo/', Ingreso_DetalleCreateView.as_view(), name='ingreso_articulo_detalle'),
	path('salir_articulo/', Salida_DetalleCreateView.as_view(), name='salida_articulo_detalle'),
	path('ingresos/', include('ingresos.urls')),
	path('salidas/', include('salidas.urls')),
	path('pdf', GeneratePDF.as_view())
	]


from typing import List
import numpy as np
class Solution_1:
    def maximalSquare(self, matrix: List[List[str]]) -> int:
        if not matrix:
            return 0
        res = 0
        m, n = len(matrix), len(matrix[0])
        for i in range(m):
            for j in range(n):
                matrix[i][j] = int(matrix[i][j])
        matrix = np.array(matrix, dtype=np.int)
        for i in range(m):
            for j in range(n):
                if matrix[i][j] == 1:
                    cur = 1
                    x, y = i, j
                    while x + 1 < m and y + 1 < n and matrix[x+1, y+1] == 1:
                        if np.sum(matrix[i:x+2, j: y+2]) == matrix[i:x+2, j:y+2].size:
                            cur = matrix[i:x+2, j: y+2].size
                            x += 1
                            y += 1
                        else:
                            break
                    if cur > res:
                        res = cur
        return res

class Solution_2:
    def maximalSquare(self, matrix: List[List[str]]) -> int:
        if not matrix:
            return 0
        max_len = 0
        m, n = len(matrix), len(matrix[0])
        dp = [[0]*(n+1) for _ in range(m+1)]
        for i in range(1, m+1):
            for j in range(1, n+1):
                if matrix[i-1][j-1] == '1':
                    dp[i][j] = min([dp[i-1][j], dp[i-1][j-1], dp[i][j-1]]) + 1
                    if dp[i][j] > max_len:
                        max_len = dp[i][j]
        return max_len ** 2

class Solution:
    def maximalSquare(self, matrix: List[List[str]]) -> int:
        if not matrix:
            return 0
        max_len = 0
        m, n = len(matrix), len(matrix[0])
        dp = [0]*(n+1)
        for i in range(1, m+1):
            tmp = 0
            for j in range(1, n+1):
                if matrix[i-1][j-1] == '1':
                    cur = min([dp[j-1], dp[j], tmp]) + 1
                    dp[j-1] = tmp
                    tmp = cur
                    #print(i, j, cur)
                    if cur > max_len:
                        max_len = cur
                else:
                    dp[j-1] = tmp
                    tmp = 0
            dp[n] = tmp
        return max_len ** 2

'''
matrix = [['1', '0', '1', '0', '0'],
          ['1', '0', '1', '1', '1'],
          ['1', '1', '1', '1', '1'],
          ['1', '0', '0', '1', '0']]
'''
matrix = [['1', '0', '1', '0'],
          ['1', '0', '1', '1'],
          ['1', '0', '1', '1'],
          ['1', '1', '1', '1']]

obj = Solution()
print(obj.maximalSquare(matrix))


from tkinter import Tk,messagebox,Button
root = Tk()
root.title('MessageBox Widget')
root.geometry('600x400+800+80')
sinfo = lambda :messagebox.showinfo('Information','Good going till now!!')
swarm = lambda :messagebox.showwarning('Warning','Good not to stop here!')
serror = lambda :messagebox.showerror('Error','Increase your speed ')
sask = lambda :messagebox.askquestion('Question','Can you do it')
saskc = lambda :messagebox.askokcancel('Question','Can you keep it up')
saskyn = lambda :messagebox.askyesno('Question','Say yes ya no?')
saskrc = lambda :messagebox.askretrycancel('Question','This much done na?')
bt1 = Button(root,text='showinfo',command=sinfo).place(x=10,y=10)
bt2 = Button(root,text='showwarning',command=swarm).place(x=80,y=10)
bt3 = Button(root,text='showerror',command=serror).place(x=160,y=10)
bt4 = Button(root,text='askquestion',command=sask).place(x=240,y=10)
bt5 = Button(root,text='askokcancel',command=saskc).place(x=10,y=50)
bt6 = Button(root,text='askyesno',command=saskyn).place(x=80,y=50)
bt7 = Button(root,text='askretrycancel',command=saskrc).place(x=160,y=50)
root.mainloop()


import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torchvision.transforms as T
import numpy as np

# if gpu is to be used
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class Network(nn.Module):

# Layer	Input	    kernal 	    Stride	# filters	Activa	Output
# conv1	96x96x1	    8x8	        4	    32	        ReLU	[1, 32, 23, 23]
# conv2	23x23x32	4x4	        2	    64	        ReLU	[1, 64, 10, 10]
# conv3	10x10x64	3x3	        1	    64	        ReLU	[64, 8, 8]=4096
# fc4	8x8x64		                    512	        ReLU	512
# fc5	512			                    2	        Linear	2

    def __init__(self):
        super(Network, self).__init__()

        self.conv1 = nn.Conv2d(
            in_channels=1, out_channels=32, kernel_size=8, stride=4)
            # output = torch.Size([1, 32, 23, 23])
        self.relu1 = nn.ReLU(inplace=True)

        self.conv2 = nn.Conv2d(
            in_channels=32, out_channels=64, kernel_size=4, stride=2)
            # output = torch.Size([1, 64, 10, 10])
        self.relu2 = nn.ReLU(inplace=True)

        self.conv3 = nn.Conv2d(
            in_channels=64, out_channels=64, kernel_size=3, stride=1)
            # output = torch.Size([1, 64, 8, 8])
        self.relu3 = nn.ReLU(inplace=True)

    def forward(self, x):
        x = x.unsqueeze(1)

        out = self.conv1(x)
        out = self.relu1(out)

        out = self.conv2(out)
        out = self.relu2(out)

        out = self.conv3(out)
        out = self.relu3(out)

        return out

nn_network = Network()

view_dim = np.array([np.zeros((96, 96))])
# print(view_dim.shape)

screen = torch.tensor(view_dim, dtype=torch.float32, device=device)
out = nn_network.forward(screen)
print(out.shape)


class Solution:
    def containsNearbyDuplicate(self, nums, k) :
        D={}
        for i in range(len(nums)):
            if nums[i] in D and i-D[nums[i]]<=k:
                return True
            else:
                D[nums[i]]=i
        return False

x=Solution()
result=x.containsNearbyDuplicate([1,0,1,1], 1)
print(result)


pessoas = []
galera = []
continuar = 'S'
cont = maiorPeso = menorPeso = 0
pessoaMaiorPeso = []
pessoaMenorPeso = []
while continuar == 'S':
    pessoas.append(input('Nome: '))
    pessoas.append(input('Peso: '))
    galera.append(pessoas[:])
    pessoas.clear()
    cont += 1
    continuar = input('Quer continuar? [S/N]').strip().upper()[0]
maiorPeso = menorPeso = galera[0][1]
for pessoa in galera:
    if pessoa[1] >= maiorPeso:
        maiorPeso = pessoa[1]
    if pessoa[1] <= menorPeso:
        menorPeso = pessoa[1]
for pessoa in galera:
    if pessoa[1] == maiorPeso:
       pessoaMaiorPeso.append(pessoa[0])
    if pessoa[1] == menorPeso:
        pessoaMenorPeso.append(pessoa[0])

print(f'O maior peso foi de {maiorPeso}. Peso de {pessoaMaiorPeso}')
print(f'O menor peso foi de {menorPeso}. Peso de {pessoaMenorPeso}')
print(f'Foram cadastradas {cont} pessoas')
print(galera)


"""empty message

Revision ID: 06c3e3242926
Revises: 460bb858ff31
Create Date: 2021-05-13 23:42:08.735959

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '06c3e3242926'
down_revision = '460bb858ff31'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('name', sa.String(length=150), nullable=False))
    op.add_column('user', sa.Column('sex', sa.String(length=120), nullable=False))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('user', 'sex')
    op.drop_column('user', 'name')
    # ### end Alembic commands ###


phonebook = {}
file = input("Enter the name of your dictionary: ")
with open(file, 'r') as phone_book:
    while True:
        name = phone_book.readline().strip('\n')
        email = phone_book.readline().strip('\n')
        if name == "" :
            break
        else:
            phonebook[name] = email

def addressBook():
    menu = {}
    menu['1'] = "Lookup Email"
    menu['2'] = "Add Name/Email"
    menu['3'] = "Change Email"
    menu['4'] = "Delete Name/Email"
    menu['5'] = "Save and Exit"
    while True:
        options = menu.keys()
        for entry in options:
            print (entry, menu[entry])

        selection = input("Please Select a Option: ")
        if selection == '1':
            emailLookup = input("Please enter the name of the contact: ")
            if emailLookup in phonebook:
                print(phonebook[emailLookup])
            else:
                print ("Sorry. no contact exists under this name.")
        elif selection == '2':
            addContact = input("Please add a new name: ")
            if addContact in phonebook:
                print("Contact name already exists, please try again.")
            else:
                phonebook[addContact] = input("Enter a email address: ")
        elif selection == '3':
            changeEmail = input("Please enter the contact's name: ")
            if changeEmail in phonebook:
                phonebook[changeEmail] = input("Enter a new email address: ")
            else:
                print("Sorry, no contact exists under this name.")

        elif selection == '4':
            contactDelete = input("Please enter the name you wish to remove: ")
            if contactDelete in phonebook:
                del phonebook[contactDelete]
            else:
                print("Sorry, no contact exists under this name.")

        elif selection == '5':
            saveFile = str(input('Update dictionary? (Yes/No)'))
            if saveFile == 'Yes':
                file = input('Enter name of save file: ')
                with open(file, 'w') as saveData:
                    saveData.write(str(phonebook))
                print("Data saved. Have a nice day.")
                break
            else:
                print("Thank you and have a nice day")
                break
        else:
            print ("Please enter a valid menu option.")
addressBook()


import time
import json
import pandas as pd
import numpy as np
import requests

def get_global_data():
    url = r"""https://view.inews.qq.com/g2/getOnsInfo?name=disease_foreign&_={0}""".format(
        int(time.time()*1000))
    result = requests.get(url)
    data = json.loads(result.json()['data'])

    return data

def get_last_data():
    import requests
    url = r"""https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5&_={0}""".format(
        int(time.time()*1000))
    result = requests.get(url)
    data = json.loads(result.json()['data'])

    return data

def get_hist_data():
    import requests
    url = r"""https://view.inews.qq.com/g2/getOnsInfo?name=disease_other&_={0}""".format(
        int(time.time()*1000))
    result = requests.get(url)
    data = json.loads(result.json()['data'])
    return data

if __name__ == '__main__':
    last_data = get_last_data()
    with open('../data/tencent/data_today.json', 'w') as f:
        json.dump(last_data, f)

    hist_data = get_hist_data()
    with open('../data/tencent/hist_data.json', 'w') as f:
        json.dump(hist_data,f)

    glob_data = get_global_data()
    with open('../data/tencent/global_data.json', 'w') as f:
        json.dump(glob_data, f)