# imports
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from TaxiFareModel.encoders import TimeFeaturesEncoder, DistanceTransformer
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from TaxiFareModel.utils import compute_rmse
import mlflow, memoized_property
from mlflow.tracking import MlflowClient

class Trainer():
    def __init__(self, X, y):
        """
            X: pandas DataFrame
            y: pandas Series
        """
        self.pipeline = None
        self.X = X
        self.y = y
        self.experiment_name = "[#871] [Berlin] [KiKar31] pipe v1"

    def set_pipeline(self):
        """defines the pipeline as a class attribute"""
        dist_pipe = Pipeline([
            ('dist_trans', DistanceTransformer()),
            ('stdscaler', StandardScaler())
        ])
        time_pipe = Pipeline([
            ('time_enc', TimeFeaturesEncoder('pickup_datetime')),
            ('ohe', OneHotEncoder(handle_unknown='ignore'))
        ])
        preproc_pipe = ColumnTransformer([
            ('distance', dist_pipe, ["pickup_latitude", "pickup_longitude", 'dropoff_latitude', 'dropoff_longitude']),
            ('time', time_pipe, ['pickup_datetime'])
        ], remainder="drop")
        pipe = Pipeline([
            ('preproc', preproc_pipe),
            ('linear_model', LinearRegression())
        ])
        self.pipeline = pipe

    def run(self):
        """set and train the pipeline"""
        return self.pipeline.fit(self.X, self.y)

    def evaluate(self, X_test, y_test):
        """evaluates the pipeline on df_test and return the RMSE"""
        y_pred = self.pipeline.predict(X_test)
        return compute_rmse(y_pred, y_test)

    @memoized_property
    def mlflow_client(self):
        mlflow.set_tracking_uri("https://mlflow.lewagon.ai/")
        return MlflowClient()

    @memoized_property
    def mlflow_experiment_id(self):
        try:
            return self.mlflow_client.create_experiment(self.experiment_name)
        except BaseException:
            return self.mlflow_client.get_experiment_by_name(self.experiment_name).experiment_id

    @memoized_property
    def mlflow_run(self):
        return self.mlflow_client.create_run(self.mlflow_experiment_id)

    def mlflow_log_param(self, key, value):
        self.mlflow_client.log_param(self.mlflow_run.info.run_id, key, value)

    def mlflow_log_metric(self, key, value):
        self.mlflow_client.log_metric(self.mlflow_run.info.run_id, key, value)

if __name__ == "__main__":
    # get data
    # clean data
    # set X and y
    # hold out
    # train
    # evaluate
    print('TODO')


# -*- coding: utf-8 -*-
"""
Created on Sun Dec 13 17:05:23 2020

@author: Ruo-Yah Lai
"""

import matplotlib.pyplot as plt
import numpy as np
import csv
import ast

def spikePlots(turns, unit, subfield, filename, df, title=""):
    """
    turns: from alleyTransitions
    filename: the file with which locations a field is in
    df: path to the files with which locations a field is in
    """
    with open(df+filename, "r") as csv_file:
        data_iter = csv.reader(csv_file)
        data = [data for data in data_iter]
    fieldLoc = ast.literal_eval(data[subfield+1][1])
    turns2 = np.empty((0,5)) #turns in the field
    for turn in turns:
        allLoc = fieldLoc + list(turn[5:8])
        if len(set(allLoc)) < len(allLoc):
            #exclude back around turns that are not in the field
            if len(set(turn[5:8])) == 2 and len(set(allLoc)) == len(allLoc)-1:
                continue
            turns2 = np.vstack((turns2, turn[:5].astype(float))) #allo before turn, ego, allo after turn, ts of exit, ts of entry

    fig, axs = plt.subplots(10,10,figsize=(20,20))
    axs = axs.flatten()
    for i in range(100):
        p = unit.position[np.logical_and(turns2[i,3]-0.5e6 < unit.position[:,0],
                                         unit.position[:,0] < turns2[i,4]+0.5e6)]
        s = unit.spikes[np.logical_and(turns2[i,3]-0.5e6 < unit.spikes[:,0],
                                         unit.spikes[:,0] < turns2[i,4]+0.5e6)]
        axs[i].plot(p[:,1], p[:,2], color=(0.5,0.5,0.5))
        axs[i].scatter(s[:,1], s[:,2], color="red", s=4, zorder=3)
        axs[i].scatter(p[0,1], p[0,2], c="green", s=4, zorder=4)
        axs[i].scatter(p[-1,1], p[-1,2], c="blue", s=4, zorder=4)
    fig.tight_layout()
    fig.suptitle(f"{title}\nGreen = beginning   Blue = end", y=1.02)


from data_loader import *
from plotter import *
from utils import *
import netCDF4 as nc
import argparse
import pandas as pd
from tkinter import *

# TODO: Add gui....eventually
# import customtkinter
#
# customtkinter.set_appearance_mode('system')
# root = customtkinter.CTk()
# root.geometry('300x400')
# button = customtkinter.CTkButton(master=root, text='Hello World!!!')
# button.place(relx=0.5, rely=0.5, anchor=CENTER)
# root.mainloop()

# TODO: add style checks
# TODO: add option to wget data
# TODO: plot sc orbit locations

# def process_spacecraft_data(g17_file, g18_file, gk2a_file, g17_deg,
# g18_deg, gk2a_deg, save_path):
def process_spacecraft_data(g17_file=None, g18_file=None, gk2a_file=None,
                            g17_deg=None, g18_deg=None, gk2a_deg=None,
                            save_path=None):
    # For multiple s/c, one day is typical, unless use aggregate_nc_file to
    # look at multiple days at a time.
    # Plots mag inclination angle
    noonmidnighttimes_dict = {}
    goes_time_fromnc = None
    goes17_bgse_stacked = goes18_bgse_stacked = gk2a_bgse_stacked = None

    if g17_file:
        goes17coloc_dataset = nc.Dataset(g17_file)
        goes17_bgse_stacked = process_goes_dataset(
            goes17coloc_dataset['b_gse'])
        goes_time_fromnc = goes_epoch_to_datetime(
            goes17coloc_dataset['time'][:])
        goes17_VDH = gse_to_vdh(goes17_bgse_stacked, goes_time_fromnc)

    if g18_file:
        goes18coloc_dataset = nc.Dataset(g18_file)
        goes18_bgse_stacked = process_goes_dataset(
            goes18coloc_dataset['b_gse'])
        goes_time_fromnc = goes_epoch_to_datetime(
            goes18coloc_dataset['time'][:])
        goes18_VDH = gse_to_vdh(goes18_bgse_stacked, goes_time_fromnc)

    if gk2a_file:
        gk2a_dataset = nc.Dataset(gk2a_file)
        gk2a_bgse_stacked = stack_gk2a_data(gk2a_dataset)
        gk2a_VDH = gse_to_vdh(gk2a_bgse_stacked, goes_time_fromnc)

    date_str = get_date_str_from_goesTime(goes_time_fromnc)

    # Used to plot 'noon' and 'midnight' times (optional arg)
    noonmidnighttimes_dict = {}
    gk2a_noon, gk2a_midnight, g17_noon, g17_midnight, g18_noon, g18_midnight \
        = [
              None] * 6

    if gk2a_deg:
        gk2a_time_diff = calculate_time_difference(float(gk2a_deg), 'E')
        gk2a_noon, gk2a_midnight = find_noon_and_midnight_time(gk2a_time_diff,
                                                               date_str,
                                                               gk2a=True)
        noonmidnighttimes_dict['gk2a'] = {'noon': gk2a_noon,
                                          'midnight': gk2a_midnight}

    if g17_deg:
        g17_time_diff = calculate_time_difference(float(g17_deg))
        g17_noon, g17_midnight = find_noon_and_midnight_time(g17_time_diff,
                                                             date_str)
        noonmidnighttimes_dict['g17'] = {'noon': g17_noon,
                                         'midnight': g17_midnight}
    if g18_deg:
        g18_time_diff = calculate_time_difference(float(g18_deg))
        g18_noon, g18_midnight = find_noon_and_midnight_time(g18_time_diff,
                                                             date_str)
        noonmidnighttimes_dict['g18'] = {'noon': g18_noon,
                                         'midnight': g18_midnight}

    # Plot B field in GSE coords:
    plot_BGSE_fromdata_ontop(goes_time_fromnc, goes17_bgse_stacked,
                             goes18_bgse_stacked, 'G17', 'G18', 'SOSMAG',
                             gk2a_bgse_stacked, date_str, save_path,
                             noonmidnighttimes_dict)

    # Plot mag incl (theta) over time:
    plot_magnetic_inclination_over_time_3sc(date_str, goes_time_fromnc,
                                            goes17_VDH, goes18_VDH, gk2a_VDH,
                                            save_path, noonmidnighttimes_dict)

    # print(gk2a_noon, gk2a_midnight, g17_noon, g17_midnight, g18_noon,
    # g18_midnight)
    # print(noonmidnighttimes_dict)

# def analyze_pickle_data_statstics(data1, data2):
#     # Perform statistical analysis on entire directories of pickle data

def main():
    # picklefile = 'Z:/Data/GK2A/model_outputs/202208/sosmag_modout_OMNI2022
    # -08-04.pickle'
    # with open(picklefile, 'rb') as file:
    #     data = pickle.load(file)
    # print(data.keys())

    parser = argparse.ArgumentParser()

    # all spacecraft data is optional, and at least one is required.
    group = parser.add_argument_group('Spacecraft Data')
    group.add_argument("--g17-file", help="File path for GOES-17 mag data")
    group.add_argument("--g18-file", help="File path for GOES-18 mag data")
    group.add_argument("--gk2a-file", help="File path for GK2A SOSMAG data")

    # Optional arguments
    parser.add_argument("--save-path", default=None,
                        help="Save path for the figure \n(optional)")

    parser.add_argument("--g17-deg", default=None,
                        help="GOES17 s/c longitude in degrees (WEST), "
                             "ex. 105\n(optional)")
    parser.add_argument("--g18-deg", default=None,
                        help="GOES18 s/c longitude in degrees (WEST), "
                             "ex. 137.0\n(optional)")
    parser.add_argument("--gk2a-deg", default=None,
                        help="GK2A s/c longitude in degrees (EAST), "
                             "ex. 128.2\n(optional)")

    args = parser.parse_args()

    # Needed to initialize:
    # goes17_bgse_stacked, goes18_bgse_stacked, gk2a_bgse_stacked, gk2a_VDH, \
    #     goes17_VDH, goes17_VDH, save_path = [None] * 7

    # Check if at least one spacecraft data is provided
    if not any([args.g17_file, args.g18_file, args.gk2a_file]):
        parser.error('At least one spacecraft data file must be provided.')

    process_spacecraft_data(g17_file=args.g17_file, g18_file=args.g18_file,
                            gk2a_file=args.gk2a_file,
                            g17_deg=args.g17_deg, g18_deg=args.g18_deg,
                            gk2a_deg=args.gk2a_deg,
                            save_path=args.save_path)

if __name__ == "__main__":
    main()


#sayının bölenleri toplamı sayıya eşit mi diye kontrol eder

top = 0
liste = []

sayi = int(input("bir sayi giriniz: "))
i=1
while i < sayi:
    if (sayi % i == 0):
        top += i
        liste.append(i)
    i+=1
print(liste)
if (top==sayi):
    print("mukemmel sayi")
else:
    print("mükemmel sayi değil")


import socket
import sys
import logging

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

# Create a TCP/IP socket
sock = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
# Connect the socket to the port where the server is listening
server_address = ('localhost', 5000)
logging.info ('connecting to {} port {}'.format (*server_address))
sock.connect (server_address)
try:
    # Send data
    message = b'00010sinitasipe'
    logging.info ('sending {!r}'.format (message))
    sock.sendall (message)
    while True:
        # Look for the response
        logging.info ("Waiting for register transaction")
        amount_received = 0
        amount_expected = int(sock.recv(5))
        while amount_received < amount_expected:
            data = sock.recv (amount_expected - amount_received)
            amount_received += len (data)
            logging.info('received {!r}'.format(data))
            logging.info ("Visualización de Asistencia por Personal ...")
            data = data.decode().split()
            try:
                opcion = data[1]
                PersonalRut = data[2]
                Fecha = data[3]
                #FechaDesde = data[4]
                #FechaHasta = data[5]

                largo = len(PersonalRut+Fecha+opcion) + 8
                message = '000{}datas {} {} {}'.format(largo,opcion,PersonalRut,Fecha).encode()
                logging.info ('sending to bbdd {!r}'.format (message))
                sock.sendall(message)
                algo = sock.recv(4096).decode()
                if algo:
                    if 'rut' in algo:
                        message = '00011asiperut'.encode()
                    elif 'fecha' in algo:
                        message = '00011asipefecha'.encode()
                    else:
                        message = '00010asipeexito {}'.format(algo).encode()
                    logging.info ('sending {!r}'.format (message))
                    sock.send(message)
            except:
                pass
            logging.info('-------------------------------')

finally:
    logging.info ('closing socket')
    sock.close ()


class Mob:

    def __init__(self, x, y, r, hp, attack, speed, various, name, ):

        self.x = x
        self.y = y
        self.r = r
        self.hp = hp
        self.attack = attack
        self.speed = speed
        self.various = various
        self.animation = 0
        self.name = name

    def receive_damege(self, damages):
        self.hp += damages

    def print_mob(self, screen, img):
        screen.blit(img, [self.x, self.y])


"""
Lession URL:
https://seir-222-sasquatch.netlify.app/second-language/week-18/day-3/labs/python-containers-lab/
"""
# Exercise 1
students = ["Joshua","Timmy","Json","Rusty","Javario", "Pi Hon"]
print(students[1])
print(students[-1])

#Exercise 2
foods = ("pizza", "pie", "Mac & Cheese", "Salsa", "Kimchi", "Louisiana fired chicken")
for food in foods:
    print(f'{food} is a good food')

# Exercise 3
for food in foods:
    if food == "Kimchi" or food == "Louisiana fired chicken":
        print(food)

#Exercise 4
home_town = {
    "city": "Long Beach",
    "state": "California",
    "population": "Lots"
}
print(f'I was born in {home_town["city"]}, {home_town["state"]} with a population of {home_town["population"]} ')

# Exercise 5
for prop in home_town:
    print(prop, home_town[prop])

# Exercise 6

cohort = []
for student in students:
    cohort.append(dict(
        name = zip(students),
        fav_food = zip(foods)
        ))
print(cohort)
food_test = zip(foods)

awesome_students = []

for student in students:
    awesome_students.append(f'{student} is awesome!')

for string in awesome_students:
    print (string)


# -*- coding:utf-8 -*-

import os
import httplib
import re
import urllib

'''
http://www.drnew.com/car-brands.html
http://www.autoheroes.com/resources/manufacturers.shtml
http://buddysurvey4u.com/search/transport-newcars/index.htm
http://weblivenews.com/sports-car-brands-list/25096/
'''

home_url = 'www.drnew.com'

def collect():

    conn = httplib.HTTPConnection(home_url)
    conn.request('GET', '/car-brands.html')
    result = conn.getresponse()
    data = result.read()
    conn.close()

    re_brands_td = re.compile('.+	', re.S)

    re_brands = re.compile('(.+?)', re.S)

    brands_td = re.search(re_brands_td, data)

    if brands_td is not None:
        f_brands = open('brands.txt', 'w')
        brands = re.findall(re_brands, brands_td.group(0))
        for brand_url, brand_name in brands:
            print(brand_url, brand_name)
            f_brands.write('%s\t%s\n' %(brand_url, brand_name))

        f_brands.close()

def collect_single_brand():

    conn = httplib.HTTPConnection(home_url)

    f = open('brands.txt')

    re_models = re.compile('.+?	', re.S)
    re_model = re.compile('href="(.+?-car-model.html)">(.+?)</a', re.S)
    for line in f:
        brand_url, brand_name = line.split('\t')
        brand_name = brand_name[:-1]

        if os.path.exists(brand_name + '.txt'):
            print('jump: %s, %s' %(brand_url, brand_name))
            continue

        print('begin: %s, %s' %(brand_url, brand_name))
        conn.request('GET', urllib.quote(brand_url))
        result = conn.getresponse()
        data = result.read()

        m_models = re.search(re_models, data)

        f_model = open(brand_name + '.txt', 'w')
        if m_models is not None:
            models = re.findall(re_model, m_models.group(0))
            for (detail_url, detail_name) in models:
                f_model.write('%s\t%s\n' %(detail_url, detail_name))

        f_model.close()

        print('end: %s, %s' %(brand_url, brand_name))
    f.close()
    conn.close()

def collect_detail():
    conn = httplib.HTTPConnection(home_url)

    f = open('brands.txt')

    for line in f:
        brand_url, brand_name = line.split('\t')
        brand_name = brand_name[:-1]

        f_brand = open(brand_name + '.txt')
        for brand in f_brand:
            detail_url, detail_name = brand.split('\t')
            detail_name = detail_name[:-1].replace('/', '$') # 记得还原

            if not os.path.exists(brand_name):
                os.mkdir(brand_name)

            if os.path.exists(brand_name + '/' + detail_name + '.txt'):
                print('jump: %s, %s' %(detail_url, detail_name))
                continue

            print('begin: %s, %s' %(detail_url, detail_name))
            conn.request('GET', urllib.quote(detail_url))
            result = conn.getresponse()
            data = result.read()

            f_detail = open(brand_name + '/' + detail_name + '.txt', 'w')

            re_detail_items = re.compile('(.+?)
', re.S)
            re_detail_item = re.compile('(.+?).+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
.+?
(.+?)	(.+?)
', re.S)

            detail_items = re.findall(re_detail_items, data)
            if detail_items is not None:
                for detail_item in detail_items:
                    detail_item_array = re.findall(re_detail_item, detail_item)

                    if detail_item_array is not None:

                        f_detail.write('\ndetail:\n')

                        for detail_item_values in detail_item_array:
                            print('save detail: %s' %(detail_item_values[0]))
                            f_detail.write(detail_item_values[0] + '\n')
                            for index in range(0, len(detail_item_values[1:]), 2):
                                f_detail.write(detail_item_values[index + 1] + '\t' + detail_item_values[index + 2] + '\n')

            f_detail.close()

        f_brand.close()

    conn.close()

    print('end')

if __name__ == '__main__':
    collect()
    collect_single_brand()
    collect_detail()


from openpyxl import load_workbook

class doExcel():

    def __init__(self,file_path,sheet_name):
        self.file_path=file_path
        self.sheet_name=sheet_name
        self.wb=load_workbook(self.file_path)
        self.sh=self.wb[self.sheet_name]
        #获取当前sheet最大的行数
        self.row_max=self.sh.max_row
        # 获取当前sheet最大的列数
        self.column_max=self.sh.max_column

    def get_sheet(self):
        sheet_list=[]
        for i in range(2,self.row_max+1):
            sheet_row_list={}
            sheet_row_list["interface_name"]=self.sh.cell(i,1).value
            sheet_row_list["interface_url"]=self.sh.cell(i,2).value
            sheet_row_list["interface_method"]=self.sh.cell(i,3).value
            sheet_row_list["interface_body"]=self.sh.cell(i,4).value
            sheet_row_list["interface_expectedOutcome"]=self.sh.cell(i,5).value
            sheet_row_list["interface_actualResults"]=self.sh.cell(i,6).value
            sheet_list.append(sheet_row_list)
        return sheet_list

    def get_cell(self,row,column):
        return self.sh.cell(row,column).value

    def set_cell(self,row,column,cellvalue):
        self.sh.cell(row=row, column=column).value = cellvalue
        self.wb.save(self.file_path)

file_path="C:\\Users\\admin\\Desktop\\test_api.xlsx"
sheet_name="Sheet1"
do=doExcel(file_path,sheet_name)
do.set_cell(10,10,"")
print(do.get_cell(10,10))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Test suite for generic classes.

"""
from __future__ import print_function, division

import unittest as ut
import numpy as np

from diffusions import (GBM, GBMparam, Vasicek, VasicekParam,
                        CIR, CIRparam, Heston, HestonParam,
                        CentTend, CentTendParam)

class GenericModelTestCase(ut.TestCase):
    """Test generic model."""

    def test_update_theta(self):
        """Test update of true parameter."""

        mean, sigma = 1.5, .2
        param = GBMparam(mean, sigma)
        gbm = GBM(param)
        param_new = GBMparam(2*mean, 2*sigma)
        gbm.update_theta(param_new)

        self.assertEqual(gbm.param, param_new)

if __name__ == '__main__':
    ut.main()


import os
import json
from flask import Flask, request, send_file, jsonify
from picamera2 import Picamera2, Preview
import time
from PIL import Image

picam2 = Picamera2()
camera_config = picam2.create_still_configuration(main={"size": (256, 256)})
picam2.set_controls({"ExposureTime": 5000})
picam2.configure(camera_config)
# picam2.start_preview(Preview.DRM)
picam2.start()
time.sleep(2)
# picam2.capture_file("test.jpg")

app = Flask(__name__)
counter = 0
# Simulated raw_image (you should replace this with your own captured image)
# For demonstration purposes, we'll assume it's already a PIL Image.
# raw_image = Image.new("RGB", (640, 480), "white")

@app.route('/')
def index():
    return 'Camera Server is running!'

@app.route('/capture', methods=['GET'])
def capture():
    try:
        raw_image = picam2.capture_image("main")
        # Flip the image horizontally
        flipped_image = raw_image.transpose(Image.FLIP_TOP_BOTTOM)

        # Create a unique filename based on the current timestamp
        filename = f"photo_{int(time.time())}.jpeg"

        # Save the flipped image as a JPEG file
        flipped_image.save(filename, "JPEG")

        # Send the image to the client
        return send_file(filename, mimetype='image/jpeg')

    except Exception as e:
        return str(e)

@app.route('/store_inference', methods=['POST'])
def store_inference():
    global counter

    data = request.get_json()
    print(data)

    infer_text = data.get('infer_text', '')
    infer_img = data.get('infer_img', '')

    # Increment the counter
    counter += 1

    # Create the JSON payload
    inference_json = {
        'infer_text': infer_text,
        'infer_img': infer_img
    }

    # Save the JSON to a file
    filename = f'inference_{counter}.json'
    file_path = os.path.join('./inferences', filename)  # Replace with your desired directory
    with open(file_path, 'w') as f:
        json.dump(inference_json, f)

    # Respond with a success message
    return jsonify({'message': f'Inference stored successfully. Saved as {filename}'})

@app.route('/get_inference', methods=['GET'])
def get_inference():
    index = request.args.get('index', type=int)

    # Generate the filename based on the index
    filename = f'inference_{index}.json'
    file_path = os.path.join("./inferences", filename)

    if os.path.exists(file_path):
        # Load the JSON from the specified file
        with open(file_path, 'r') as f:
            inference_data = json.load(f)
        return jsonify(inference_data)
    else:
        return jsonify({'error': f'Inference file for index {index} not found.'}), 404

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8000)


"""
East Text detection
"""
import cv2
import numpy as np
from imutils.object_detection import non_max_suppression
from cfir_game_lens.utils import Box, GameCoverImage, ImageSize
from dataclasses import astuple

class EAST:  # pylint: disable=too-few-public-methods
    """East Class
    """

    LAYER_NAMES = ["feature_fusion/Conv_7/Sigmoid", "feature_fusion/concat_3"]
    EAST_MODEL_FILE = "frozen_east_text_detection.pb"
    SCALE_FACTOR = 1.0
    MEAN_SUBSTRACTION = (123.68, 116.78, 103.94)
    MIN_CONFIDENCE = 0.5

    def __init__(self):
        self._net = cv2.dnn.readNet(self.EAST_MODEL_FILE)

    @classmethod
    def _resize_image(cls, img: GameCoverImage):
        new_size = ImageSize()
        new_size.height = round(img.size.height / 32) * 32
        new_size.width = round(img.size.width / 32) * 32
        new_img = GameCoverImage(img=cv2.resize(img.img, new_size.get_rev_tuple()))
        return new_img

    def _get_blob(self, img: GameCoverImage):
        blob_args = [img.img, self.SCALE_FACTOR, img.size.get_rev_tuple(), self.MEAN_SUBSTRACTION]
        blob = cv2.dnn.blobFromImage(*blob_args, swapRB=True, crop=False)
        return blob

    @classmethod
    def _build_box(cls, offset_x, offset_y, angle, loc_data):
        (offset_x, offset_y) = (offset_x * 4.0, offset_y * 4.0)
        cos = np.cos(angle)
        sin = np.sin(angle)
        height = loc_data[0] + loc_data[2]
        width = loc_data[1] + loc_data[3]
        end_x = int(offset_x + (cos * loc_data[1]) + (sin * loc_data[2]))
        end_y = int(offset_y - (sin * loc_data[1]) + (cos * loc_data[2]))
        start_x = int(end_x - width)
        start_y = int(end_y - height)
        res = Box(start_x, start_y, end_x, end_y)
        return res

    def _build_boxes(self, scores, geometry):
        boxes = []
        confidences = []
        (num_rows, num_cols) = scores.shape[2:4]
        for y in range(0, num_rows):  # pylint: disable=invalid-name
            # extract the scores (probabilities), followed by the
            # geometrical data used to derive potential bounding box
            # coordinates that surround text
            scores_data = scores[0, 0, y]
            loc_data_arr = [geometry[0, 0, y], geometry[0, 1, y], geometry[0, 2, y], geometry[0, 3, y]]
            angles_data = geometry[0, 4, y]

            # loop over the number of columns
            for x in range(0, num_cols):  # pylint: disable=invalid-name
                if scores_data[x] < self.MIN_CONFIDENCE:
                    continue
                loc_data_arr_x = [loc_data[x] for loc_data in loc_data_arr]
                box = self._build_box(x, y, angles_data[x], loc_data_arr_x)
                boxes.append(box)
                confidences.append(scores_data[x])
        return boxes, confidences

    def find_text(self, image: GameCoverImage):
        """Find boxes of text in image

        Arguments:
            image {GameCoverImage} -- [description]

        Returns:
            List[Box] -- Boxes with text
        """
        # run east model
        resized_img = self._resize_image(image)
        self._net.setInput(self._get_blob(resized_img))
        (scores, geometry) = self._net.forward(self.LAYER_NAMES)

        # create boxes from east output and filter according to confidences\scores
        boxes, confidences = self._build_boxes(scores, geometry)
        boxes_as_tuple = [astuple(x) for x in boxes]
        boxes = non_max_suppression(np.array(boxes_as_tuple), probs=confidences)
        boxes_res = [Box(*x) for x in boxes]

        # fix  images ratio
        ratio_width = image.size.width / float(resized_img.size.width)
        ratio_height = image.size.height / float(resized_img.size.height)
        for box in boxes_res:
            box.fix_sizes(ratio_width, ratio_height)
        return boxes_res


import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn
from torch.utils.data import Dataset, DataLoader, TensorDataset
import torch.nn.functional as F
from torch.autograd import Variable
from sklearn.model_selection import train_test_split

import numpy as np
import time
import pandas as pd
import matplotlib.pyplot as plt
import os
from PIL import Image
import gc

from utils import *
from data_handler import *
from model import *

'''
Train the model.
'''

def train(model, criterion, optimizer, trainloader, valloader, startepoch, nepochs, save):
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    print('Training...')
    print(time.ctime())
    torch.autograd.set_detect_anomaly(True)
    for epoch in range(startepoch, startepoch+nepochs):
        print('Starting Epoch %d:' %(epoch), time.ctime())
        start_epoch = time.time()
        model.train()
        train_loss, train_dist = [], []
        for batch_num, (data_in) in enumerate(trainloader):
            data_in = data_in.to(device)
            start_batch = time.time()

            optimizer.zero_grad()
            means, stddevs, corrs = model(data_in, tprob)

            mask = (data_in[:,:,1:,:] < 0)
            loss = criterion(data_in[:,:,1:,:].to(device), means[:,:,:-1,:],\
                             stddevs[:,:,:-1,:], corrs[:,:,:-1,:], mask[:,:,:,0].to(device))
            loss = loss.sum()
            loss = (loss / (mask < 1).sum()).clamp(max=1e18)
            loss.backward()

            nn.utils.clip_grad_norm_(model.parameters(), 10.0)
            optimizer.step()

            dist = calc_dist(data_in[:,:,1:,:].detach().cpu(), means[:,:,:-1,:].detach().cpu(),\
                             mask.detach().cpu())

            gc.collect()
            torch.cuda.empty_cache()
            train_loss.append(loss.detach().cpu().item())
            train_dist.append(dist.item())
            del loss, mask, means, stddevs, corrs, data_in

            if epoch == 0 and batch_num == 0:
                print('Single Batch Time: %d min %d sec' %((time.time()-start_batch)//60, (time.time()-start_batch)%60))

        val_loss, val_dist = 0, 0
        if epoch % 4 == 0:
            val_loss, val_dist = validate(model, criterion, valloader)

        if save:
            torch.save(model.state_dict(), 'model_'+str(epoch)+'.pt')
            torch.save(optimizer.state_dict(), 'optimizer_'+str(epoch)+'.pt')

        stop_epoch = time.time()
        min_epoch = (stop_epoch - start_epoch) // 60
        sec_epoch = (stop_epoch - start_epoch) % 60

        print("Epoch: %d, Run Time: %d min, %d sec" %(epoch, min_epoch, sec_epoch))
        print('Train Loss: {:.3f}, Train Avg Dist: {:.2f}'.format(np.mean(train_loss), np.mean(train_dist)))
        if (epoch % 4) == 0:
            print('Val Loss: {:.3f}, Val Avg Dist: {:.2f}'.format(val_loss, val_dist))
        print("==============================================")

def validate(model, criterion, valloader):
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    model.eval()
    val_loss, val_dist = [], []
    for batch_num, (data_in) in enumerate(valloader):
        data_in = data_in.to(device)

        means, stddevs, corrs = [], [], []

        means, stddevs, corrs = model(data_in, tprob=1)

        mask = (data_in[:,:,1:,:] < 0)
        loss = criterion(data_in[:,:,1:,:].to(device), means[:,:,:-1,:],\
                         stddevs[:,:,:-1,:], corrs[:,:,:-1,:], mask[:,:,:,0].to(device))
        loss = loss.sum()
        loss = loss / (mask < 1).sum()

        dist = calc_dist(data_in[:,:,1:,:].detach().cpu(), means[:,:,:-1,:].detach().cpu(),\
                         mask.detach().cpu())

        gc.collect()
        torch.cuda.empty_cache()
        val_loss.append(loss.detach().cpu().item())
        val_dist.append(dist.item())
        del loss, mask, means, stddevs, corrs, data_in

    return np.mean(val_loss), np.mean(val_dist), np.mean(val_evaldist)#, np.mean(val_finaldist)

def nllLoss(target, mean, stddev, corr, mask):
    x1, x2 = target[:,:,:,0], target[:,:,:,1]
    m1, m2 = mean[:,:,:,0], mean[:,:,:,1]
    std1, std2 = stddev[:,:,:,0].clamp(min=1e-6), stddev[:,:,:,1].clamp(min=1e-6)
    corr = corr[:,:,:].squeeze(-1)

    Z = pow((x1-m1)/std1,2) + pow((x2-m2)/std2,2) - 2*corr*(((x1-m1)*(x2-m2))/(std1*std2))
    N = (1 / (2*np.pi*std1*std2*torch.sqrt(1-pow(corr,2).clamp(max=1-1e-6))))
    a = torch.log(N.clamp(min=1e-6))
    b = (-Z/(2*(1-pow(corr,2))))
    L = a + b
    L = L.masked_fill_(mask,0)
    return -L

def run(startepoch, nepochs, modelfilename, optfilename, save):
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    print(device)
    torch.autograd.set_detect_anomaly(True)
    gc.collect()
    torch.cuda.empty_cache()

    # get the data
    trainloader, valloader = get_dataloaders()

    # define the model
    model = SocialLSTM()
    if modelfilename is not None:
        model.load_state_dict(modelfilename, map_location=device)
    model = model.to(device)

    # define the criterion and optimizer
    criterion = nllLoss
    learning_rate = 3e-2
    weight_decay = 5e-5
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
    if optfilename is not None:
        optimizer.load_state_dict(torch.load(optfilename))

    train(model, criterion, optimizer, trainloader, valloader, startepoch, nepochs, save)

if __name__ == "__main__":
    params = get_params()

    startepoch, nepochs = params.startepoch, params.nepochs
    modelfilename, optfilename = params.load_model, params.load_optimizer

    modelfilename = 'model_0.pt'
    optfilename = 'optimizer_0.pt'

    save = False

    startepoch = 0
    nepochs = 100

    run(startepoch, nepochs, modelfilename, optfilename, save)


"""
Migration scripts
"""

import click
from packaging.version import Version

from brewblox_ctl import actions, click_helpers, const, migration, sh, utils

@click.group(cls=click_helpers.OrderedGroup)
def cli():
    """Global command group"""

def check_version(prev_version: Version):
    """Verify that the previous version is sane and sensible"""
    if prev_version == Version('0.0.0'):
        utils.error('This configuration was never set up. Please run brewblox-ctl setup first')
        raise SystemExit(1)

    if prev_version > Version(const.CFG_VERSION):
        utils.error('Your system is running a version newer than the selected release. ' +
                    'This may be due to switching release tracks.' +
                    'You can use the --from-version flag if you know what you are doing.')
        raise SystemExit(1)

def check_dirs():
    utils.info('Checking data directories...')
    dirs = [
        './traefik',
        './auth',
        './redis',
        './victoria',
        './mosquitto',
        './spark/backup',
    ]
    sh('mkdir -p ' + ' '.join(dirs))

def apply_config_files():
    """Apply system-defined configuration from config dir"""
    utils.info('Updating configuration files...')
    sh(f'cp -f {const.DIR_DEPLOYED_CONFIG}/traefik-cert.yaml ./traefik/')
    sh(f'cp -f {const.DIR_DEPLOYED_CONFIG}/docker-compose.shared.yml ./')
    shared_cfg = utils.read_shared_compose()
    usr_cfg = utils.read_compose()

    usr_cfg['version'] = shared_cfg['version']
    utils.write_compose(usr_cfg)

def check_env_vars():
    utils.info('Checking .env variables...')
    utils.defaultenv()

def bind_localtime():
    shared_cfg = utils.read_shared_compose()
    usr_cfg = utils.read_compose()

    changed = False
    localtime_volume_str = '/etc/localtime:/etc/localtime:ro'
    localtime_volume = {
        'type': 'bind',
        'source': '/etc/localtime',
        'target': '/etc/localtime',
        'read_only': True,
    }

    for (name, service) in usr_cfg['services'].items():
        name: str
        service: dict

        if name in shared_cfg['services']:
            continue

        volumes = service.get('volumes', [])
        if localtime_volume in volumes:
            continue
        if localtime_volume_str in volumes:
            continue

        changed = True
        utils.info(f'Mounting localtime in `{name}` service...')
        volumes.append(localtime_volume.copy())
        service['volumes'] = volumes

    if changed:
        utils.write_compose(usr_cfg)

def bind_spark_backup():
    usr_cfg = utils.read_compose()

    changed = False
    backup_volume = {
        'type': 'bind',
        'source': './spark/backup',
        'target': '/app/backup',
    }

    for (name, service) in usr_cfg['services'].items():
        name: str
        service: dict

        if not service.get('image', '').startswith('ghcr.io/brewblox/brewblox-devcon-spark'):
            continue

        volumes = service.get('volumes', [])
        present = False
        for volume in volumes:
            if (isinstance(volume, str) and volume.endswith(':/app/backup')) \
                    or (isinstance(volume, dict) and volume.get('target') == '/app/backup'):
                present = True
                break

        if present:
            continue

        changed = True
        utils.info(f'Mounting backup volume in `{name}` service...')
        volumes.append(backup_volume.copy())
        service['volumes'] = volumes

    if changed:
        utils.write_compose(usr_cfg)

def downed_migrate(prev_version):
    """Migration commands to be executed without any running services"""
    check_dirs()
    apply_config_files()
    actions.add_particle_udev_rules()
    actions.edit_avahi_config()

    if prev_version < Version('0.3.0'):
        migration.migrate_compose_split()

    if prev_version < Version('0.6.0'):
        migration.migrate_compose_datastore()

    if prev_version < Version('0.6.1'):
        migration.migrate_ipv6_fix()

    if prev_version < Version('0.8.0'):
        migration.migrate_ghcr_images()

    if prev_version < Version('0.9.0'):
        migration.migrate_tilt_images()

    # Not related to a specific release
    check_env_vars()
    bind_localtime()
    bind_spark_backup()

def upped_migrate(prev_version):
    """Migration commands to be executed after the services have been started"""
    if prev_version < Version('0.6.0'):
        utils.warn('')
        utils.warn('Brewblox now uses a new configuration database.')
        utils.warn('To migrate your data, run:')
        utils.warn('')
        utils.warn('    brewblox-ctl database from-couchdb')
        utils.warn('')

    if prev_version < Version('0.7.0'):
        utils.warn('')
        utils.warn('Brewblox now uses a new history database.')
        utils.warn('To migrate your data, run:')
        utils.warn('')
        utils.warn('    brewblox-ctl database from-influxdb')
        utils.warn('')

@cli.command()
@click.option('--update-ctl/--no-update-ctl',
              default=True,
              help='Update brewblox-ctl.')
@click.option('--update-ctl-done',
              is_flag=True,
              hidden=True)
@click.option('--pull/--no-pull',
              default=True,
              help='Update docker service images.')
@click.option('--update-system-packages/--no-update-system-packages',
              default=True,
              envvar=const.ENV_KEY_UPDATE_SYSTEM_PACKAGES,
              help='Update system packages.')
@click.option('--migrate/--no-migrate',
              default=True,
              help='Migrate Brewblox configuration and service settings.')
@click.option('--prune/--no-prune',
              default=True,
              help='Remove unused docker images.')
@click.option('--from-version',
              default='0.0.0',
              envvar=const.ENV_KEY_CFG_VERSION,
              help='[ADVANCED] Override version number of active configuration.')
def update(update_ctl, update_ctl_done, pull, update_system_packages, migrate, prune, from_version):
    """Download and apply updates.

    This is the one-stop-shop for updating your Brewblox install.
    You can use any of the options to fine-tune the update by enabling or disabling subroutines.

    By default, all options are enabled.

    --update-ctl/--no-update-ctl: Whether to download and install new versions of
    of brewblox-ctl. If this flag is set, update will download the new version
    and then restart itself. This way, the migrate is done with the latest version of brewblox-ctl.

    If you're using dry run mode, you'll notice the hidden option --update-ctl-done.
    You can use it to watch the rest of the update: it\'s a flag to avoid endless loops.

    --pull/--no-pull. Whether to pull docker images.
    This is useful if any of your services is using a local image (not from Docker Hub).

    --update-system-packages/--no-update-system-packages determines whether generic system packages
    are updated during the brewblox-ctl update.

    --migrate/--no-migrate. Updates regularly require changes to configuration.
    Required changes are applied here.

    --prune/--no-prune. Updates to docker images can leave unused old versions
    on your system. These can be pruned to free up disk space.
    This includes all images and volumes on your system, and not just those created by Brewblox.

    \b
    Steps:
        - Check whether any system fixes must be applied.
        - Update brewblox-ctl.
        - Stop services.
        - Update Avahi config.
        - Update system packages.
        - Migrate configuration files.
        - Pull Docker images.
        - Prune unused Docker images and volumes.
        - Start services.
        - Migrate service configuration.
        - Write version number to .env file.
    """
    utils.check_config()
    utils.confirm_mode()
    utils.cache_sudo()
    sudo = utils.optsudo()

    prev_version = Version(from_version)
    shipped_version = Version(const.CFG_VERSION)
    check_version(prev_version)

    if not update_ctl_done:
        utils.info(f'Starting update for brewblox {utils.getenv(const.ENV_KEY_RELEASE)}...')

    if update_ctl and not update_ctl_done:
        utils.info('Updating brewblox-ctl...')
        utils.pip_install('pip')
        actions.install_ctl_package()
        # Restart update - we just replaced the source code
        sh(' '.join(['exec', const.CLI, *const.ARGS[1:], '--update-ctl-done']))
        return

    if update_ctl:
        actions.uninstall_old_ctl_package()
        actions.deploy_ctl_wrapper()

    actions.check_compose_plugin()

    utils.info('Stopping services...')
    sh(f'{sudo}docker compose down')

    if update_system_packages:
        actions.update_system_packages()

    if migrate:
        downed_migrate(prev_version)

    if pull:
        utils.info('Pulling docker images...')
        sh(f'{sudo}docker compose pull')

    if prune:
        utils.info('Pruning unused images...')
        sh(f'{sudo}docker image prune -f > /dev/null')
        utils.info('Pruning unused volumes...')
        sh(f'{sudo}docker volume prune -f > /dev/null')

    utils.info('Starting services...')
    sh(f'{sudo}docker compose up -d')

    if migrate:
        upped_migrate(prev_version)
        utils.info(f'Configuration version: {prev_version} -> {shipped_version}')
        utils.setenv(const.ENV_KEY_CFG_VERSION, const.CFG_VERSION)

@cli.command()
def update_ctl():
    """Download and update brewblox-ctl itself."""
    utils.confirm_mode()
    actions.install_ctl_package()
    actions.uninstall_old_ctl_package()
    actions.deploy_ctl_wrapper()


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 26 09:28:19 2022

@author: bjk_a
"""

#1.kutuphaneler
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# veri yukleme
veriler = pd.read_csv('maaslar.csv')

x = veriler.iloc[:,1:2]
y = veriler.iloc[:,2:]
X = x.values
Y = y.values

#linear regression
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(X,Y)

plt.scatter(X,Y,color='red')
plt.plot(x,lin_reg.predict(X), color = 'blue')
plt.show()

#polynomial regression
from sklearn.preprocessing import PolynomialFeatures
poly_reg = PolynomialFeatures(degree = 2)
x_poly = poly_reg.fit_transform(X)
print(x_poly)
lin_reg2 = LinearRegression()
lin_reg2.fit(x_poly,y)
plt.scatter(X,Y,color = 'red')
plt.plot(X,lin_reg2.predict(poly_reg.fit_transform(X)), color = 'blue')
plt.show()

from sklearn.preprocessing import PolynomialFeatures
poly_reg = PolynomialFeatures(degree = 4)
x_poly = poly_reg.fit_transform(X)
print(x_poly)
lin_reg2 = LinearRegression()
lin_reg2.fit(x_poly,y)
plt.scatter(X,Y,color = 'red')
plt.plot(X,lin_reg2.predict(poly_reg.fit_transform(X)), color = 'blue')
plt.show()

#tahminler

print(lin_reg.predict([[11]]))
print(lin_reg.predict([[6.6]]))

print(lin_reg2.predict(poly_reg.fit_transform([[6.6]])))
print(lin_reg2.predict(poly_reg.fit_transform([[11]])))

#verilerin olceklenmesi
from sklearn.preprocessing import StandardScaler

sc1=StandardScaler()

x_olcekli = sc1.fit_transform(X)

sc2=StandardScaler()
y_olcekli = np.ravel(sc2.fit_transform(Y.reshape(-1,1)))

from sklearn.svm import SVR

svr_reg = SVR(kernel='rbf')
svr_reg.fit(x_olcekli,y_olcekli)

plt.scatter(x_olcekli,y_olcekli,color='red')
plt.plot(x_olcekli,svr_reg.predict(x_olcekli),color='blue')

plt.show()
print(svr_reg.predict([[11]]))
print(svr_reg.predict([[6.6]]))

from sklearn.tree import DecisionTreeRegressor
r_dt = DecisionTreeRegressor(random_state=0)
#veriyi fit ediyoruz (X ve Y numpy arrayleri x ve y olan df leri değil)
#yani X,Y arasında ilişki kuruyor, eğitim seviyesine (x) göre aldığı maaş (y)
r_dt.fit(X,Y)

# 3 farklı değer de x z k aynı sonucu veriyor, çünkü aynı aralıktalar
Z = X + 0.5
K = X - 0.4

# X ve Y arası plotu çiz
plt.scatter(X,Y, color='red')

# X değerini çiz r.dt nin tahminini kullan
plt.plot(x,r_dt.predict(X), color='blue')

plt.plot(x,r_dt.predict(Z),color='green')
plt.plot(x,r_dt.predict(K),color='yellow')
print(r_dt.predict([[11]]))
print(r_dt.predict([[6.6]]))


'''
PALINDROM Slovo, ktore je rovnake odporedu aj odzadu
'''

def palindrome(slovo: str) -> bool:
    for i in range(len(slovo)):
        t = slovo[:i] + slovo[i+1:]
        if t == t[::-1]:
            return True
    return False

s = 'KOROK'
print(palindrome(slovo=s))


import os
import json

import pandas as pd
import numpy as np

reffile = "reference.json"
# errfile = "errors.json"

datfile = os.path.join(".", "data",
    "DOHMH_New_York_City_Restaurant_Inspection_Results.csv")

data = pd.read_csv(datfile, index_col=[0])
data = data.sort_index()
original_cols = data.columns

with open(reffile, "r") as file:
    yelp = json.load(file)

def tabulate(data, yelp):
    out = pd.DataFrame(index=data.index.unique())
    # generate a histogram of categories
    categories = {}
    for key in yelp:
        if ("yelp" in yelp[key]) and ("categories" in yelp[key]["yelp"]):
            for cat in yelp[key]["yelp"]["categories"]:
                if cat["title"] in categories:
                    categories[cat["title"]] += 1
                else:
                    categories[cat["title"]] = 1

    hist = pd.DataFrame({"ct": categories})
    hist = hist.sort_values("ct")[::-1]

    # find minimum number of categories required to meet threshold
    threshold = 0.95
    for ee, n_categories in enumerate(range(1, len(hist)+1)):
        totl = 0
        succ = 0
        for key in yelp:
            if (("yelp" in yelp[key])
                and ("categories" in yelp[key]["yelp"]) \
                and (len(yelp[key]["yelp"]["categories"]) > 0)):
                totl += 1
                if any(
                    hist.index[:n_categories].isin(
                        map(lambda d: d["title"], yelp[key]["yelp"]["categories"])
                    )
                ):
                    succ += 1
        if succ / totl > threshold:
            break

    hist = hist.reindex(hist.index[:ee])

    yelp_cat = dict.fromkeys(data.index.unique())
    for key in yelp_cat:
        cats = list(
            map(
                lambda d: d["title"],
                    yelp[str(key)]["yelp"]["categories"]
                    if ((str(key) in yelp)
                        and ("yelp" in yelp[str(key)])
                        and ("categories" in yelp[str(key)]["yelp"]))
                    else []
            ))
        mask = hist.index.isin(cats)
        if len(hist.index[mask]):
            yelp_cat[key] = hist.index[mask][0]
        else:
            yelp_cat[key] = "Other"
    out["yelp_cat"] = pd.Series(yelp_cat).reindex(out.index)

    keydata = {
            int(x) :
            (len(yelp[x]["yelp"]["price"]) * 1.
                if ("price" in yelp[x]["yelp"])
                else np.nan)
            for x in yelp
        }
    out["price"] = pd.Series(keydata).reindex(out.index)

    keydata = {
        int(x) :
        (yelp[x]["yelp"]["rating"] if ("rating" in yelp[x]["yelp"])
            else np.nan)
        for x in yelp
    }
    out["rating"] = pd.Series(keydata).reindex(out.index)

    keydata = {
        int(x) : (yelp[x]["yelp"]["review_count"] if ("review_count" in yelp[x]["yelp"])
            else np.nan)
        for x in yelp
    }
    out["review_count"] = pd.Series(keydata).reindex(out.index)

    keydata = {
        int(x) : ((
                yelp[x]["yelp"]["coordinates"]["latitude"],
                yelp[x]["yelp"]["coordinates"]["longitude"]
            ) if ("coordinates" in yelp[x]["yelp"])
            else (np.nan, np.nan))
        for x in yelp
    }
    longs, lats = map(
        lambda ary: pd.Series(ary, index=map(int, yelp.keys())).reindex(out.index),
        zip(*pd.Series(keydata))
    )
    out["longitude"], out["latitude"] = longs, lats

    keydata = {
        int(x) : ((
                min(float(j["start"]) for j in yelp[x]["yelp"]["hours"][0]["open"]),
                max(float(j["end"]) +
                    (0. if float(j["end"]) > float(j["start"]) else 2400.)
                    for j in yelp[x]["yelp"]["hours"][0]["open"])
            ) if ("hours" in yelp[x]["yelp"])
            else (np.nan, np.nan))
        for x in yelp
    }
    opens, closes = map(
        lambda ary: pd.Series(ary, index=map(int, yelp.keys())).reindex(out.index),
        zip(*pd.Series(keydata))
    )
    out["opens"], out["closes"] = opens, closes
    # buggy
    out["is_overnight"] = out["closes"] > 2400.

    return out

tabulated = tabulate(data, yelp)

tabulated.to_csv("tbl_yelp.csv")


import os
import bs4
from nltk.corpus.reader.api import CorpusReader
from nltk.corpus.reader.api import CategorizedCorpusReader
import nltk
import time
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
from sklearn.feature_extraction import text
from nltk import sent_tokenize
from nltk import wordpunct_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.decomposition import PCA
from numpy.linalg import svd
import numpy as np
from functools import reduce
import matplotlib.pyplot as plt
#1시간정도걸림

#nltk.download('wordnet')
#nltk.download('punkt')
#nltk.download('averaged_perceptron_tagger')
#nltk.download('omw-1.4')

CAT_PATTERN = r'([a-su-z]+)/.*'
DOC_PATTERN = r'(?!\.)[a-z_\s]+/[0-9]+\.html'
TAGS = []
title_TAGS = ['h1']
abstract_TAGS = ['blockquote']

class HTMLCorpusReader(CategorizedCorpusReader, CorpusReader):
    """
    A corpus reader for raw HTML documents to enable preprocessing.
    """

    def __init__(self, root, fileids=DOC_PATTERN, encoding='utf8',
                 tags=TAGS, **kwargs):
        """
        Initialize the corpus reader.  Categorization arguments
        (``cat_pattern``, ``cat_map``, and ``cat_file``) are passed to
        the ``CategorizedCorpusReader`` constructor.  The remaining
        arguments are passed to the ``CorpusReader`` constructor.
        """
        # Add the default category pattern if not passed into the class.
        if not any(key.startswith('cat_') for key in kwargs.keys()):
            kwargs['cat_pattern'] = CAT_PATTERN

        # Initialize the NLTK corpus reader objects
        CategorizedCorpusReader.__init__(self, kwargs)
        CorpusReader.__init__(self, root, fileids, encoding)

        # Save the tags that we specifically want to extract.
        self.tags = tags

    def resolve(self, fileids, categories):
        """
        Returns a list of fileids or categories depending on what is passed
        to each internal corpus reader function. Implemented similarly to
        the NLTK ``CategorizedPlaintextCorpusReader``.
        """
        if fileids is not None and categories is not None:
            raise ValueError("Specify fileids or categories, not both")

        if categories is not None:
            return self.fileids(categories)
        return fileids

    def docs(self, fileids=None, categories=None):
        """
        Returns the complete text of an HTML document, closing the document
        after we are done reading it and yielding it in a memory safe fashion.
        """
        # Resolve the fileids and the categories
        fileids = self.resolve(fileids, categories)

        # Create a generator, loading one document into memory at a time.
        for path in self.abspaths(fileids):
            with open(path, 'r', encoding='UTF-8') as f:
                yield f.read()

    def sizes(self, fileids=None, categories=None):
        """
        Returns a list of tuples, the fileid and size on disk of the file.
        This function is used to detect oddly large files in the corpus.
        """
        # Resolve the fileids and the categories
        fileids = self.resolve(fileids, categories)

        # Create a generator, getting every path and computing filesize
        for path in self.abspaths(fileids):
            yield os.path.getsize(path)

def describe(paragraphs, fileids, categories):
    started = time.time()

    counts = nltk.FreqDist()
    tokens = nltk.FreqDist()

    for para in paragraphs:
        counts['paras'] += 1

        for sent in nltk.sent_tokenize(para):
            counts['sents'] += 1

            for word in nltk.wordpunct_tokenize(sent):
                counts['words'] += 1
                tokens[word] += 1

    n_fileids = len(fileids)
    n_topics = len(categories)

    return {
        'files':  n_fileids,
        'topics': n_topics,
        'paragraphs':  counts['paras'],
        'sentences':  counts['sents'],
        'words':  counts['words'],
        'vocabulary size':  len(tokens),
        'lexical diversity': float(counts['words']) / float(len(tokens)),
        'paragraphs per document':  float(counts['paras']) / float(n_fileids),
        'sentences per paragraph':  float(counts['sents']) / float(counts['paras']),
        'secs':   time.time() - started,
    }

def paras(htmls, TAGS):
    for html in htmls:
        soup = bs4.BeautifulSoup(html, 'lxml')
        for element in soup.find_all(TAGS):
            yield element.text
        soup.decompose()

def sents(paragraph):
    for sentence in sent_tokenize(paragraph):
        yield sentence

def words(paragraph):
    for sentence in sents(paragraph):
        for word in wordpunct_tokenize(sentence):
            yield word

def sklearn_tfidf_vectorize(corpus):
    my_stop_words = text.ENGLISH_STOP_WORDS.union(['abstract', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
                                                   'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
                                                   'w', 'x', 'y', 'z', ''])
    tfidf = TfidfVectorizer(token_pattern='[a-zA-Z]*', stop_words=my_stop_words, min_df=5, sublinear_tf=True, max_df=0.85)
    res = tfidf.fit_transform(corpus)
    print(tfidf.get_feature_names()[:100])
    return res

def mindf005_sklearn_tfidf_vectorize(corpus):
    my_stop_words = text.ENGLISH_STOP_WORDS.union(['abstract', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
                                                   'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
                                                   'w', 'x', 'y', 'z', ''])
    tfidf = TfidfVectorizer(token_pattern='[a-zA-Z]*', stop_words=my_stop_words, min_df=0.05, sublinear_tf=True, max_df=0.85)
    res = tfidf.fit_transform(corpus)
    print(tfidf.get_feature_names()[:100])
    return res

def mindf001_sklearn_tfidf_vectorize(corpus):
    my_stop_words = text.ENGLISH_STOP_WORDS.union(['abstract', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
                                                   'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
                                                   'w', 'x', 'y', 'z', ''])
    tfidf = TfidfVectorizer(token_pattern='[a-zA-Z]*', stop_words=my_stop_words, min_df=0.01, sublinear_tf=True, max_df=0.85)
    res = tfidf.fit_transform(corpus)
    print(tfidf.get_feature_names()[:100])
    return res

def mindf0001_sklearn_tfidf_vectorize(corpus):
    my_stop_words = text.ENGLISH_STOP_WORDS.union(['abstract', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
                                                   'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
                                                   'w', 'x', 'y', 'z', ''])
    tfidf = TfidfVectorizer(token_pattern='[a-zA-Z]*', stop_words=my_stop_words, min_df=0.001, sublinear_tf=True, max_df=0.85)
    res = tfidf.fit_transform(corpus)
    print(tfidf.get_feature_names()[:100])
    return res

title_corpus = HTMLCorpusReader('', CAT_PATTERN, DOC_PATTERN, tags=title_TAGS)
title_fileids = title_corpus.fileids()
title_documents = title_corpus.docs(categories=title_corpus.categories())
title_htmls = list(title_documents)

abstract_corpus = HTMLCorpusReader('', CAT_PATTERN, DOC_PATTERN, abstract_TAGS)
abstract_fileids = abstract_corpus.fileids()
abstract_documents = abstract_corpus.docs(categories=abstract_corpus.categories())
abstract_htmls = list(abstract_documents)

title_categories = title_corpus.categories()
abstract_categories = abstract_corpus.categories()

title_paragraphs = list(paras(title_htmls, title_TAGS))
temp_title_paragraphs = []
for para in title_paragraphs:
    if "Title:" in para: # and len(para)>30
        temp_title_paragraphs.append(para.strip('Title:\n'))
title_paragraphs = temp_title_paragraphs
print("title_paragraphs len: ", len(title_paragraphs))
print("descreibe title_paragraphs", describe(title_paragraphs, title_fileids, title_categories))

abstract_paragraphs = list(paras(abstract_htmls, abstract_TAGS))
print("abstract_paragraphs len: ", len(abstract_paragraphs))
print("descreibe abstract_paragraphs", describe(abstract_paragraphs, abstract_fileids, abstract_categories))

papers_list = []
for key, value in zip(title_paragraphs, abstract_paragraphs):
    temp_dict = dict()
    temp_dict['title'] = key
    temp_dict['abstract'] = value
    papers_list.append(temp_dict)

print("papers_list[:5] : ", papers_list[:5])

print("title paragraphs[:5] : ", title_paragraphs[:5])

sparse_title_tf_idf = sklearn_tfidf_vectorize(title_paragraphs)
print(sparse_title_tf_idf)
print("type of sparse_title_tf_idf : ", type(sparse_title_tf_idf))

title_tf_idf = sparse_title_tf_idf.todense()
print(title_tf_idf)
print("type of title_tf_idf : ", type(title_tf_idf))

title_tf_idf_df = pd.DataFrame(title_tf_idf)
print(title_tf_idf_df.head())
print("title_tf_idf_df.shape : ", title_tf_idf_df.shape)

df = pd.DataFrame(papers_list, columns={'title', 'abstract'})
print(df.head())
print("df.shape : ", df.shape)
print("abstract_paragraphs: ", abstract_paragraphs[:5])

abstract_words = []
for p in abstract_paragraphs:
    abstract_words.append(list(words(p)))
print(abstract_words[0])
lemma_abstract_words = []
n=WordNetLemmatizer()
for words in abstract_words:
    lemma_abstract_words.append([n.lemmatize(w) for w in words])
print(lemma_abstract_words[0])
print(len(lemma_abstract_words))

lemma_abstract_paragraphs = []
for l in lemma_abstract_words:
    lemma_abstract_paragraphs.append(" ".join(l))
print(lemma_abstract_paragraphs[0])
print(len(lemma_abstract_paragraphs))

lemma_sparse_tf_idf = sklearn_tfidf_vectorize(lemma_abstract_paragraphs)
print(lemma_sparse_tf_idf)
print("type of lemma_sparse_tf_idf : ", type(lemma_sparse_tf_idf))

lemma_tf_idf = lemma_sparse_tf_idf.todense()
print(lemma_tf_idf)
print("type of lemma_tf_idf : ", type(lemma_tf_idf))

lemma_tf_idf_df = pd.DataFrame(lemma_tf_idf)
print(lemma_tf_idf_df.head())
print("lemma_tf_idf_df.shape : ", lemma_tf_idf_df.shape)

sparse_tf_idf = sklearn_tfidf_vectorize(abstract_paragraphs)
print(sparse_tf_idf)
print("type of sparse_tf_idf : ", type(sparse_tf_idf))

tf_idf = sparse_tf_idf.todense()
print(tf_idf)
print("type of tf_idf : ", type(tf_idf))

tf_idf_df = pd.DataFrame(tf_idf)
print(tf_idf_df.head())
print("tf_idf_df.shape : ", tf_idf_df.shape)

mindf005_sparse_tf_idf = mindf005_sklearn_tfidf_vectorize(abstract_paragraphs)
print(mindf005_sparse_tf_idf)
print("type of mindf005_sparse_tf_idf : ", type(mindf005_sparse_tf_idf))

mindf005_tf_idf = mindf005_sparse_tf_idf.todense()
print(mindf005_tf_idf)
print("type of mindf005_tf_idf : ", type(mindf005_tf_idf))

mindf005_tf_idf_df = pd.DataFrame(mindf005_tf_idf)
print(mindf005_tf_idf_df.head())
print("mindf005_tf_idf_df.shape : ", mindf005_tf_idf_df.shape)

mindf001_sparse_tf_idf = mindf001_sklearn_tfidf_vectorize(abstract_paragraphs)
print(mindf001_sparse_tf_idf)
print("type of mindf001_sparse_tf_idf : ", type(mindf001_sparse_tf_idf))

mindf001_tf_idf = mindf001_sparse_tf_idf.todense()
print(mindf001_tf_idf)
print("type of mindf001_tf_idf : ", type(mindf001_tf_idf))

mindf001_tf_idf_df = pd.DataFrame(mindf001_tf_idf)
print(mindf001_tf_idf_df.head())
print("mindf001_tf_idf_df.shape : ", mindf001_tf_idf_df.shape)

mindf0001_sparse_tf_idf = mindf0001_sklearn_tfidf_vectorize(abstract_paragraphs)
print(mindf0001_sparse_tf_idf)
print("type of mindf0001_sparse_tf_idf : ", type(mindf0001_sparse_tf_idf))

mindf0001_tf_idf = mindf0001_sparse_tf_idf.todense()
print(mindf0001_tf_idf)
print("type of mindf0001_tf_idf : ", type(mindf0001_tf_idf))

mindf0001_tf_idf_df = pd.DataFrame(mindf0001_tf_idf)
print(mindf0001_tf_idf_df.head())
print("mindf0001_tf_idf_df.shape : ", mindf0001_tf_idf_df.shape)

mindf005_lemma_sparse_tf_idf = mindf005_sklearn_tfidf_vectorize(lemma_abstract_paragraphs)
print(mindf005_lemma_sparse_tf_idf)
print("type of mindf005_lemma_sparse_tf_idf : ", type(mindf005_lemma_sparse_tf_idf))

mindf005_lemma_tf_idf = mindf005_lemma_sparse_tf_idf.todense()
print(mindf005_lemma_tf_idf)
print("type of mindf005_lemma_tf_idf : ", type(mindf005_lemma_tf_idf))

mindf005_lemma_tf_idf_df = pd.DataFrame(mindf005_lemma_tf_idf)
print(mindf005_lemma_tf_idf_df.head())
print("mindf005_lemma_tf_idf_df.shape : ", mindf005_lemma_tf_idf_df.shape)

mindf001_lemma_sparse_tf_idf = mindf001_sklearn_tfidf_vectorize(lemma_abstract_paragraphs)
print(mindf001_lemma_sparse_tf_idf)
print("type of mindf001_lemma_sparse_tf_idf : ", type(mindf001_lemma_sparse_tf_idf))

mindf001_lemma_tf_idf = mindf001_lemma_sparse_tf_idf.todense()
print(mindf001_lemma_tf_idf)
print("type of mindf001_lemma_tf_idf : ", type(mindf001_lemma_tf_idf))

mindf001_lemma_tf_idf_df = pd.DataFrame(mindf001_lemma_tf_idf)
print(mindf001_lemma_tf_idf_df.head())
print("mindf001_lemma_tf_idf_df.shape : ", mindf001_lemma_tf_idf_df.shape)

mindf0001_lemma_sparse_tf_idf = mindf0001_sklearn_tfidf_vectorize(lemma_abstract_paragraphs)
print(mindf0001_lemma_sparse_tf_idf)
print("type of mindf0001_lemma_sparse_tf_idf : ", type(mindf0001_lemma_sparse_tf_idf))

mindf0001_lemma_tf_idf = mindf0001_lemma_sparse_tf_idf.todense()
print(mindf0001_lemma_tf_idf)
print("type of mindf0001_lemma_tf_idf : ", type(mindf0001_lemma_tf_idf))

mindf0001_lemma_tf_idf_df = pd.DataFrame(mindf0001_lemma_tf_idf)
print(mindf0001_lemma_tf_idf_df.head())
print("mindf0001_lemma_tf_idf_df.shape : ", mindf0001_lemma_tf_idf_df.shape)

title_tf_idf_df.to_csv('title_tf_idf_df.csv')
df.to_csv('df.csv')
tf_idf_df.to_csv('tf_idf_df.csv')
mindf005_tf_idf_df.to_csv('mindf005_tf_idf_df.csv')
mindf001_tf_idf_df.to_csv('mindf001_tf_idf_df.csv')
mindf0001_tf_idf_df.to_csv('mindf0001_tf_idf_df.csv')
lemma_tf_idf_df.to_csv('lemma_tf_idf_df.csv')
mindf005_lemma_tf_idf_df.to_csv('mindf005_lemma_tf_idf_df.csv')
mindf001_lemma_tf_idf_df.to_csv('mindf001_lemma_tf_idf_df.csv')
mindf0001_lemma_tf_idf_df.to_csv('mindf0001_lemma_tf_idf_df.csv')
print("csv saving done")


import numpy as np
from scipy.optimize import fmin_bfgs
from MILpy.functions.noisyORlossWeights import noisyORlossWeights
from MILpy.functions.noisyORlossAlphas import noisyORlossAlphas
from MILpy.functions.traindecstump import traindecstump

class MILBoost(object):

    def __init__(self):
        self._alpha = None
        self._H = None
        self._T = None

    def fit(self,train_bags,train_labels, errtol = 1e-15,T=100,**kwargs):
        """
        @param train_bags : a sequence of n bags; each bag is an m-by-k array-like
        object containing m instances with k features
        @param train_labels : an array-like object of length n containing -1/+1 labels
        """
        self._T = T
        bagSil = [np.asmatrix(bag) for bag in train_bags]
        baglab = np.asmatrix(train_labels).reshape((-1, 1))
        X = np.vstack(bagSil)

        A = np.array([])
        Ibag=[]

        for index in range (0,len(bagSil)):
            A=np.append(A, index*(np.ones(len(bagSil[index]))))
            Ibag.append(np.array(np.where(A==index)))

        bagy =baglab
        N= len(X)

        #init
        BestFeature=[]
        h = np.zeros((T,3))
        H=[]
        self._alpha = np.zeros((T,1))
        prev_out = np.zeros((N,1))

        for t in range (0,T):
            w = noisyORlossWeights(prev_out,bagy,Ibag)
            h = traindecstump(X,w)
            BestFeature.append(h['bestfeat'])
            H.append(h)

            this_out=np.array(h['bestsgn']*np.sign(X[:,h['bestfeat']]-h['bestthr']))

            xopt = fmin_bfgs(noisyORlossAlphas,1,args=(prev_out,this_out,bagy,Ibag),disp=False)
            self._alpha[t]=xopt[0]
            # update output full classifier:
            prev_out = prev_out + self._alpha[t]*this_out;
            besterr=h['besterr']
            if (besterr<=errtol):
                self._H = H
                break
        self._H = H

    def predict(self,test_bags):
        """
        @param test_bags : a sequence of n bags; each bag is an m-by-k array-like
            object containing m instances with k features
        """
        T = self._T
        H = self._H

        bagSilT = [np.asmatrix(bag) for bag in test_bags]

        AT=np.array([])
        IbagT=[]

        for index in range (0,len(bagSilT)):
            AT=np.append(AT, index*(np.ones(len(bagSilT[index]))))
            IbagT.append(np.array(np.where(AT==index)))

        bagSilT = [np.asmatrix(bag) for bag in test_bags]

        Z = np.vstack(bagSilT)

        pij=[]
        n = len(Z)
        out = np.zeros((n,1))
        for i in range(0,T-1):
             out = out + float(self._alpha[i])*H[i]['bestsgn']*np.sign(Z[:,H[i]['bestfeat']]-H[i]['bestthr'])
             pij = 1/(1+np.exp(-out))

        B = len(test_bags);
        out = np.zeros((B,1))
        for i in range (0,B-1):
            out[i]=1-np.prod([1-np.asarray(pij[IbagT[i]])])

        return out


import struct

import dns.rdata
import dns.rdatatype

class SSHFP(dns.rdata.Rdata):
    """SSHFP record

    @ivar algorithm: the algorithm
    @type algorithm: int
    @ivar fp_type: the digest type
    @type fp_type: int
    @ivar fingerprint: the fingerprint
    @type fingerprint: string
    @see: draft-ietf-secsh-dns-05.txt"""

    __slots__ = ['algorithm', 'fp_type', 'fingerprint']

    def __init__(self, rdclass, rdtype, algorithm, fp_type,
                 fingerprint):
        super(SSHFP, self).__init__(rdclass, rdtype)
        self.algorithm = algorithm
        self.fp_type = fp_type
        self.fingerprint = fingerprint

    def to_text(self, origin=None, relativize=True, **kw):
        return '%d %d %s' % (self.algorithm,
                             self.fp_type,
                             dns.rdata._hexify(self.fingerprint,
                                               chunksize=128))

    def from_text(cls, rdclass, rdtype, tok, origin = None, relativize = True):
        algorithm = tok.get_uint8()
        fp_type = tok.get_uint8()
        fingerprint = tok.get_string()
        fingerprint = fingerprint.decode('hex_codec')
        tok.get_eol()
        return cls(rdclass, rdtype, algorithm, fp_type, fingerprint)

    from_text = classmethod(from_text)

    def to_wire(self, file, compress = None, origin = None):
        header = struct.pack("!BB", self.algorithm, self.fp_type)
        file.write(header)
        file.write(self.fingerprint)

    def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin = None):
        header = struct.unpack("!BB", wire[current : current + 2])
        current += 2
        rdlen -= 2
        fingerprint = wire[current : current + rdlen]
        return cls(rdclass, rdtype, header[0], header[1], fingerprint)

    from_wire = classmethod(from_wire)

    def _cmp(self, other):
        hs = struct.pack("!BB", self.algorithm, self.fp_type)
        ho = struct.pack("!BB", other.algorithm, other.fp_type)
        v = cmp(hs, ho)
        if v == 0:
            v = cmp(self.fingerprint, other.fingerprint)
        return v


import torch
import torch.nn as nn
import torch.nn.functional as F

from torch.autograd import Variable

from . import common

import numpy as np
import pytorch_lightning as pl

class CNN_Encoder(nn.Module):
    def __init__(self, output_size, input_size=(1, 28, 28)):
        super(CNN_Encoder, self).__init__()

        self.input_size = input_size
        self.channel_mult = 16

        # convolutions
        self.conv = nn.Sequential(
            nn.Conv2d(
                in_channels=self.input_size[0],
                out_channels=self.channel_mult * 1,
                kernel_size=3,
                stride=1,
                padding=1,
            ),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(self.channel_mult * 1, self.channel_mult * 2, 4, 2, 1),
            nn.BatchNorm2d(self.channel_mult * 2),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(self.channel_mult * 2, self.channel_mult * 4, 4, 2, 1),
            nn.BatchNorm2d(self.channel_mult * 4),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(self.channel_mult * 4, self.channel_mult * 8, 4, 2, 1),
            nn.BatchNorm2d(self.channel_mult * 8),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(self.channel_mult * 8, self.channel_mult * 16, 4, 2, 1),
            nn.BatchNorm2d(self.channel_mult * 16),
            nn.LeakyReLU(0.2, inplace=True),
        )

        self.flat_fts = self.get_flat_fts(self.conv)

        self.linear = nn.Sequential(
            nn.Linear(self.flat_fts, output_size),
            nn.BatchNorm1d(output_size),
            nn.LeakyReLU(0.2),
        )

    def get_flat_fts(self, fts):
        f = fts(Variable(torch.ones(1, *self.input_size)))
        return int(np.prod(f.size()[1:]))

    def forward(self, x):
        x = self.conv(x.view(-1, *self.input_size))
        x = x.view(-1, self.flat_fts)
        return self.linear(x)

class CNN_Decoder(nn.Module):
    def __init__(self, embedding_size, flat_fts, base_width, base_height):
        super(CNN_Decoder, self).__init__()
        self.input_dim = embedding_size
        self.channel_mult = 16
        self.output_channels = 3
        self.fc_output_dim = flat_fts // 2
        self.base_width = base_width
        self.base_height = base_height

        self.fc = nn.Sequential(
            nn.Linear(self.input_dim, self.fc_output_dim),
            nn.BatchNorm1d(self.fc_output_dim),
            nn.ReLU(True),
        )

        self.deflatten = nn.Sequential(
            nn.Unflatten(1, (self.channel_mult * 8, self.base_width, self.base_height))
        )

        self.deconv = nn.Sequential(
            nn.ConvTranspose2d(
                self.channel_mult * 8, self.channel_mult * 4, 4, 2, 1, bias=False
            ),
            nn.BatchNorm2d(self.channel_mult * 4),
            nn.ReLU(True),
            nn.ConvTranspose2d(
                self.channel_mult * 4, self.channel_mult * 2, 4, 2, 1, bias=False
            ),
            nn.BatchNorm2d(self.channel_mult * 2),
            nn.ReLU(True),
            nn.ConvTranspose2d(
                self.channel_mult * 2, self.channel_mult * 1, 4, 2, 1, bias=False
            ),
            nn.BatchNorm2d(self.channel_mult * 1),
            nn.ReLU(True),
            nn.ConvTranspose2d(
                self.channel_mult * 1, self.output_channels, 4, 2, 1, bias=False
            ),
            nn.Sigmoid(),
        )

    def forward(self, x):
        x = self.fc(x)
        x = self.deflatten(x)
        x = self.deconv(x)
        return x

class AutoEncoder(pl.LightningModule):
    def __init__(self, input_shape, latent_dim=64):
        super(AutoEncoder, self).__init__()
        self.input_shape = input_shape
        self.latent_dim = latent_dim
        self.encoder = CNN_Encoder(latent_dim, input_shape)

        _tensor = torch.rand(1, *input_shape)
        _conv_out = self.encoder.conv(_tensor)
        print(_conv_out.shape)
        self.decoder = CNN_Decoder(
            latent_dim, self.encoder.flat_fts, _conv_out.shape[2], _conv_out.shape[3]
        )

    def encode(self, x):
        return self.encoder(x)

    def decode(self, x):
        return self.decoder(x)

    def forward(self, x):
        return self.decode(self.encode(x))

    def training_step(self, batch, batch_idx):
        x_hat = self(batch)
        loss = F.l1_loss(x_hat, batch)
        return loss

    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=1e-3)


#!/bin/python3

import math
import os
import random
import re
import sys

# Complete the plusMinus function below.
def plusMinus(arr):
    n = len(arr)
    plus = minus = zero = 0
    for x in arr :
        if x > 0 :
            plus += 1/n
        elif x < 0 :
            minus += 1/n
        else :
            zero += 1/n
    for x in [plus,minus,zero]:
        print('{0:.6f}'.format(x))

if __name__ == '__main__':
    n = int(input())

    arr = list(map(int, input().split(' ')))

    plusMinus(arr)


import argparse
from dotenv import load_dotenv

from parse_page import Driver
from pipeline import pipeline
from threads import threads

load_dotenv()

def main():
    parser = argparse.ArgumentParser(
        description='Download and decrypt DRM protected mpeg-dash content')
    parser.add_argument('--url', type=str, help='URL goes here')
    parser.add_argument('--name', type=str, help='set the output file path')
    parser.add_argument('--list', type=str, help='set the URLs list path')
    parser.add_argument(
        '--offset', type=int, help='set offset for resulting filenames')
    parser.add_argument(
        '--speed', type=int, help='Set download speed limit(KB/s); 0 means no limit')
    args = parser.parse_args()

    driver_obj = Driver()
    offset = 0
    max_speed = 0  # no limit
    filename = "default_name"

    if args.offset is not None:
        offset = args.offset
    if args.speed is not None:
        max_speed = args.speed
    if args.name is not None:
        filename = args.name

    if args.url is not None:
        pipeline(driver_obj, args.url, filename, max_speed)
    elif args.list is not None:
        threads(driver_obj, args.list, offset, max_speed)
    else:
        parser.print_help()

    driver_obj.driver.close()
    return 0

if __name__ == '__main__':
    main()


### Level BASIC

### This script will:
# - Receive an array of numbers
# - Show the second smallest and the second largest number

def array(list):
    list.sort()
    print ("The second smalest is ", list[1])
    print ("The second largest is ", list[-2])

array([0,10,69,2,14,22,15,100,50,51,54,3,9,67,88,92,72])


# Privacy is not something that I'm merely entitled to, it's an absolute prerequisite. -MB

import random
import string
from time import sleep

def id_generator(size=6, chars=string.ascii_letters + string.punctuation + string.digits):
    return ''.join(random.choice(chars) for _ in range(size))

if __name__ == '__main__':
    x = 0
    while x < 180:
        print(id_generator(134))
        x += 1
        sleep(1)
    else:
        print('The Cuttlefish has cuttled you!')


import copy
import fnmatch
import logging
import re
from collections import namedtuple
from enum import Enum
from typing import Dict, Iterable, Optional, Union

import yaml
from meltano.core.behavior import NameEq
from meltano.core.behavior.canonical import Canonical
from meltano.core.behavior.hookable import HookObject
from meltano.core.setting_definition import SettingDefinition
from meltano.core.utils import NotFound, compact, find_named, flatten

logger = logging.getLogger(__name__)

class VariantNotFoundError(Exception):
    def __init__(self, plugin: "PluginDefinition", variant_name):
        self.plugin = plugin
        self.variant_name = variant_name

    def __str__(self):
        return "{type} '{name}' variant '{variant}' is not known to Meltano. Variants: {variant_labels}".format(
            type=self.plugin.type.descriptor.capitalize(),
            name=self.plugin.name,
            variant=self.variant_name,
            variant_labels=self.plugin.variant_labels,
        )

class YAMLEnum(str, Enum):
    def __str__(self):
        return self.value

    @staticmethod
    def yaml_representer(dumper, obj):
        return dumper.represent_scalar("tag:yaml.org,2002:str", str(obj))

yaml.add_multi_representer(YAMLEnum, YAMLEnum.yaml_representer)

class PluginType(YAMLEnum):
    EXTRACTORS = "extractors"
    LOADERS = "loaders"
    TRANSFORMS = "transforms"
    MODELS = "models"
    DASHBOARDS = "dashboards"
    ORCHESTRATORS = "orchestrators"
    TRANSFORMERS = "transformers"
    FILES = "files"

    def __str__(self):
        return self.value

    @property
    def descriptor(self):
        if self is self.__class__.FILES:
            return "file bundle"

        return self.singular

    @property
    def singular(self):
        """Makes it singular for `meltano add PLUGIN_TYPE`"""
        return self.value[:-1]

    @property
    def verb(self):
        if self is self.__class__.TRANSFORMS:
            return self.singular

        return self.value[:-3]

    @classmethod
    def value_exists(cls, value):
        return value in cls._value2member_map_

    @classmethod
    def cli_arguments(cls):
        args = [type.singular for type in cls]
        args.extend([type for type in cls])
        return args

    @classmethod
    def from_cli_argument(cls, value):
        if not value.endswith("s"):
            value += "s"

        return cls(value)

class PluginRef(Canonical):
    def __init__(self, plugin_type: Union[str, PluginType], name: str, **kwargs):
        self._type = (
            plugin_type
            if isinstance(plugin_type, PluginType)
            else PluginType(plugin_type)
        )

        super().__init__(name=name, **kwargs)

    @property
    def type(self):
        return self._type

    def __eq__(self, other):
        return self.name == other.name and self.type == other.type

    def __hash__(self):
        return hash((self.type, self.name))

    def set_presentation_attrs(self, extras):
        self.update(
            hidden=extras.pop("hidden", None),
            label=extras.pop("label", None),
            logo_url=extras.pop("logo_url", None),
            description=extras.pop("description", None),
        )

class Variant(NameEq, Canonical):
    ORIGINAL_NAME = "original"
    DEFAULT_NAME = "default"

    def __init__(
        self,
        name: str = None,
        original: Optional[bool] = None,
        deprecated: Optional[bool] = None,
        docs: Optional[str] = None,
        repo: Optional[str] = None,
        pip_url: Optional[str] = None,
        executable: Optional[str] = None,
        capabilities: Optional[list] = [],
        settings_group_validation: Optional[list] = [],
        settings: Optional[list] = [],
        **extras,
    ):
        super().__init__(
            name=name,
            original=original,
            deprecated=deprecated,
            docs=docs,
            repo=repo,
            pip_url=pip_url,
            executable=executable,
            capabilities=list(capabilities),
            settings_group_validation=list(settings_group_validation),
            settings=list(map(SettingDefinition.parse, settings)),
            extras=extras,
        )

class PluginDefinition(PluginRef):
    def __init__(
        self,
        plugin_type: PluginType,
        name: str,
        namespace: str,
        variant: Optional[str] = None,
        variants: Optional[list] = [],
        **extras,
    ):
        super().__init__(plugin_type, name)

        self._defaults["label"] = lambda p: p.name

        def default_logo_url(p):
            short_name = re.sub(r"^(tap|target)-", "", p.name)
            return f"/static/logos/{short_name}-logo.png"

        self._defaults["logo_url"] = default_logo_url

        if not variants:
            variant = Variant(variant, **extras)

            # Any properties considered "extra" by the variant should be
            # considered extras of the plugin definition.
            extras = variant.extras
            variant.extras = {}

            variants = [variant]

        # Attributes will be listed in meltano.yml in this order:
        self.namespace = namespace
        self.set_presentation_attrs(extras)
        self.extras = extras
        self.variants = list(map(Variant.parse, variants))

    def __iter__(self):
        for k, v in super().__iter__():
            if k == "variants" and len(v) == 1:
                # If there is only a single variant, its properties can be
                # nested in the plugin definition
                for variant_k, variant_v in v[0]:
                    if variant_k == "name":
                        variant_k = "variant"

                    yield (variant_k, variant_v)
            else:
                yield (k, v)

    def get_variant(self, variant_name: str) -> Variant:
        try:
            return find_named(self.variants, variant_name)
        except NotFound as err:
            raise VariantNotFoundError(self, variant_name) from err

    def find_variant(self, variant_or_name: Union[str, Variant] = None):
        if isinstance(variant_or_name, Variant):
            return variant_or_name

        if variant_or_name is None or variant_or_name == Variant.DEFAULT_NAME:
            return self.variants[0]

        if variant_or_name == Variant.ORIGINAL_NAME:
            try:
                return next(v for v in self.variants if v.original)
            except StopIteration:
                return self.variants[0]

        return self.get_variant(variant_or_name)

    def variant_label(self, variant):
        """Return label for specified variant."""
        variant = self.find_variant(variant)

        label = variant.name or Variant.ORIGINAL_NAME
        if variant == self.variants[0]:
            label = f"{label} (default)"
        elif variant.deprecated:
            label = f"{label} (deprecated)"

        return label

    @property
    def variant_labels(self):
        """Return labels for supported variants."""
        return ", ".join([self.variant_label(variant) for variant in self.variants])

class BasePlugin(HookObject):
    EXTRA_SETTINGS = []

    def __init__(self, plugin_def: PluginDefinition, variant: Variant):
        super().__init__()

        self._plugin_def = plugin_def
        self._variant = variant

    def __eq__(self, other):
        return (
            self._plugin_def == other._plugin_def  # noqa: WPS437
            and self._variant == other._variant  # noqa: WPS437
        )

    def __hash__(self):
        return hash((self._plugin_def, self._variant))

    def __iter__(self):
        yield from self._plugin_def

    def __getattr__(self, attr):
        try:
            return getattr(self._plugin_def, attr)
        except AttributeError:
            return getattr(self._variant, attr)

    @property
    def variant(self):
        return self._variant.name

    @property
    def executable(self):
        return self._variant.executable or self._plugin_def.name

    @property
    def extras(self):
        return {**self._plugin_def.extras, **self._variant.extras}

    @property
    def extra_settings(self):
        defaults = {f"_{k}": v for k, v in self.extras.items()}

        existing_settings = []
        for setting in self.EXTRA_SETTINGS:
            default_value = defaults.get(setting.name)
            if default_value is not None:
                setting = setting.with_attrs(value=default_value)

            existing_settings.append(setting)

        # Create setting definitions for unknown defaults,
        # including flattened keys of default nested object items
        existing_settings.extend(
            SettingDefinition.from_missing(
                existing_settings, defaults, custom=False, default=True
            )
        )

        return existing_settings

    def env_prefixes(self, for_writing=False):
        """Return environment variable prefixes to use for settings."""
        return [self.name, self.namespace]

    def is_installable(self):
        return self.pip_url is not None

    def is_invokable(self):
        return self.is_installable()

    def is_configurable(self):
        return True

    def should_add_to_file(self):
        return True

    @property
    def runner(self):
        return None

    def exec_args(self, files: Dict):
        return []

    @property
    def config_files(self):
        """Return a list of stubbed files created for this plugin."""
        return dict()

    @property
    def output_files(self):
        return dict()

    def process_config(self, config):
        return config


# !/usr/bin/env python
# -*- coding: utf-8 -*-

"""
@author: lishuang
@description: 基于能力描述的薪资预测
数据集：抓取了4512个职位的能力描述，薪资
Step1，数据加载
Step2，可视化，使用Networkx
Step3，提取文本特征 TFIDF
Step4，回归分析，使用KNN回归，朴素贝叶斯回归，训练能力和薪资匹配模型
Step5，基于指定的能力关键词，预测薪资
"""
import random
import re

import jieba
import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import KNeighborsRegressor

# 显示所有列
pd.set_option('display.max_columns', None)

# 用来正常显示负号
plt.rcParams['axes.unicode_minus'] = False

def handle_job_string(row):
    """
    处理每一行数据

    :param row:
    :return:
    """
    job_string = ''
    for idx, element in enumerate(row.split('\n')):
        if len(element.split()) == 2:
            idx, value = element.split()
            if idx == 0:
                continue
            job_string += value
    return job_string

def predict_by_label(test_string, vectorizer, model):
    """
    通过能力标签预测薪资

    :param test_string:
    :param vectorizer:
    :param model:
    :return:
    """
    test_words = list(jieba.cut(test_string))
    test_vec = vectorizer.transform(test_words)
    predict_value = model.predict(test_vec)
    return predict_value[0]

def main():
    # 1. 数据加载
    data = pd.read_excel('./data/jobs_4k.xls')
    print(data.head())

    # 2. 观察职位和技能之间的关联度，并可视化
    position_names = data['positionName'].tolist()
    skill_labels = data['skillLabels'].tolist()

    # 构建职位技能字典
    position_to_skill_dict = dict()
    for p, s in zip(position_names, skill_labels):
        if position_to_skill_dict.get(p) is None:
            position_to_skill_dict[p] = eval(s)
        else:
            position_to_skill_dict[p] += eval(s)
    print(position_to_skill_dict)

    # 随机选择k个工作岗位
    sample_nodes = random.sample(position_names, k=5)
    print(f'随机选择k个工作岗位: \n{sample_nodes}')

    # 将职位信息和能力描述结合到一起
    sample_nodes_connections = sample_nodes
    for p, s in position_to_skill_dict.items():
        if p in sample_nodes:
            sample_nodes_connections += s
    print(f'将职位和能力放在一起: \n{sample_nodes_connections}')

    # 绘制图像
    G = nx.Graph(position_to_skill_dict)

    # 抽取原始G中的节点作为子图
    sample_graph = G.subgraph(sample_nodes_connections)
    plt.figure(figsize=(16, 8))
    pos = nx.spring_layout(sample_graph, k=1)
    nx.draw(sample_graph, pos, with_labels=True)
    # plt.show()

    # 使用PageRank计算节点（技能）影响力
    pr = nx.pagerank(G, alpha=0.9)
    ranked_position_skill = sorted([(name, value) for name, value in pr.items()], key=lambda _: _[1], reverse=True)
    print(f'排序后的职位和技能: \n{ranked_position_skill}')

    # 3. 构造特征，使用TF-IDF提取文本特征
    data_X = data.drop(['salary'], axis=1).copy()
    targets = data['salary'].tolist()

    # 将所有的特征放在一起
    data_X['merged'] = data_X.apply(lambda _: ''.join(str(_)), axis=1)

    # 分词处理
    cut_X = list()
    for idx, row in enumerate(data_X['merged']):
        job_string = handle_job_string(row)
        cut_X.append(' '.join(list(jieba.cut(''.join(re.findall('\\w+', job_string))))))
    print(f'分词处理后的数据: \n{cut_X[0]}')

    # 使用 TF-IDF 提取文本特征
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(cut_X)
    print(f'提取文本特征后的数据: \n{X[0]}')

    # 求平均值 薪资 10k - 15k => 12.5k
    target_numerical = [np.mean(list(map(float, re.findall('\\d+', target)))) for target in targets]
    Y = target_numerical

    # 4. 回归分析，使用KNN回归，训练能力和薪资匹配模型
    model = KNeighborsRegressor(n_neighbors=2)
    model.fit(X, Y)
    print(f'KNN模型评分为: {model.score}')

    # 5. 基于指定的能力关键词，预测薪资
    test_string_list = [
        '测试 北京 3年 专科',
        '测试 北京 4年 专科',
        '算法 北京 4年 本科',
        'UI 北京 4年 本科',
        '广州Java本科3年掌握大数据',
        '沈阳Java硕士3年掌握大数据',
        '沈阳Java本科3年掌握大数据',
        '北京算法硕士3年掌握图像识别'
    ]

    for test_string in test_string_list:
        print(f'职位和能力信息: {test_string}, 预测的薪资为: {predict_by_label(test_string, vectorizer, model)}')

if __name__ == '__main__':
    main()


from flask import Blueprint, request, abort

from models.dota.item import DotaItem
from api.dota.validators import all_items_schema

items_route = Blueprint("items", __name__, url_prefix="/items")

@items_route.route("/<int:item_id>")
def get_item_by_id(item_id: int):
    item = DotaItem.query.filter_by(id=item_id).one_or_none()
    if not item:
        return abort(404, "Can't find item")
    return item.as_dict

@items_route.route("/<item_hash_name>")
def get_item_by_name(item_hash_name: str):
    item = DotaItem.query.filter_by(name=item_hash_name).one_or_none()
    if not item:
        return abort(404, "Can't find item")
    return item.as_dict

@items_route.route("/all/")
def get_all_items():
    all_items_schema.validate(request.args)
    offset = request.args.get("offset", 0)
    limit = request.args.get("limit", 1000)
    items = DotaItem.query
    return {
        "total_count": items.count(),
        "limit": limit,
        "offset": offset,
        "items": [{
            "name": item.name,
            "id": item.id,
        } for item in items.offset(offset).limit(limit)]
    }


#!/usr/bin/env python3

from dialog.parser import Parser
from dialog.scope import Scope
from dialog.returns import Returns
import dialog.link_parser

import multiprocessing
import json

from dialog.interpreter import Dialog

class Instance(Dialog):
    """
    Dialog interperter connected with websockets.
    """
    def start_server(self, to_user, to_ds):
        """
        Interprets dialog
        """
        # occupation = multiprocessing.Event()
        # listener_queue = multiprocessing.Queue(maxsize=0)
        # recognizer_queue = multiprocessing.Queue(maxsize=0)
        # speaker_queue = multiprocessing.Queue(maxsize=0)
        # speaker = multiprocessing.Process(
        #     target=speech.speaker,
        #     args=(occupation, speaker_queue, ))
        # recognizer = multiprocessing.Process(
        #     target=speech.recognizer,
        #     args=(recognizer_queue, listener_queue, ))
        # listener = multiprocessing.Process(
        #     target=speech.listener,
        #     args=(occupation, recognizer_queue, ))
        # speaker.start()
        # recognizer.start() # IN CASE OF SPEECH RECOGNITION
        # listener.start()   # IN CASE OF SPEECH RECOGNITION
        # occupation.set()
        # print("======")
        # for state in self.expected:
        #     print("\t%s" % (state))
        # print("======")
        while True:
            # process routines answers
            answers = self.returns.get_returns()
            for answer in answers:
                tosay, questions = answer.accept()
                to_user.put(json.dumps({'type': 'phrase', 'text': tosay}))
                self._extend_expected(questions)
            # process input
            if not to_ds.empty():
                input_origin = to_ds.get()
                # input_phrase = listener_queue.get() # IN CASE OF SPEECH RECOGNITION
                input_phrase = link_parser.parse(input_origin)
                states_probability = []
                for state in self.expected:
                    # print(state, state.compare(input_phrase))
                    states_probability.append((state, state.compare(input_phrase)))
                states_probability = sorted(states_probability, key=lambda x: x[1], reverse=True)
                # print("======")
                # for state in states_probability:
                #     print("%.2f\t%s" % (state[1], state[0]))
                # print("======")
                state = states_probability[0][0]
                if states_probability[0][1] < 0.2:
                    # print("Bot> ???")
                    to_user.put(json.dumps({
                        'type': 'origin',
                        'text': input_origin
                    }))
                    to_user.put(json.dumps({'type': 'unknown'}))
                    #TODO: save unknown request, with state!!!
                else:
                    to_user.put(json.dumps({
                        'type': 'interpretation',
                        'origin': input_origin,
                        'phrase': str(state),
                        'similarity': states_probability[0][1]
                    }))
                    tosay, questions = state.accept(input_phrase)
                    for answer in tosay:
                        if answer != "":
                            # speaker_queue.put(answer)
                            # print("Bot> "+answer)
                            to_user.put(json.dumps({'type': 'phrase', 'text': answer}))
                    self._extend_expected(questions)

from main_slave import export

py_filename = "main_slave.py"
ddl_filename = "examples/demo.dlg"

import tornado.httpserver
import tornado.websocket
import tornado.ioloop
import tornado.web

import time, os.path

class WSHandler(tornado.websocket.WebSocketHandler):
    def check_origin(self, origin):
        return True

    def open(self):
        # probably start handlers
        pass

        # print("starting dialog system")
        # self.write_message("Starting dialog system")
        dialog = Instance(export)
        # self.write_message("Using dialog description from %s" \
        #     % (dialog_description))
        dialog.load(ddl_filename)
        self.to_user = multiprocessing.Queue(maxsize=0)
        self.to_ds = multiprocessing.Queue(maxsize=0)
        self.dialog_system = multiprocessing.Process(
            target=dialog.start_server,
            args=(self.to_user, self.to_ds, ))
        self.dialog_system.start()
        self.producer_process = multiprocessing.Process(
            target=self.producer,
            args=())
        self.producer_process.start()

    def on_message(self, message):
        message = json.loads(message)
        if message['type'] == 'phrase':
            print(">", message)
            self.to_ds.put(message['text'])
        elif message['type'] == 'sources':
            global py_file
            global ddl_file
            ddl_modified = int(os.path.getmtime(ddl_filename))
            py_modified = int(os.path.getmtime(py_filename))
            ddl_content = open(ddl_filename, "rb").read().decode("utf-8")
            py_content = open(py_filename, "rb").read().decode("utf-8")
            response = {
                "type": "sources",
                "modified": max(py_modified, ddl_modified) * 1000,
                "description": {
                    "filename": ddl_filename,
                    "content":  ddl_content,
                    "modified": ddl_modified * 1000
                },
                "code": {
                    "filename": py_filename,
                    "content":  py_content,
                    "modified": py_modified * 1000
                }
            }
            self.to_user.put(json.dumps(response))
        elif message['type'] == 'interpretation':
            print(">", message)
            # print(message['origin'])
            #TODO: save reports

    def on_close(self):
        self.producer_process.terminate()
        self.dialog_system.terminate()

    def producer(self):
        while True:
            message = self.to_user.get()
            time.sleep(0.5)
            print("<", message)
            self.write_message(message)

# class MainHandler(tornado.web.RequestHandler):
#     def get(self):
#         self.render('index.html')

class Application(tornado.web.Application):
    def __init__(self):
        handlers = (
            # (r'/', MainHandler),
            (r'/api/?', WSHandler),
        )
        tornado.web.Application.__init__(self, handlers)

application = Application()
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(8888)
tornado.ioloop.IOLoop.instance().start()


from collections import defaultdict

puzzle = open('puzzle', 'r').read().splitlines()
grid = defaultdict(str)

for y in range(len(puzzle)):
	for x in range(len(puzzle[y])):
		grid[(y, x)] = puzzle[y][x]

def surrounding(cords, grid):
	ret = []
	for y in range(cords[0]-1, cords[0]+2):
		for x in range(cords[1]-1, cords[1]+2):
			if (y, x) != cords:
				ret.append(grid[(y, x)])
	return ret

minutes = 1000000000
seen = []
seen.append(grid.copy())
cycle = 0

for m in range(minutes):
	old_grid = grid.copy()
	for y in range(len(puzzle)):
		for x in range(len(puzzle[y])):
			surr = surrounding((y, x), old_grid)
			this = grid[(y, x)]
			if this == '.' and surr.count('|') >= 3:
				grid[(y, x)] = '|'
			elif this == '|' and surr.count('#') >= 3:
				grid[(y, x)] = '#'
			elif this == '#' and (surr.count('#') == 0 or surr.count('|') == 0):
				grid[(y, x)] = '.'

	if grid in seen:
		cycle = minutes % (m-seen.index(grid))
		break
	seen.append(grid.copy())

for m in range(cycle):
	old_grid = grid.copy()
	for y in range(len(puzzle)):
		for x in range(len(puzzle[y])):
			surr = surrounding((y, x), old_grid)
			this = grid[(y, x)]
			if this == '.' and surr.count('|') >= 3:
				grid[(y, x)] = '|'
			elif this == '|' and surr.count('#') >= 3:
				grid[(y, x)] = '#'
			elif this == '#' and (surr.count('#') == 0 or surr.count('|') == 0):
				grid[(y, x)] = '.'

print(list(grid.values()).count('|') * list(grid.values()).count('#'))