from tensorflow.python.keras import losses
from autoencoder.autoencoders.base import BaseAutoencoder

import matplotlib.pyplot as plt

def build_autoencoder(latent_dimension):
    autoencoder = BaseAutoencoder(latent_dimension)
    autoencoder.compile(optimizer='adam', loss=losses.MeanSquaredError())
    return autoencoder

def fit_autoencoder(autoencoder, x_train, x_test):
    autoencoder.fit(x_train, x_train,
                    epochs=10,
                    shuffle=True,
                    validation_data=(x_test, x_test))

def show_autoencoder_result(results_count, autoencoder, x_test):
    encoded_images = autoencoder.encoder(x_test).numpy()
    decoded_images = autoencoder.decoder(encoded_images).numpy()

    plt.figure(figsize=(20, 4))
    for i in range(results_count):
        # display original
        ax = plt.subplot(2, results_count, i + 1)
        plt.imshow(x_test[i])
        plt.title("original")
        plt.gray()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

        # display reconstruction
        ax = plt.subplot(2, results_count, i + 1 + results_count)
        plt.imshow(decoded_images[i])
        plt.title("reconstructed")
        plt.gray()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
    plt.show()


import requests
from lxml import etree
import time

url = 'https://pic.netbian.com/'
header = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3880.400 QQBrowser/10.8.4554.400'
}
rep = requests.get(url, headers=header)   # 请求网页
text = rep.text            # 获取网页的全部源代码
# etree.HTML()可以用来解析字符串格式的HTML文档对象，将传进去的字符串转变成_Element对象。
html = etree.HTML(text)
# 作为_Element对象，可以方便的使用getparent()、remove()、xpath()等方法。
lists = html.xpath('//div[@class="slist"]/ul/li/a/span/img/@src')

for list in lists:
    full_url = url + list
    re = requests.get(full_url)
    with open('D:/mine/Python/photo/+%s.jpg' % time.time(), 'wb') as f:
        f.write(re.content)   # 用content属性获得bytes对象，二进制数据


# PAHADA MAKER---------------------------------------
def pahada(x,y = 10):
   i = 0
   for i in range(y):
      f = (i + 1) * x
      print(f)
# You can use this program to write tables of any number to any extent
# x = Number
# y = Range in which you want the multiplication table to print


#!/usr/bin/env python
# coding: utf-8

# In[1]:

get_ipython().system('pip install bs4')

# In[3]:

get_ipython().system('pip install requests')

# In[4]:

get_ipython().system('pip install pandas')

# In[232]:

from bs4 import BeautifulSoup
import requests
import pandas as pd

# In[295]:

URL = "https://www.amazon.in/s?k=bags&crid=2M096C61O4MLT&qid=1653308124&sprefix=ba%2Caps%2C283&ref=sr_pg_1"

# In[296]:

# Headers for request
HEADERS = ({'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', 'Accept-Language': 'en-US, en;q=0.5'})

# In[297]:

# HTTP Request
webpage = requests.get(URL,headers=HEADERS)

# In[298]:

webpage

# In[299]:

type(webpage.content)

# In[300]:

# Soup Object containiang all data
soup = BeautifulSoup(webpage.content, "html.parser")

# PRODUCT URL

# In[301]:

# # Fetch links as List of Tag Objects
# links = soup.find_all("a", attrs={'class':'a-link-normal s-underline-text s-underline-link-text s-link-style a-text-normal'})

# In[60]:

# link=[]

# for i in range(10):
#     link.append(links[i].get('href'))

# In[ ]:

# # list of links
# list

# PRODUCT NAME

# In[72]:

# Fetch Names as List of Tag Objects
# Names = soup.find_all("span", attrs={'class':'a-size-medium a-color-base a-text-normal'})

# In[79]:

# Name=[]

# for i in range(10):
#     Name.append(Names[i].text)

# In[88]:

#list of Names
# Name

# PRODUCT PRICE

# In[82]:

# Fetch Price as List of Tag Objects
# Prices = soup.find_all("span", attrs={'class':'a-price-whole'})

# In[89]:

# Price=[]

# for i in range(10):
#     Price.append(Prices[i].text)

# In[90]:

# list of prices
# Price

# PRODUCT RATING

# In[112]:

# Fetch Price as List of Tag Objects
# Ratings = soup.find_all("div", attrs={'class':'a-row a-size-small'})

# In[117]:

# Ratings[0].text[0:3]

# In[120]:

# Rating=[]

# for i in range(10):
#     Rating.append(Ratings[i].text[0:3])

# In[121]:

# list of ratings
# Rating

# PRODUCT REVIEWS

# In[125]:

# Fetch Price as List of Tag Objects
# Reviews = soup.find_all("span", attrs={'class':'a-size-base s-underline-text'})

# In[127]:

# Reviews[0].text

# In[128]:

# Review=[]

# for i in range(10):
#     Review.append(Reviews[i].text)

# In[129]:

# list of ratings
# Review

# ## function for all items

# In[316]:

link=[]
Name=[]
Price=[]
Rating=[]
Review=[]
Description=[]
ASIN=[]
ProductDsc=[]
Manufacturer=[]

# In[317]:

def collector(link,Name,Price,Rating,Review,Description):

    # Product URLs
    links = soup.find_all("a", attrs={'class':'a-link-normal s-underline-text s-underline-link-text s-link-style a-text-normal'})
    for i in range(15):
        a=links[i].get('href')
        link.append(a)
        new_webpage = requests.get("https://www.amazon.com"+a,headers=HEADERS)
        new_soup = BeautifulSoup(new_webpage.content, "html.parser")
        Description.append(soup.find_all("ul", attrs={'class':'a-unordered-list a-vertical a-spacing-mini'}))

    # Product Names
    Names = soup.find_all("span", attrs={'class':'a-size-medium a-color-base a-text-normal'})
    for i in range(15):
        Name.append(Names[i].text)

    #product price
    Prices = soup.find_all("span", attrs={'class':'a-price-whole'})
    for i in range(15):
        Price.append(Prices[i].text)

    # Product rating
    Ratings = soup.find_all("div", attrs={'class':'a-row a-size-small'})
    for i in range(15):
        Rating.append(Ratings[i].text[0:3])

    # Prodcut reviews
    Reviews = soup.find_all("span", attrs={'class':'a-size-base s-underline-text'})
    for i in range(15):
        Review.append(Reviews[i].text)

# In[318]:

collector(link,Name,Price,Rating,Review,Description)

# In[319]:

AmazonData=pd.DataFrame()

# In[320]:

AmazonData['URLs']=link
AmazonData['Name'] = Name
AmazonData['Price'] = Price
AmazonData['Rating'] = Rating
AmazonData['Review'] = Review
AmazonData['Description'] = Description

# In[321]:

AmazonData

# In[308]:

AmazonData.to_csv('AmazonData(part1).csv')

Web Scraping (Amazon) - PART 1 (1) (2).py


from __future__ import print_function

from apiclient import discovery
from httplib2 import Http
from oauth2client import file, client, tools

SCOPES = 'https://www.googleapis.com/auth/spreadsheets.readonly'

# The ID and range of a sample spreadsheet.
SAMPLE_SPREADSHEET_ID = '1ZH-JVx8LL09L0stBu2RBF-zi1ElqOtQapWFQAQBDvnc'
SAMPLE_RANGE_NAME = 'Class Data!A2:B3'
"""
def sendEmails() :
  sheet = SpreadsheetApp.getActiveSheet()
  startRow = 2
  numRows = 2
  dataRange = sheet.getRange(startRow, 1, numRows, 2)
  # Fetch values for each row in the Range.
  data = dataRange.getValues()
  for (i in data):
    row = data[i]
    emailAddress = row[0]  ' First column
    expDate = row[1] 'Second column
    name = row[2]
    subject = 'Sending emails from a Spreadsheet'
    message = name + 'Your certificate expires on ' + expDate
    MailApp.sendEmail(emailAddress, subject, message)

"""

def main():
  """Shows basic usage of the Sheets API>
  Prints values from a sample spreadsheet.
  """
  store = file.Storage('token.json')
  creds = store.get()
  if not creds or creds.invalid:
    flow = client.flow_from_clientsecrets('client_id.json', SCOPES)
    creds = tools.run_flow(flow, store)
  service = discovery.build('sheets', 'v4', http=creds.authorize(Http()))

  # Call the Sheets API
  SPREADSHEET_ID = '1ZH-JVx8LL09L0stBu2RBF-zi1ElqOtQapWFQAQBDvnc'
  RANGE_NAME = 'Class Data!A2:B3'
  result = service.spreadsheets().values().get(spreadsheetId=SPREADSHEET_ID, range=RANGE_NAME).execute()
  values = result.get('values', [])
  if not values:
    print('No data found.')
  else:
    print('Results:')
    for row in values:
      # Print columns A and E, which correspond to indices 0 and 4.
      emal = row[0]
      expdate = row[1]
      print(emal, expdate)

if __name__ == '__main__':
    main()

#https://developers.google.com/drive/api/v2/reference/files/get


import argparse
import os
import glob

class sgf_data():

    def __init__(self, boardsize):
        self.size = boardsize
        self.komi = 6.5
        self.history = []
        self.move_cnt = 0
        self.content = ""
        self.is_save = True

    def import_file(self, file_path):
        with open(file_path, 'r', encoding='utf-8') as f:
            # print(file_path)
            self.content = f.read()
            lines = self.content.split("\n")
            for line in lines:
                sgf_str = line.rstrip("\n")
                while len(sgf_str) > 3:
                    open_br = sgf_str.find("[")
                    close_br = sgf_str.find("]")
                    if open_br < 0 or close_br < 0:
                        # print("[Error] open_br < 0 or close_br < 0 -> %s" % file_path)
                        break
                    elif close_br == 0:
                        # print("[Error] close_br == 0 -> %s" % file_path)
                        sgf_str = sgf_str[close_br + 1:]
                        continue

                    # print(f"sgf_str[0:open_br]: {sgf_str[0:open_br]}")
                    key = sgf_str[0:open_br].lstrip(";").replace(' ', '')
                    # print(f"key: {key}")
                    val = sgf_str[open_br + 1:close_br]
                    # print(f"val: {val}")

                    if key == "SZ":
                        if val == str(self.size):
                            self.size = int(val)
                        else:
                            self.is_save = False
                            print(f"{self.size}路盤ではありません。")
                    elif key == "KM":
                        if val == "375":
                            self.komi = 7.5
                        elif val == "0":
                            if "HA[0]" in self.content:
                                if "RU[Japanese]" in self.content:
                                    self.komi = 6.5
                                elif "RU[Chinese]" in self.content:
                                    self.komi = 7.5
                            else:
                                self.is_save = False
                                print(f"コミ({val})は互先のものではありません。")
                    elif key == "B" or key == "W":
                        # print(val)
                        if key == "B":
                            self.history.append(val)
                        elif key == "W":
                            self.history.append(val)
                        self.move_cnt += 1

                    # print(sgf_str)
                    sgf_str = sgf_str[close_br + 1:]

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--input_dir", "-d", help="Input SGFs directory path.", type=str)
    parser.add_argument("--output_dir", "-o", help="Output SGFs directory path.", type=str)
    parser.add_argument("--boardsize", "-b", default=19, help="Board size.", type=int)
    parser.add_argument("--min_move_count", "-m", default=50, help="Minimum movement number.", type=int)
    args = parser.parse_args()

    input_files = []
    if args.input_dir:
        input_files = glob.glob(os.path.join(args.input_dir, "*"))

    if args.output_dir:
        out_dir = args.output_dir
        if not os.path.exists(out_dir):
            os.mkdir(out_dir)

    sd_list = []
    for sgf in input_files:
        print(sgf)
        sd_list.append(sgf_data(args.boardsize))
        sd_list[-1].import_file(sgf)

    for i, sd in enumerate(sd_list):
        if sd.is_save and sd.move_cnt < args.min_move_count:
            sd.is_save = False
            print(f"最低必要手数({str(args.min_move_count)})を満たしていません。")
        content = sd.content
        if "KM[375]" in sd.content:
            content = sd.content.replace("KM[375]", "KM[7.5]")
        elif "KM[0]" in sd.content and "RU[Japanese]" in sd.content:
            if "HA[0]" in sd.content:
                if "RU[Japanese]" in sd.content:
                    content = sd.content.replace("KM[0]", "KM[6.5]")
                elif "RU[Chinese]" in sd.content:
                    content = sd.content.replace("KM[0]", "KM[7.5]")
        if sd.is_save:
            out_path = os.path.join(out_dir, os.path.basename(input_files[i]))
            with open(out_path, "w", encoding="utf-8") as w:
                w.write(content)

if __name__ == "__main__":
    main()


import os
import gc

# Set loglevel to suppress tensorflow GPU messages
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'

import tensorflow as tf
import datasets
from change_detector import ChangeDetector
from image_translation import ImageTranslationNetwork
from change_priors import Degree_matrix, ztz, image_in_patches
from config import get_config_kACE
from decorators import image_to_tensorboard
import numpy as np
import matplotlib.pyplot as plt

class Kern_AceNet(ChangeDetector):   # 类Kern AceNet继承changeDetector类
    def __init__(self, translation_spec, **kwargs):
        """
                Input:
                    translation_spec - dict with keys 'enc_X', 'enc_Y', 'dec_X', 'dec_Y'.(带有键“enc_X，(编码器X)”，“ enc_Y”，“ dec_X(解码器X)”，“ dec_Y”的字典。)
                                       Values are passed as kwargs to the
                                       respective ImageTranslationNetwork's  (值作为kwargs传递到相应的ImageTranslationNetwork的)
                    cycle_lambda=2 - float, loss weight
                    cross_lambda=1 - float, loss weight
                    l2_lambda=1e-3 - float, loss weight
                    kernels_lambda - float, loss weight
                    learning_rate=1e-5 - float, initial learning rate for
                                         ExponentialDecay
                    clipnorm=None - gradient norm clip value, passed to
                                    tf.clip_by_global_norm if not None
                    logdir=None - path to log directory. If provided, tensorboard
                                  logging of training and evaluation is set up at
                                  'logdir/timestamp/' + 'train' and 'evaluation'  （日志目录的路径。 如果提供了训练和评估的张量板日志记录，则设置为“ logdir / timestamp /” +“ train”和“ evaluation”）
        """

        super().__init__(**kwargs)

        self.cycle_lambda = kwargs.get("cycle_lambda", 0.2)   # 找到字典kwargs中键为cycle_lambda所对应的值，如果指定键不存在时则返回设定的默认值，也就是0.2
        self.cross_lambda = kwargs.get("cross_lambda", 0.1)
        self.recon_lambda = kwargs.get("recon_lambda", 0.1)
        # self.wave_lambda = kwargs.get("wave_lambda", 0.5)
        self.l2_lambda = kwargs.get("l2_lambda", 1e-6)
        self.kernels_lambda = kwargs.get("kernels_lambda", 1)
        self.aps = kwargs.get("affinity_patch_size", 20)
        self.min_impr = kwargs.get("minimum improvement", 1e-3)
        self.patience = kwargs.get("patience", 10)

        # print(translation_spec["enc_X"])
        # print(translation_spec["enc_Y"])
        '''
        字典translation_spec中的键enc_X对应的值也是一个字典，值为{'input_chs': 7, 'filter_spec': [50, 50, 3]}
        字典translation_spec中的键enc_Y对应的值也是一个字典，值为{'input_chs': 10, 'filter_spec': [50, 50, 3]}
        '''
        # encoders of X and Y
        self._enc_x = ImageTranslationNetwork(
            **translation_spec["enc_X"], name="enc_X", l2_lambda=self.l2_lambda
        )  # 函数调用使用**：调用函数时分配参数，此处将enc_x对应的值(字典)分配给ImageTranslationNetwork的input_chs和filter_spec
           # 这里返回的self._enc_x是ImageTranslationNetwork的对象
        self._enc_y = ImageTranslationNetwork(
            **translation_spec["enc_Y"], name="enc_Y", l2_lambda=self.l2_lambda
        ) # 同上
        # print(self._enc_x )
        # print(translation_spec["dec_X"])
        # print(translation_spec["dec_Y"])
        '''
        字典translation_spec中的键dec_X对应的值也是一个字典，值为{'input_chs': 3, 'filter_spec': [50, 50, 7*4]}
        字典translation_spec中的键dec_Y对应的值也是一个字典，值为{'input_chs': 3, 'filter_spec': [50, 50, 10*4]}
        '''
        # decoder of X and Y
        self._dec_x = ImageTranslationNetwork(
            **translation_spec["dec_X"], name="dec_X", l2_lambda=self.l2_lambda
        )
        self._dec_y = ImageTranslationNetwork(
            **translation_spec["dec_Y"], name="dec_Y", l2_lambda=self.l2_lambda
        )

        self.loss_object = tf.keras.losses.MeanSquaredError()   # 均方误差
        # self.loss_object: <tensorflow.python.keras.losses.MeanSquaredError object at 0x00000269382F0E10>

        self.train_metrics["cycle_x"] = tf.keras.metrics.Sum(name="cycle_x MSE sum")
        # self.train_metrics["cycle_x"]:<tensorflow.python.keras.metrics.Sum object at 0x000001F62F39FEB8>
        self.train_metrics["cross_x"] = tf.keras.metrics.Sum(name="cross_x MSE sum")
        self.train_metrics["recon_x"] = tf.keras.metrics.Sum(name="recon_x MSE sum")
        self.train_metrics["cycle_y"] = tf.keras.metrics.Sum(name="cycle_y MSE sum")
        self.train_metrics["cross_y"] = tf.keras.metrics.Sum(name="cross_y MSE sum")
        self.train_metrics["recon_y"] = tf.keras.metrics.Sum(name="recon_y MSE sum")
        self.train_metrics["krnls"] = tf.keras.metrics.Sum(name="krnls MSE sum")
        self.train_metrics["l2"] = tf.keras.metrics.Sum(name="l2 MSE sum")
        self.train_metrics["total"] = tf.keras.metrics.Sum(name="total MSE sum")

        # self.train_metrics是一个字典，键有cycle_x、cross_x等，它们的值为<tensorflow.python.keras.metrics.Sum object...>

        # Track total loss history for use in early stopping  跟踪总损失历史记录以用于提前停止
        self.metrics_history["total"] = []   # metrics_history是字典，total为里面的一个键，这里为将空列表设为其值

    def save_all_weights(self):
        self._enc_x.save_weights(self.log_path + "/weights/_enc_x/")
        self._enc_y.save_weights(self.log_path + "/weights/_enc_y/")
        self._dec_x.save_weights(self.log_path + "/weights/_dec_x/")
        self._dec_y.save_weights(self.log_path + "/weights/_dec_y/")

    def load_all_weights(self, folder):
        self._enc_x.load_weights(folder + "/weights/_enc_x/")
        self._enc_y.load_weights(folder + "/weights/_enc_y/")
        self._dec_x.load_weights(folder + "/weights/_dec_x/")
        self._dec_y.load_weights(folder + "/weights/_dec_y/")

    @image_to_tensorboard()
    def enc_x(self, inputs, training=False):
        """ Wraps encoder call for TensorBoard printing and image save   包装编码器以进行TensorBoard(tesnorflow的可视化工具)打印和图像保存"""
        return self._enc_x(inputs, training)

    @image_to_tensorboard()
    def dec_x(self, inputs, training=False):
        return self._dec_x(inputs, training)

    @image_to_tensorboard()
    def enc_y(self, inputs, training=False):
        return self._enc_y(inputs, training)

    @image_to_tensorboard()
    def dec_y(self, inputs, training=False):
        return self._dec_y(inputs, training)

    def early_stopping_criterion(self):    # 提前结束循环函数
        temp = tf.math.reduce_min([self.stopping, self.patience]) + 1   # 第一个temp为1（0 + 1）
        # print('bbb',self.stopping)
        # print('aaa',temp)
        self.stopping.assign_add(1)  # self.stopping从0开始，每次一个进度条就加1
        last_losses = np.array(self.metrics_history["total"][-(temp):])  # last_losses是一个列表，每个进度条之后都会增加一个值
        # print('chengwei test last_losses',temp,last_losses)
        idx_min = np.argmin(last_losses)  # 取出最小的total_loss值的下标放入idx_min中
        if idx_min == (temp - 1):
            self.save_all_weights()
        while idx_min > 0:
            idx_2nd_min = np.argmin(last_losses[:idx_min])
            improvement = last_losses[idx_2nd_min] - last_losses[idx_min]
            if improvement > self.min_impr:
                break
            else:
                idx_min = idx_2nd_min
        stop = idx_min == 0 and self.stopping > self.patience
        tf.print(
            "total_loss",
            last_losses[-1],
            "Target",
            last_losses[idx_min],
            "Left",
            self.patience - (temp - 1) + idx_min,
        )
        return stop   # stop是一个布尔值

    @tf.function  #将模型以图模式运行
    def __call__(self, inputs, training=False):
        print('yeah here you are')
        x, y = inputs  # training为True时，x形状为(10,100,100,7),Training为False时，x形状为(1,1534,808,7)
                       # training为True时，y形状为(10,100,100,10),Training为False时，y形状为(1,1534,808,10)
        # print('chengwei test x', x.shape, x)
        # print('chengwei test y', y.shape, y)
        tf.debugging.Assert(tf.rank(x) == 4, [x.shape])  # tf.rank(x) == 4条件为假则打印[x.shape]这个张量
        tf.debugging.Assert(tf.rank(y) == 4, [y.shape])

        if training:
            print('True')
            x_code, y_code = self._enc_x(x, training), self._enc_y(y, training)  # 这里的x_code和y_code的形状都为(10,100,100,3)
            # print(type(x_code))
            # x_temp = x_code.numpy()
            # plt.matshow(x_temp[0, :, :, 2], cmap='viridis')
            print('chengwei test x_code1', x_code.shape, x_code)
            print('chengwei test y_code1', y_code.shape, y_code)
            x_hat, y_hat = self._dec_x(y_code, training), self._dec_y(x_code, training)  # 这里的x_hat的形状为(10,100,100,7),y_hat的形状为(10,100,100,10)
            print('chengwei test x_hat1', x_hat.shape, x_hat)
            print('chengwei test y_hat1', y_hat.shape, y_hat)
            x_dot, y_dot = (     # 这里的x_dot形状为(10,100,100,7),y_dot形状为(10,100,100,10)
                self._dec_x(self._enc_y(y_hat, training), training),
                self._dec_y(self._enc_x(x_hat, training), training),
            )
            print('chengwei test x_dot1', x_dot.shape, x_dot)
            print('chengwei test y_dot1', y_dot.shape, y_dot)
            x_tilde, y_tilde = (   # 这里的x_tilde形状为(10,100,100,7),y_tilde形状为(10,100,100,10)
                self._dec_x(x_code, training),
                self._dec_y(y_code, training),
            )
            print('chengwei test x_tilde1', x_tilde.shape, x_tilde)
            print('chengwei test y_tilde1', y_tilde.shape, y_tilde)

            # zx_t_zy = ztz(image_in_patches(x_code, 20), image_in_patches(y_code, 20))

            zx_t_zy = ztz(
                tf.image.central_crop(x_code, 0.4), tf.image.central_crop(y_code, 0.4)
            )
            print('chengwei test zx_t_zy:', zx_t_zy)
            retval = [x_code, y_code, x_hat, y_hat, x_dot, y_dot, x_tilde, y_tilde, zx_t_zy]

        else:
            print('False')
            x_code, y_code = self.enc_x(x, name="x_code"), self.enc_y(y, name="y_code")  # 这里的x_code和y_code形状都为(1,1534,808,3)
            print('chengwei test x_code2', x_code.shape, x_code)
            print('chengwei test y_code2', y_code.shape, y_code)
            x_tilde, y_tilde = (    # 这里的x_tilde形状为(1,1534,808,7),y_tilde形状为(1,1534,808,10)
                self.dec_x(x_code, name="x_tilde"),
                self.dec_y(y_code, name="y_tilde"),
            )
            print('chengwei test x_tilde2', x_tilde.shape, x_tilde)
            print('chengwei test y_tilde2', y_tilde.shape, y_tilde)
            x_hat, y_hat = (      # 这里的x_hat形状为(1,1534,808,7),y_hat形状为(1,1534,808,10)
                self.dec_x(y_code, name="x_hat"),
                self.dec_y(x_code, name="y_hat"),
            )
            print('chengwei test x_hat2', x_hat.shape, x_hat)
            print('chengwei test y_hat2', y_hat.shape, y_hat)
            difference_img = self._difference_img(x_tilde, y_tilde, x_hat, y_hat)
            # print('chengwei test difference_img1:', difference_img)
            retval = difference_img

        return retval

    # tf.config.run_functions_eagerly(True)

    @tf.function
    def _train_step(self, x, y, clw):
        """
        Input:
        x - tensor of shape (bs, ps_h, ps_w, c_x)
        y - tensor of shape (bs, ps_h, ps_w, c_y)
        clw - cross_loss_weight, tensor of shape (bs, ps_h, ps_w, 1)
        """
        # odd_row_clw = clw[:, 0::2]
        # even_row_clw = clw[:, 1::2]
        # temp_clw = (odd_row_clw + even_row_clw) / 2
        # # print('chenwgei test temp_clw:', temp_clw)
        # odd_colum_clw = temp_clw[:, :, 0::2]
        # even_colum_clw = temp_clw[:, :, 1::2]
        # wave_clw = (odd_colum_clw + even_colum_clw) / 2

        with tf.GradientTape() as tape:
            print('aaaa1')
            x_code, y_code, x_hat, y_hat, x_dot, y_dot, x_tilde, y_tilde, ztz = self(
                [x, y], training=True
            )
            print('hhhh1')
            Kern = 1.0 - Degree_matrix(
                tf.image.central_crop(x, 0.2), tf.image.central_crop(y, 0.2)
            )
            # print('chengwei test Kern:', Kern)

            kernels_loss = self.kernels_lambda * self.loss_object(Kern, ztz)
            # wave_loss = self.wave_lambda * self.loss_object(x_code, y_code)
            # print('chengwei test kernels_loss:', kernels_loss)

            # print('chengwei test _enc_x.losses:', self._enc_x.losses)

            # _enc_x.losses是一个列表：[<tf.Tensor 'enc_X/enc_X-00/kernel/Regularizer/mul:0' shape=() dtype=float32>,
            # <tf.Tensor 'enc_X/enc_X-01/kernel/Regularizer/mul:0' shape=() dtype=float32>,
            # <tf.Tensor 'enc_X/enc_X-02/kernel/Regularizer/mul:0' shape=() dtype=float32>]

            l2_loss_k = sum(self._enc_x.losses) + sum(self._enc_y.losses)
            # print('chengwei test l2_loss_k:', l2_loss_k)
            targets_k = (
                self._enc_x.trainable_variables + self._enc_y.trainable_variables
            )
            # targets_k:
            # [<tf.Variable 'enc_X/enc_X-00/kernel:0' shape=(3, 3, 28, 50) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-00/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-01/kernel:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-01/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-02/kernel:0' shape=(3, 3, 50, 3) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-02/bias:0' shape=(3,) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-00/kernel:0' shape=(3, 3, 40, 50) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-00/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-01/kernel:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-01/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-02/kernel:0' shape=(3, 3, 50, 3) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-02/bias:0' shape=(3,) dtype=float32>]

            gradients_k = tape.gradient(kernels_loss + l2_loss_k, targets_k)
            # gradients_k:
            # [<tf.Tensor 'AddN:0' shape=(3, 3, 28, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall:3' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_1:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall:5' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_2:0' shape=(3, 3, 50, 3) dtype=float32>,
            # <tf.Tensor 'PartitionedCall:7' shape=(3,) dtype=float32>,
            # <tf.Tensor 'AddN_3:0' shape=(3, 3, 40, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall:9' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_4:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall:11' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_5:0' shape=(3, 3, 50, 3) dtype=float32>,
            # <tf.Tensor 'PartitionedCall:13' shape=(3,) dtype=float32>]

            # print('chengwei test clipnorm1:', self.clipnorm)

            # self.clipnorm = 1
            if self.clipnorm is not None:
                gradients_k, _ = tf.clip_by_global_norm(gradients_k, self.clipnorm)
                # _: Tensor("global_norm/global_norm:0", shape=(), dtype=float32)

            self._optimizer_k.apply_gradients(zip(gradients_k, targets_k))
            print('yeah1')

        with tf.GradientTape() as tape:
            print('aaaa2')
            x_code, y_code, x_hat, y_hat, x_dot, y_dot, x_tilde, y_tilde, ztz = self(
                [x, y], training=True
            )
            print('aaaa2')
            l2_loss = (
                sum(self._enc_x.losses)
                + sum(self._enc_y.losses)
                + sum(self._dec_x.losses)
                + sum(self._dec_y.losses)
            )
            # print('chengwei test l2_loss:', l2_loss)
            cycle_x_loss = self.cycle_lambda * self.loss_object(x, x_dot)
            cross_x_loss = self.cross_lambda * self.loss_object(y, y_hat, clw)
            recon_x_loss = self.recon_lambda * self.loss_object(x, x_tilde)
            cycle_y_loss = self.cycle_lambda * self.loss_object(y, y_dot)
            cross_y_loss = self.cross_lambda * self.loss_object(x, x_hat, clw)
            recon_y_loss = self.recon_lambda * self.loss_object(y, y_tilde)

            total_loss = (
                cycle_x_loss
                + cross_x_loss
                + recon_x_loss
                + cycle_y_loss
                + cross_y_loss
                + recon_y_loss
                + l2_loss
            )

            # print('chenwgei test total_loss:', total_loss)
            targets_all = (
                self._enc_x.trainable_variables
                + self._enc_y.trainable_variables
                + self._dec_x.trainable_variables
                + self._dec_y.trainable_variables
            )
            # print('chengwei test targets_all:', targets_all)
            # targets_all:
            # [<tf.Variable 'enc_X/enc_X-00/kernel:0' shape=(3, 3, 28, 50) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-00/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-01/kernel:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-01/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-02/kernel:0' shape=(3, 3, 50, 3) dtype=float32>,
            # <tf.Variable 'enc_X/enc_X-02/bias:0' shape=(3,) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-00/kernel:0' shape=(3, 3, 40, 50) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-00/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-01/kernel:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-01/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-02/kernel:0' shape=(3, 3, 50, 3) dtype=float32>,
            # <tf.Variable 'enc_Y/enc_Y-02/bias:0' shape=(3,) dtype=float32>,
            # <tf.Variable 'dec_X/dec_X-00/kernel:0' shape=(3, 3, 3, 50) dtype=float32>,
            # <tf.Variable 'dec_X/dec_X-00/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'dec_X/dec_X-01/kernel:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Variable 'dec_X/dec_X-01/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'dec_X/dec_X-02/kernel:0' shape=(3, 3, 50, 28) dtype=float32>,
            # <tf.Variable 'dec_X/dec_X-02/bias:0' shape=(28,) dtype=float32>,
            # <tf.Variable 'dec_Y/dec_Y-00/kernel:0' shape=(3, 3, 3, 50) dtype=float32>,
            # <tf.Variable 'dec_Y/dec_Y-00/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'dec_Y/dec_Y-01/kernel:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Variable 'dec_Y/dec_Y-01/bias:0' shape=(50,) dtype=float32>,
            # <tf.Variable 'dec_Y/dec_Y-02/kernel:0' shape=(3, 3, 50, 40) dtype=float32>,
            # <tf.Variable 'dec_Y/dec_Y-02/bias:0' shape=(40,) dtype=float32>]

            gradients_all = tape.gradient(total_loss, targets_all)
            # gradients_all:
            # [<tf.Tensor 'AddN_6:0' shape=(3, 3, 28, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:3' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_7:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:5' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_8:0' shape=(3, 3, 50, 3) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:7' shape=(3,) dtype=float32>,
            # <tf.Tensor 'AddN_9:0' shape=(3, 3, 40, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:9' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_10:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:11' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_11:0' shape=(3, 3, 50, 3) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:13' shape=(3,) dtype=float32>,
            # <tf.Tensor 'AddN_12:0' shape=(3, 3, 3, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:15' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_13:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:17' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_14:0' shape=(3, 3, 50, 28) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:19' shape=(28,) dtype=float32>,
            # <tf.Tensor 'AddN_15:0' shape=(3, 3, 3, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:21' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_16:0' shape=(3, 3, 50, 50) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:23' shape=(50,) dtype=float32>,
            # <tf.Tensor 'AddN_17:0' shape=(3, 3, 50, 40) dtype=float32>,
            # <tf.Tensor 'PartitionedCall_1:25' shape=(40,) dtype=float32>]

            # print('chengwei test gradients_all:', gradients_all)

            # self.clipnorm = 1
            if self.clipnorm is not None:
                gradients_all, _ = tf.clip_by_global_norm(gradients_all, self.clipnorm)
                # _: Tensor("global_norm_1/global_norm:0", shape=(), dtype=float32)

            self._optimizer_all.apply_gradients(zip(gradients_all, targets_all))
            print('yeah2')

        self.train_metrics["cycle_x"].update_state(cycle_x_loss)
        self.train_metrics["cross_x"].update_state(cross_x_loss)
        self.train_metrics["recon_x"].update_state(recon_x_loss)
        self.train_metrics["cycle_y"].update_state(cycle_y_loss)
        self.train_metrics["cross_y"].update_state(cross_y_loss)
        self.train_metrics["recon_y"].update_state(recon_y_loss)
        self.train_metrics["krnls"].update_state(kernels_loss)
        self.train_metrics["l2"].update_state(l2_loss)
        self.train_metrics["total"].update_state(total_loss)

        # self.train_metrics:
        # {'cycle_x': <tensorflow.python.keras.metrics.Sum object at 0x00000252338471D0>,
        # 'cross_x': <tensorflow.python.keras.metrics.Sum object at 0x0000025233847550>,
        # 'recon_x': <tensorflow.python.keras.metrics.Sum object at 0x0000025233847828>,
        # 'cycle_y': <tensorflow.python.keras.metrics.Sum object at 0x0000025233847B00>,
        # 'cross_y': <tensorflow.python.keras.metrics.Sum object at 0x0000025233847DD8>,
        # 'recon_y': <tensorflow.python.keras.metrics.Sum object at 0x0000025233847FD0>,
        # 'krnls': <tensorflow.python.keras.metrics.Sum object at 0x00000252338543C8>,
        # 'l2': <tensorflow.python.keras.metrics.Sum object at 0x00000252338546A0>,
        # 'total': <tensorflow.python.keras.metrics.Sum object at 0x0000025233854978>}

        print('end')

def test(DATASET="Texas", CONFIG=None):
    """
    1. Fetch data (x, y, change_map)
    2. Compute/estimate A_x and A_y (for patches)
    3. Compute change_prior
    4. Define dataset with (x, A_x, y, A_y, p). Choose patch size compatible
       with affinity computations. 选择与亲和度计算兼容的补丁大小。
    5. Train CrossCyclicImageTransformer unsupervised
        a. Evaluate the image transformations in some way?
    6. Evaluate the change detection scheme
        a. change_map = threshold [(x - f_y(y))/2 + (y - f_x(x))/2]
    """
    if CONFIG is None:
        CONFIG = get_config_kACE(DATASET)
        '''CONFIG是一个字典，{'debug': False, 'clipnorm': 1, 'learning_rate': 0.0001, 'l2_lambda': 1e-06,
                             'logdir': 'logs/Texas/20210525-151916', 'save_images': True, 'channel_x': [1, 2, 3],
                             'channel_y': [3, 4, 7], 'filter_': <function decorated_median_filter.<locals>.median_filter2d at 0x00000138B12BE730>,
                             'final_filter': <function decorated_gaussian_filter.<locals>.gauss_filter at 0x00000138B12BE840>,
                             'patience': 10, 'minimum improvement': 0.001, 'list_epochs': [50, 50, 50, 50],
                             'batches': 10, 'batch_size': 10, 'patch_size': 100, 'affinity_batch_size': 500, 'affinity_patch_size': 20,
                             'affinity_stride': 5, 'epochs': 200, 'evaluation_frequency': 20, 'cycle_lambda': 1, 'cross_lambda': 1,
                             'recon_lambda': 1, 'kernels_lambda': 1}'''
        # print('chengwei test CONFIG:', CONFIG)

    print(f"Loading {DATASET} data")
    x_im, y_im, EVALUATE, (C_X, C_Y) = datasets.fetch(DATASET, **CONFIG)
    # print(list(EVALUATE.as_numpy_iterator()))
    '''x_im是形状shape=(1, 1534, 808, 7)的张量
       y_im是形状shape=(1, 1534, 808, 10)的张量
       EVALUATE：<TensorSliceDataset shapes: ((1534, 808, 7), (1534, 808, 10), (1534, 808, 1)), types: (tf.float32, tf.float32, tf.bool)>
       C_X: 7
       C_Y: 10'''
    # print('chengwei test x_im:', x_im)
    # print('chengwei test y_im:', y_im)
    # print('chengwei test EVALUATE:', type(EVALUATE))
    # print('chengwei test C_X:', C_X)
    # print('chengwei test C_Y:', C_Y)

    if tf.config.list_physical_devices("GPU") and not CONFIG["debug"]:
        C_CODE = 3
        print("here")
        TRANSLATION_SPEC = {
            "enc_X": {"input_chs": C_X, "filter_spec": [50, 50, C_CODE]},
            "enc_Y": {"input_chs": C_Y, "filter_spec": [50, 50, C_CODE]},
            "dec_X": {"input_chs": C_CODE, "filter_spec": [50, 50, C_X * 4]},
            "dec_Y": {"input_chs": C_CODE, "filter_spec": [50, 50, C_Y * 4]},
        }
    else:
        print("why here?")
        C_CODE = 1
        TRANSLATION_SPEC = {
            "enc_X": {"input_chs": C_X, "filter_spec": [C_CODE]},
            "enc_Y": {"input_chs": C_Y, "filter_spec": [C_CODE]},
            "dec_X": {"input_chs": C_CODE, "filter_spec": [C_X]},
            "dec_Y": {"input_chs": C_CODE, "filter_spec": [C_Y]},
        }
    print("Change Detector Init")
    cd = Kern_AceNet(TRANSLATION_SPEC, **CONFIG)
    print("Training")
    training_time = 0
    cross_loss_weight = tf.expand_dims(tf.zeros(x_im.shape[:-1], dtype=tf.float32), -1)  # -1表示最后一维
    # cross_loss_weight.shape=(1,1534,808,1)
    # print('chengwei test cross_loss_weight:', cross_loss_weight)
    for epochs in [50, 50, 50, 50]:  # CONFIG["list_epochs"] = [50,50,50,50]
        CONFIG.update(epochs=epochs)  # 更新CONFIG键epochs的值，改为50
        tr_gen, dtypes, shapes = datasets._training_data_generator(
            x_im[0], y_im[0], cross_loss_weight[0], CONFIG["patch_size"]
        )
        '''tr_gen: <function _training_data_generator.<locals>.gen at 0x000001EF8A2D36A8>
           dtypes: (tf.float32, tf.float32, tf.float32)
           shapes: (TensorShape([100, 100, 7]), TensorShape([100, 100, 10]), TensorShape([100, 100, 1]))
        '''

        TRAIN = tf.data.Dataset.from_generator(tr_gen, dtypes, shapes)

        '''TRAIN: <FlatMapDataset shapes: ((100, 100, 7), (100, 100, 10), (100, 100, 1)), types: (tf.float32, tf.float32, tf.float32)>
           <class 'tensorflow.python.data.ops.dataset_ops.FlatMapDataset'>'''
        TRAIN = TRAIN.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)  #使用 Dataset.prefetch() 方法进行数据预加载后的训练流程，在 GPU 进行训练的同时 CPU 进行数据预加载，提高了训练效率
        '''TRAIN: <PrefetchDataset shapes: ((100, 100, 7), (100, 100, 10), (100, 100, 1)), types: (tf.float32, tf.float32, tf.float32)>
           <class 'tensorflow.python.data.ops.dataset_ops.PrefetchDataset'>'''
        tr_time, _ = cd.train(TRAIN, evaluation_dataset=EVALUATE, **CONFIG)
        # _: <tf.Variable 'Variable:0' shape=() dtype=int64, numpy=50>
        # EVALUATE.batch(1): <BatchDataset shapes: ((None, 1534, 808, 7), (None, 1534, 808, 10), (None, 1534, 808, 1)), types: (tf.float32, tf.float32, tf.bool)>
        for x, y, _ in EVALUATE.batch(1):
            '''x:(1,1534,808,7)
               y:(1,1534,808,10)
               _:(1,1534,808,1)'''
            alpha = cd([x, y])  # shape=(1, 1534, 808, 1), dtype=float32
        cross_loss_weight = 1.0 - alpha
        training_time += tr_time

    cd.load_all_weights(cd.log_path)
    cd.final_evaluate(EVALUATE, **CONFIG)
    final_kappa = cd.metrics_history["cohens kappa"][-1]
    print('final_kappa:', final_kappa)
    final_acc = cd.metrics_history["ACC"][-1]
    print('final_acc:', final_acc)
    performance = (final_kappa, final_acc)
    timestamp = cd.timestamp
    epoch = cd.epoch.numpy()
    speed = (epoch, training_time, timestamp)
    del cd
    gc.collect()
    return performance, speed

if __name__ == "__main__":
    test("Texas")
    # test("California")
    # test("Italy")
    # test("France")
    # test("Shuguang")
    # test("Wuhan")


class Solution:
    def threeSum(self, nums: List[int]) -> List[List[int]]:
        def two_sum(target, index):
            temp_result = []
            for i in range(index, len(nums)):
                if (target - nums[i]) in nums[i+1:]:
                    temp_result.append([-target, nums[i], target - nums[i]])
            return temp_result

        result = []
        for i in range(len(nums)):
            for j in two_sum(-nums[i], i+1):
                j.sort()
                if j not in result:
                    result.append(j)
        return result


import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
from awsglueml.transforms import EntityDetector
from pyspark.sql.types import StringType
from awsglue.dynamicframe import DynamicFrame
from pyspark.sql.functions import *
from awsglue import DynamicFrame

def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame:
    for alias, frame in mapping.items():
        frame.toDF().createOrReplaceTempView(alias)
    result = spark.sql(query)
    return DynamicFrame.fromDF(result, glueContext, transformation_ctx)

args = getResolvedOptions(sys.argv, ["JOB_NAME"])
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args["JOB_NAME"], args)

# Script generated for node MySQL - Customers
MySQLCustomers_node1677029003448 = glueContext.create_dynamic_frame.from_catalog(
    database="gluestudio",
    table_name="gluestudio_customers",
    transformation_ctx="MySQLCustomers_node1677029003448",
)

# Script generated for node Aggregate - Sales
AggregateSales_node1677030461398 = glueContext.create_dynamic_frame.from_catalog(
    database="gluestudio",
    table_name="sales-aggregation-report",
    transformation_ctx="AggregateSales_node1677030461398",
)

# Script generated for node Change Schema (Apply Mapping)
ChangeSchemaApplyMapping_node1677029070654 = ApplyMapping.apply(
    frame=MySQLCustomers_node1677029003448,
    mappings=[
        ("zip", "string", "zip", "string"),
        ("country", "string", "country", "string"),
        ("address", "string", "address", "string"),
        ("gender", "string", "gender", "string"),
        ("city", "string", "city", "string"),
        ("prefix", "string", "prefix", "string"),
        ("dob", "date", "dob", "string"),
        ("last_name", "string", "last_name", "string"),
        ("customer_id", "int", "customer_id", "int"),
        ("first_name", "string", "first_name", "string"),
    ],
    transformation_ctx="ChangeSchemaApplyMapping_node1677029070654",
)

# Script generated for node SQL Query
SqlQuery0 = """
select zip as customer_zip, sum_total_sales from myDataSource
where product_type like 'Exercise Pen%' and sum_total_sales >= 2000

"""
SQLQuery_node1677030523756 = sparkSqlQuery(
    glueContext,
    query=SqlQuery0,
    mapping={"myDataSource": AggregateSales_node1677030461398},
    transformation_ctx="SQLQuery_node1677030523756",
)

# Script generated for node Detect PII - Names
entity_detector = EntityDetector()
classified_map = entity_detector.classify_columns(
    ChangeSchemaApplyMapping_node1677029070654, [], 0.55, 0.85
)

def maskDf(df, keys):
    if not keys:
        return df
    df_to_mask = df.toDF()
    for key in keys:
        df_to_mask = df_to_mask.withColumn(key, lit("Current resident"))
    return DynamicFrame.fromDF(df_to_mask, glueContext, "updated_masked_df")

DetectPIINames_node1677029124013 = maskDf(
    ChangeSchemaApplyMapping_node1677029070654, list(classified_map.keys())
)

# Script generated for node Detect PII - Dates
entity_detector = EntityDetector()
detected_df = entity_detector.detect(
    DetectPIINames_node1677029124013, ["DateOfbirth"], "DetectedEntities"
)

def replace_cell(original_cell_value, sorted_reverse_start_end_tuples):
    if sorted_reverse_start_end_tuples:
        for entity in sorted_reverse_start_end_tuples:
            to_mask_value = original_cell_value[entity[0] : entity[1]]
            original_cell_value = original_cell_value.replace(
                to_mask_value, "Current resident"
            )
    return original_cell_value

def row_pii(column_name, original_cell_value, detected_entities):
    if column_name in detected_entities.keys():
        entities = detected_entities[column_name]
        start_end_tuples = map(
            lambda entity: (entity["start"], entity["end"]), entities
        )
        sorted_reverse_start_end_tuples = sorted(
            start_end_tuples, key=lambda start_end: start_end[1], reverse=True
        )
        return replace_cell(original_cell_value, sorted_reverse_start_end_tuples)
    return original_cell_value

row_pii_udf = udf(row_pii, StringType())

def recur(df, remaining_keys):
    if len(remaining_keys) == 0:
        return df
    else:
        head = remaining_keys[0]
        tail = remaining_keys[1:]
        modified_df = df.withColumn(
            head, row_pii_udf(lit(head), head, "DetectedEntities")
        )
        return recur(modified_df, tail)

keys = DetectPIINames_node1677029124013.toDF().columns
updated_masked_df = recur(detected_df.toDF(), keys)
updated_masked_df = updated_masked_df.drop("DetectedEntities")

DetectPIIDates_node1677030205574 = DynamicFrame.fromDF(
    updated_masked_df, glueContext, "updated_masked_df"
)

# Script generated for node Join - Agg Sales/PII
JoinAggSalesPII_node1677030718697 = Join.apply(
    frame1=SQLQuery_node1677030523756,
    frame2=DetectPIIDates_node1677030205574,
    keys1=["customer_zip"],
    keys2=["zip"],
    transformation_ctx="JoinAggSalesPII_node1677030718697",
)

# Script generated for node Amazon S3
AmazonS3_node1677030919840 = glueContext.getSink(
    path="s3://glue-workshop-us-east-1-983991953101/glue-output/mailing-list/",
    connection_type="s3",
    updateBehavior="UPDATE_IN_DATABASE",
    partitionKeys=[],
    compression="snappy",
    enableUpdateCatalog=True,
    transformation_ctx="AmazonS3_node1677030919840",
)
AmazonS3_node1677030919840.setCatalogInfo(
    catalogDatabase="gluestudio", catalogTableName="mailing-list"
)
AmazonS3_node1677030919840.setFormat("glueparquet")
AmazonS3_node1677030919840.writeFrame(JoinAggSalesPII_node1677030718697)
job.commit()


import os
import sys
import time
import inspect
import traceback
import multiprocessing as mp
from multiprocessing import sharedctypes as shm
import io
import numpy as np

class PipeWrapper(object):
    def __init__(self, pipe=None, log_filename='', cmd_event=None, **kwargs):
        self.pipe = pipe
        self.log_filename = log_filename
        self.cmd_event = cmd_event

    def log_error(self, err, mode='a'):
        if self.log_filename != '':
            traceback.print_exc(None, err)
            with open(self.log_filename, mode) as fp:
                err.seek(0)
                fp.write(err.read())

    def log_str(self, s, mode="a", newline=True):
        if self.log_filename != '':
            if newline and not s.endswith("\n"):
                s += "\n"
            with open(self.log_filename, mode) as fp:
                fp.write(s)

class FuncProxy(object):
    '''
    Interface for calling functions in remote processes.
    '''
    def __init__(self, name, pipe, event, lock, log_filename=''):
        '''
        Constructor for FuncProxy

        Parameters
        ----------
        name : string
            Name of remote function to call
        pipe : mp.Pipe instance
            multiprocessing pipe through which to send data (function name, arguments) and receive the result
        event : mp.Event instance
            A flag to set which is multiprocessing-compatible (visible to both the current and the remote processes)

        Returns
        -------
        FuncProxy instance
        '''
        self.pipe = pipe
        self.name = name
        self.event = event
        self.lock = lock
        self.log_filename = log_filename

    def log_error(self, err, mode='a'):
        if self.log_filename != '':
            traceback.print_exc(None, err)
            with open(self.log_filename, mode) as fp:
                err.seek(0)
                fp.write(err.read())

    def log_str(self, s, mode="a", newline=True):
        if self.log_filename != '':
            if newline and not s.endswith("\n"):
                s += "\n"
            with open(self.log_filename, mode) as fp:
                fp.write(s)

    def __call__(self, *args, **kwargs):
        '''
        Return the result of the remote function call

        Parameters
        ----------
        *args, **kwargs : positional arguments, keyword arguments
            To be passed to the remote function associated when the object was created

        Returns
        -------
        function result
        '''
        self.lock.acquire()
        self.log_str('lock acquired')

        # block until the remote process unsets the event. Acts as a lock
        n_ticks = 0
        while self.event.is_set() and n_ticks < 100:
            n_ticks += 1
            time.sleep(0.1)

        if n_ticks >= 100:
            raise Exception("Out of sync!")

        # print("FuncProxy.__call__", self.name, args, kwargs)

        self.pipe.send((self.name, args, kwargs))
        self.event.set()
        # resp = self.pipe.recv()
        if self.pipe.poll(10):
            resp = self.pipe.recv()
        else:
            raise Exception("FuncProxy: remote object failed to respond")

        self.lock.release()
        self.log_str('lock released')
        return resp

class ObjProxy(PipeWrapper):
    def __init__(self, target_class, *args, **kwargs):
        self.target_class = target_class
        is_instance_method = lambda n: inspect.isfunction(getattr(self.target_class, n))
        self.methods = set(filter(is_instance_method, dir(self.target_class)))

        super().__init__(*args, **kwargs)
        self.lock = mp.Lock()

    def __getattr__(self, attr):
        self.log_str("remotely getting attribute: %s\n" % attr)

        methods = object.__getattribute__(self, "methods")
        pipe = object.__getattribute__(self, "pipe")
        cmd_event = object.__getattribute__(self, "cmd_event")
        lock = object.__getattribute__(self, "lock")

        if attr in methods:
            self.log_str("returning function proxy for %s" % attr)
            return FuncProxy(attr, pipe, cmd_event, lock, log_filename=self.log_filename)
        else:
            self.log_str("sending __getattribute__ over pipe")
            fn_getattr = FuncProxy("__getattribute__", pipe, cmd_event, lock, log_filename=self.log_filename)
            return fn_getattr(attr)

    def set(self, attr, value):
        self.log_str("ObjProxy.setattr")
        pipe = object.__getattribute__(self, "pipe")
        cmd_event = object.__getattribute__(self, "cmd_event")
        lock = object.__getattribute__(self, "lock")
        setattr_fn = FuncProxy('__setattr__', pipe, cmd_event, lock, log_filename=self.log_filename)

        setattr_fn(attr, value)
        self.log_str("Finished setting remote attr %s to %s" % (str(attr), str(value)))

    def terminate(self):
        self.pipe.send(None)

class DataPipe(PipeWrapper):
    pass

def call_from_remote(x):
    return x

def call_from_parent(x):
    return x

class RPCProcess(mp.Process):
    """mp.Process which implements remote procedure call (RPC) through a mp.Pipe object"""
    def __init__(self, target_class=object, target_kwargs=dict(), log_filename=''):
        super().__init__()
        self.cmd_pipe = None
        self.data_pipe = None
        self.log_filename = log_filename
        if self.log_filename != '':
            with open(self.log_filename, 'w') as f:
                f.write('')

        self.target = None
        self.target_class = target_class
        self.target_kwargs = target_kwargs

        self.cmd_event = mp.Event()
        self.status = mp.Value('b', 1) # mp boolean used for terminating the remote process

        self.target_proxy = None
        self.data_proxy = None

    def __getattr__(self, attr):
        """ Redirect attribute access to the target object if the
        attribute can't be found in the process wrapper """
        try:
            if self.target_proxy is not None and self.status.value > 0:
                try:
                    return getattr(self.target_proxy, attr)
                except:
                    raise AttributeError("RPCProcess: could not forward getattr %s to target of type %s" % (attr, self.target_class))
            else:
                raise AttributeError("RPCProcess: target proxy not initialized")
        except:
            raise AttributeError("Could not get RPCProcess attribute: %s" % attr)

    def log_error(self, err, mode='a'):
        if self.log_filename != '':
            traceback.print_exc(None, err)
            with open(self.log_filename, mode) as fp:
                err.seek(0)
                fp.write(err.read())

    def log_str(self, s, mode="a", newline=True):
        if self.log_filename != '':
            if newline and not s.endswith("\n"):
                s += "\n"
            with open(self.log_filename, mode) as fp:
                fp.write(s)

    @call_from_remote
    def target_constr(self):
        try:
            self.target = self.target_class(**self.target_kwargs)
            if hasattr(self.target, 'start'):
                self.target.start()
        except Exception as e:
            print("RPCProcess.target_constr: unable to start source!")
            print(e)

            import io
            err = io.StringIO()
            self.log_error(err, mode='a')
            err.seek(0)

            self.status.value = -1

    @call_from_remote
    def target_destr(self, ret_status, msg):
        pass

    @call_from_parent
    def start(self):
        self.cmd_pipe, pipe_end2 = mp.Pipe()
        self.data_pipe, data_pipe2 = mp.Pipe()

        self.target_proxy = ObjProxy(self.target_class, pipe=pipe_end2,
            log_filename=self.log_filename, cmd_event=self.cmd_event)
        self.data_proxy = DataPipe(data_pipe2)
        super().start()
        return self.target_proxy, self.data_proxy

    def check_run_condition(self):
        return self.status.value > 0

    def is_enabled(self):
        return self.status.value > 0

    @call_from_parent
    def stop(self):
        self.status.value = -1;

    def __del__(self):
        '''Stop the process when the object is destructed'''
        if self.status.value > 0:
            self.stop()

    def is_cmd_present(self):
        return self.cmd_event.is_set()

    def clear_cmd(self):
        self.log_str("clearing")
        self.cmd_event.clear()

    @call_from_remote
    def loop_task(self):
        time.sleep(0.01)

    @call_from_remote
    def run(self):
        self.log_str("RPCProcess.run")

        self.target_constr()

        try:
            while self.is_enabled():
                if not self.check_run_condition():
                    self.log_str("The target's termination condition was reached")
                    break

                if self.is_cmd_present():
                    self.proc_rpc_command()
                    self.clear_cmd()

                self.loop_task()

            self.log_str("RPCProcess.run: end of while loop")
            ret_status, msg = 0, ''
        except KeyboardInterrupt:
            ret_status, msg = 1, 'KeyboardInterrupt'
        except:
            import traceback
            traceback.print_exc()
            err = io.StringIO()
            self.log_error(err, mode='a')
            err.seek(0)
            ret_status, msg = 1, err.read()
        finally:
            self.target_destr(ret_status, msg)
            self.status.value = -1

    @call_from_remote
    def proc_rpc_command(self):
        cmd = self.cmd_pipe.recv()
        self.log_str("Received command: %s" % str(cmd))

        if cmd is None:
            self.stop()
            return

        try:
            fn_name, cmd_args, cmd_kwargs = cmd
            fn = getattr(self.target, fn_name)
            self.log_str("Function: " + str(fn))

            fn_output = fn(*cmd_args, **cmd_kwargs)
            self.cmd_pipe.send(fn_output)

            self.log_str('Done with command: %s, output=%s\n\n' % (fn_name, fn_output))
        except Exception as e:
            err = io.StringIO()
            self.log_error(err, mode='a')

            self.cmd_pipe.send(e)


#!/home/work/python/bin/python
# -*- coding:utf-8 -*-
'''
监控脚本，监控地名天气阿拉丁在CMS的redis中是否为第一位候选，并立即短信报警
不是第一位的话立即短信报警
依赖库：
    1. urllib
    2. urllib2
    3. json
	4. ConfigParser
	5. unittest
	6. os

作者：lijingtao@baidu.com
时间：2015-12-18

'''
import urllib
import urllib2
import json
import ConfigParser
import unittest
import os

from send_sms import cms_monitor

def execute_request(url='',data={},method='post'):
    if method=='post':
        req = urllib2.Request(url=url,data=urllib.urlencode(data))
    else:
        req = urllib2.Request(url=url)
    try:
        req_data = urllib2.urlopen(req)
        res = json.loads(req_data.read())
    except Exception as e:
        #print e
        res = {'errno':'xxx','errmsg':'failed'}
    return res

def setUp():
    config = ConfigParser.ConfigParser()
    config_dir = os.path.join(os.path.dirname(__file__), "conf.ini")
    config.read(config_dir)
    targetUrl = config.get("cms2redis", "targetUrl")
    return targetUrl

def is_top(targetUrl, list_data):
    """检查redis中天气阿拉丁词条异常"""
    resp = execute_request(targetUrl+"/list", list_data)
    try:
        if len(resp['data']) == 0:
            return False
        arr_word = resp['data'][0]
        if arr_word['comment'] != 'weather aladin' or arr_word['is_cache'] == 1 or arr_word['is_learn'] == 1 or arr_word['type'] != 2 or arr_word['is_top'] != 1 or arr_word['is_righttop'] != 1:
            return False
        return True
    except Exception as e:
        return False

def main():
    """测试case1：检查地名在redis中的候选，天气阿拉丁词条一定是第一位，并检查aladdin的策略是否正确"""
    list_data = {
            "pron":"",
            "device_type":0
            }
    arr_err_pron = []
    targetUrl = setUp()
    with open(os.getcwd() + "/place", 'r') as fopen:
        for line in fopen:
            pron = line.strip()
            list_data['pron'] = pron
            list_data['device_type'] = 1
            flag1 = is_top(targetUrl, list_data)
            list_data['device_type'] = 2
            flag2 = is_top(targetUrl, list_data)
            if False == flag1:
                arr_err_pron.append("(%s,%d)" % (pron, 1))
            if False == flag2:
                arr_err_pron.append("(%s,%d)" % (pron, 2))
        if len(arr_err_pron) != 0:
            errs = ",".join(arr_err_pron)
            cms_monitor(u"CMSredis中天气阿拉丁词条异常[{}]".format(errs.decode('utf-8')))

if __name__=='__main__':
    main()
    exit(0)


from collections import Counter, defaultdict
from .converter import save_metadata
from docx import Document
from docx.shared import RGBColor
from numpy import median, std
from os.path import dirname, isfile, realpath
from re import finditer, MULTILINE, search
from requests import get
from string import Template
from xml.etree import ElementTree

PATH_TO_DIRECTORY_OF_THIS_FILE = dirname(realpath(__file__))

with open(PATH_TO_DIRECTORY_OF_THIS_FILE + "/prep/stopwords.txt", encoding="utf-8") as f:
    stopwords = f.read().strip().split("\n")
with open(PATH_TO_DIRECTORY_OF_THIS_FILE + "/prep/tags.txt", encoding="utf-8") as f:
    tags = sorted(f.read().strip().split("\n"), key=lambda tag: -1*len(tag))

def flatten(iterable, levels=100):
    flattened = []
    levels -= 1
    for item in iterable:
        if isinstance(item, list) and levels >= 0:
            flattened.extend(flatten(item, levels))
        else:
            flattened.append(item)
    return flattened

def simplify_positions(positions):
    #positions = [3, 4, 5, 6, 10, 11, 13, 14, 15, 16, 20, 21, 22, 23, 24, 25, 28, 29, 30, 31]
    #positions = [3, 6, 10, 11, 13, 16, 20, 25, 28, 31]
    return [position for position in positions if position - 1 not in positions or position + 1 not in positions]

def join_nested_items(_input, joiner="\n"):
    text = ""
    if isinstance(_input, str) or isinstance(_input, str):
        text = _input
    elif isinstance(_input, list):
        for item in _input:
            if isinstance(item, list):
                text += join_nested_items(item)
            elif isinstance(item, str) or isinstance(item, str):
                text += joiner + item
    return text

def count_nested_items(iterable):
    count = 0
    for item in iterable:
        if isinstance(item, list):
            count += count_nested_items(item)
        else:
            count += 1
    return count

# not splitting this correctly!!!! need to figure out offsetting even with iterables
# or maybe not returning stuff with correct depth
def split_by_indices(items, positions, offset=0, return_i=False):
    #last_position = len(flatten(items))
    offset_positions = [position - offset for position in positions]
    #if 2 in positions and 27 in positions and 54 in positions:
    #print "\n\nstarting split_by_indices with "
    #print "\titerable:", items
    #print "\tpositions:", positions
    #print "\toffset:", offset
    #print "\toffset_positions:", offset_positions
    try:
        #result = [item for item in [iterable[offset_positions[current_index - 1] if current_index > 0 else 0 : current_position] for current_index, current_position in enumerate(offset_positions)] if item]
        result = []
        sublist = []
        i = 0
        for item in items:
            if isinstance(item, list):
                if sublist:
                    result.append(sublist)
                item, i = split_by_indices(item, positions, offset=i, return_i=True)
                result.append(item)
            else:
                if i in offset_positions:
                    result.append(sublist)
                    sublist = [item]
                else:
                    sublist.append(item)
                i += 1
        if sublist: result.append(sublist)
        if return_i:
            return result, i
        else:
            return result

    except Exception as e:
        print("EXCEPTION trying to split by indices:", e)

# should I lowercase everything??
def tokenize(text):
    punctuation = ["%3A","%3B","%3C","%3D","%3E","%3F","{","}","-","_","/",":","_",".","\\",">","<","&","=","+","'",'"',"&","#","%2C",")","(","@","?",":","!",";",","]
    for p in punctuation:
        text = text.replace(p, " ")
    terms = text.split()
    terms = [term for term in terms if 3 < len(term) < 10 and not search("\d", term) and term not in stopwords]
    return terms

def is_metadata(inpt, debug=False):
  try:
    if debug: print("starting is_metadata with", type(inpt))
    _type = str(type(inpt))
    if isinstance(inpt, str) or isinstance(inpt, str):
        if isfile(inpt):
            if inpt.endswith(".txt"):
                with open(inpt, encoding="utf-8") as f:
                    text = f.read()
                return extract_metadata_from_text(text)
            elif inpt.endswith(".doc") or inpt.endswith(".docx"):
                with open(inpt, "rb") as f:
                    text = "\n".join([p.text for p in Document(f).paragraphs])
        elif inpt.strip().startswith("http") and inpt.strip().count("\n") < 3:
            if debug: print("inpt is a url")
            text = get(inpt).text
        else:
            if debug: print("inpt is regular text")
            text = inpt
    elif _type == "<class 'django.core.files.uploadedfile.InMemoryUploadedFile'>":
        if inpt.name.endswith(".doc") or inpt.name.endswith(".docx"):
            text = "\n".join([p.text.encode("utf-8") for p in Document(inpt).paragraphs])
        elif inpt.name.endswith(".csv") or inpt.name.endswith(".tsv"):
            text = inpt.read()
        else:
            if debug: print("[metadata-extractor] is_metadata returning False because filetype not supported yet")
            return False # return false if not type of supported file
    else:
        if debug: print("[metadata-extractor] is_metadata returning False because filetype not supported yet")
        return False # return false if not type of supported file

    tag_parts = flatten([tag.split() for tag in tags])
    tokens = tokenize(text)
    number_of_tokens = len(tokens)
    if number_of_tokens == 0:
        return False
    else:
        percentage = float(len([token for token in tokens if token in tag_parts])) / number_of_tokens
        if debug: print("percentage:", percentage)

        # if more than 5% of the input is metadata terms, it's about metadata
        return percentage > 0.05
  except Exception as e:
    print("CAUGHT EXCEPTION in is_metadata:", e)
    raise e

def extract_metadata(inpt):
    _type = str(type(inpt))
    if isinstance(inpt, str) or isinstance(inpt, str):
        if isfile(inpt):
            print("filepath")
            if inpt.endswith(".txt"):
                with open(inpt, encoding="utf-8") as f:
                    text = f.read()
                return extract_metadata_from_text(text)
            elif inpt.endswith(".doc") or inpt.endswith(".docx"):
                with open(inpt, "rb") as f:
                    doc = Document(f)
                return extract_metadata_from_doc(doc)
    elif _type == "<class 'django.core.files.uploadedfile.InMemoryUploadedFile'>":
        doc = Document(inpt)
        return extract_metadata_from_doc(doc)

def extract_metadata_from_doc(doc, debug=False):

    tree = treeify(doc)

    #print "tree:", type(tree)

    level = 0
    while level <= 10:
        #print "level:", level
        counters = []
        items = flatten(tree, levels=level)
        for item in items:
            text = join_nested_items(item)
            #print "\tlen text:", len(text)
            counter = Counter()
            for tag in tags:
                counter[tag] += text.count(tag)
                #count = counter[tag]
                #if count > 0:
                #    print "\t",tag,":", count
            #print "\t\tcounter:", counter.most_common(5)
            counters.append(counter)

        averages = Counter()
        more_than_two = 0
        for tag in tags:
            averages[tag] = average = median([counter[tag] for counter in counters])
            if average > 2:
                more_than_two += 1

        print("more_than_two:", more_than_two)
        print("averages:", averages.most_common(5))
        if more_than_two < 3:
            print("breaking on level", level)
            break
        else:
            level += 1

    if debug:
        for level in [0,1,2,3,4]:
            with open("/tmp/level_" + str(level) + ".txt", "wb") as f:
                for i, item in enumerate(flatten(tree, levels=level)):
                   text = join_nested_items(item)
                   f.write("\n" + "|*|" + str(i) + "|*|" + text.encode("utf-8"))

    metadata = []
    for item in items:
        for d in parse_text(join_nested_items(item)):
            if d:
                metadata.append(d)

    #print "metadata:", metadata
    return metadata

# only works if one layer in text now
def parse_text(text):

    print("starting parse_text with:", type(text), len(text))

    layers = []
    layer = {}

    hit_tag = False
    for line in text.split("\n"):
        for tag in tags:
            if tag in line:
                hit_tag = True
                break
        if hit_tag:
            break
        if line and 5 < len(line) < 50:
            name = line.strip()
            #print "name:", name
            layer['NAME'] = name

    pattern = "^(?P<key>" + "|".join([tag.replace("(","\(").replace(")","\)") for tag in tags]) + ")[(\r\n)|\n](?P<value>[^\n]+)$"
    #print "pattern:", pattern
    #print "[]" * 30
    for mg in finditer(pattern, text, MULTILINE):
        try: print(mg.group("key"), ":>>:", mg.group("value"))
        except: pass
        layer[mg.group("key")] = mg.group("value")
    #print "\nlayer:", layer
    layers.append(layer)
    """
    active_tag = None
    active_text = ""
    for p in text.split("\n"):
        for tag in tags:
            if tag in p:
                if tag in layer:
                    layers.append(layer)
                    layer = {}
                    active_tag = tag
                else:
                    if active_text:
                        layer[tag] = active_text
                        print "tag:", tag
                    active_tag = tag
            else:
                active_text += p
    """

    return layers

def treeify(doc):

    d = {}

    paragraphs = doc.paragraphs
    number_of_paragraphs = len(paragraphs)

    for i, p in enumerate(paragraphs):
        bold = p.style.font.bold or False
        color = p.style.font.color.rgb or RGBColor(0x00, 0x00, 0x00)
        italic = p.style.font.italic or False
        size = (p.style.font.size and p.style.font.size.pt) or (doc.styles['Normal'].font.size and doc.styles['Normal'].font.size.pt) or 12
        key = (bold, color, italic, size)
        if len(p.text) > 5:
            if key in d:
                d[key]['positions'].append(i)
            else:
                d[key] = {"bold": bold, "positions": [i], "size": size}

    for key in d:
        positions = d[key]['positions']
        d[key]['positions_for_splitting'] = simplify_positions(positions)
        d[key]['count'] = len(positions)
        d[key]['std'] = std(positions) / number_of_paragraphs

    # filter out styles that don't have an even distribution, which is defined as std of at least 25%
    d = dict([(key, value) for key, value in list(d.items()) if value['std'] > 0.25])

    for k in d:
        v = d[k]
        print("k:", k)
        print("\tcount:", v['count'])
        print("\tsize:", v['size'])
        print("\tstd:", v['std'])

    bolds = []
    counts = []
    sizes = []
    stds = []
    for key, value in list(d.items()):
        bolds.append(value['bold'])
        counts.append(value['count'])
        sizes.append(value['size'])
        stds.append(value['std'])

    bolds.sort()
    bolds.reverse()

    counts.sort()

    sizes.sort()
    sizes.reverse()

    stds.sort()
    stds.reverse()

    print("\n")
    print("bolds:", bolds)
    print("counts:", counts)
    print("sizes:", sizes)
    print("stds:", stds)

    for key, value in list(d.items()):
        # multiplying std by 0.9 makes it a little less important in tie-breaking situations
        value['score'] = bolds.index(value['bold']) + counts.index(value['count']) + stds.index(value['std']) * 0.9 + sizes.index(value['size'])

    styles = sorted(list(d.items()), key=lambda t: t[1]['score'])

    for level, (key, style) in enumerate(styles):
        print("\n\nlevel:", level)
        print("key:", key)
        print("bold:", style['bold'])
        print("count:", style['count'])
        print("positions:", style['positions'][:20])
        print("positions_for_splitting:", style['positions_for_splitting'][:20], "...")
        print("size:", style['size'])
        print("std:", style['std'])
        print("score:", style['score'])

    paragraphs = [p.text for p in paragraphs]
    #tree = split_by_indices(paragraphs, styles[0][1]['positions_for_splitting'])
    # xxx
    tree = split_by_indices(paragraphs, [p-1 for p in styles[0][1]['positions_for_splitting']])
    for key, style in styles[1:]:
        print("\nstyle:", key)
        #tree = [split_by_indices(item, style['positions_for_splitting'], offset=sum(map(count_nested_items, tree[0:i]))) for i, item in enumerate(tree)]
        # xxx
        tree = [split_by_indices(item, [p-1 for p in style['positions_for_splitting']], offset=sum(map(count_nested_items, tree[0:i]))) for i, item in enumerate(tree)]
        print("tree:", count_nested_items(tree))

    return tree

def extract_metadata_from_text(text):
    print("starting extract_metadata_from_text with ", len(text))

    pieces = get_pieces(text)

def get_pieces(text):
    print("starting get_pieces with", len(text))
    #paragraphs = [p for p in text.split("\n") if len(p) >= 3]
    paragraphs = [p for p in text.split("\n")]
    print("paragraphs = ", paragraphs[:10])
    number_of_docs = len(paragraphs)
    print("number_of_docs:", number_of_docs)

    # do tfidf
    document_counts = Counter()
    term_counts = []
    #term_frequencies = []
    pieces = []
    for paragraph in paragraphs:
        piece = {}
        piece['text'] = paragraph
        piece['tokens'] = tokens = tokenize(paragraph)
        document_counts.update(set(tokens))
        piece['number_of_tokens'] = number_of_tokens = len(tokens)
        piece['term_count'] = term_count = Counter(tokens)
        term_counts.append(term_count)
        piece['term_frequency'] = dict([ ( token, float(count) / number_of_tokens ) for token, count in list(term_count.items()) ])
        pieces.append(piece)

    document_frequency = dict([(term, float(count) / number_of_docs) for term, count in list(document_counts.items())])
    #print "document_frequency:", document_frequency

    for piece in pieces:
        piece['term_tfidf'] = term_tfidf = sorted([(term, float(frequency) / document_frequency[term]) for term, frequency in list(piece['term_frequency'].items())], key=lambda t: -1*t[1])
        piece['terms'] = [term for term, tfidf in term_tfidf]

    pieces = pieces

    def find_and_merge_pieces(pieces):
        #print "starting find_and_merge_pieces with", len(pieces), "pieces"
        print(".", end=' ')
        #raw_input("press enter to continue")
        number_of_pieces = len(pieces)
        for i in range(number_of_pieces):
            current_piece = pieces[i]
            #print "current_piece:", current_piece
            #raw_input("press enter to continue")
            if i == 0:
                previous_piece = current_piece
            else:
                #print "\n\n"
                #print "text:", current_piece['text']
                #print "terms:", current_piece['terms']
                combined_term_count = previous_piece['term_count'] + current_piece['term_count']
                #print "combined_term_count:", combined_term_count
                number_of_terms = sum(combined_term_count.values())
                combined_term_frequency = dict([(term, float(count) / number_of_terms) for term, count in list(combined_term_count.items())])
                combined_term_tfidf = sorted([(term, float(frequency) / document_frequency[term]) for term, frequency in list(combined_term_frequency.items())], key=lambda t: -1*t[1])
                combined_terms = [term for term, tfidf in combined_term_tfidf]
                number_of_combined_terms = len(combined_terms)
                #print "previous_terms = ", previous_piece['terms']
                #print "combined terms are", combined_terms
                #print "x:", [term in combined_terms[:8] for term in previous_piece['terms'][:4]]
                number_of_previous_terms = len(previous_piece['terms'])
                if number_of_previous_terms == 0 or float(sum([term in combined_terms[:4] for term in previous_piece['terms'][:3]])) / len(previous_piece['terms']) >= 0.75:
                    #print "merging"
                    previous_piece['text'] += "\n" + current_piece['text']
                    previous_piece['term_count'] = combined_term_count
                    previous_piece['term_frequency'] = combined_term_frequency
                    previous_piece['terms'] = combined_terms
                    previous_piece['term_tfidf'] = combined_term_tfidf
                    pieces.remove(current_piece) # only specific one or all similar
                    return True

                previous_piece = current_piece
    while find_and_merge_pieces(pieces):
        pass

    pieces.reverse()
    print("len(pieces):", len(pieces))

    while find_and_merge_pieces(pieces):
        pass

    pieces.reverse()
    print("len(pieces):", len(pieces))

    with open("/tmp/output.txt", "wb") as f:
        f.write(("\n" + "=" * 25 + "\n").join([p['text'].encode("utf-8") for p in pieces]))


from objects.accessories import accessory_collection
from objects.enchantments import enchantment_collection

class Player:
    player_number = 0
    shop_name = 'Player'
    potions = 0
    extra_dice_available = [4, 3, 3, 0, 0, 0]
    material_mastery_marks = 0
    energy_mastery_marks = 0
    affixes = {}
    completed_affixes = []
    sponsored_adventurer = None
    adventurer_orders_completed = 0
    completed_orders = []
    crafting_points = 0
    material_mastery_points = 0
    research_points = 0
    energy_mastery_points = 0
    order_points = 0
    charm_bonus = 0
    adventurer_points = 0
    total_points = 0

    def __init__(self, player_number, adventurer, name=None):
        self.player_number = player_number
        self.shop_name = name or f'Shop {player_number}'
        self.sponsored_adventurer = adventurer
        self.affixes.update(accessory_collection)
        self.affixes.update(enchantment_collection)

    def complete_order(self, order):
        self.completed_orders.append(order)
        self.order_points += order.value

    def check_mastery(self, mastery):
        result = False
        if mastery.material in ['steel', 'wood', 'leather']:
            result = self.check_material_mastery(mastery)
        elif mastery.material in ['enchantment', 'arcane', 'wild']:
            result = self.check_material_mastery(mastery)
        return result

    def check_material_mastery(self, mastery):
        if (self.material_mastery_marks >= mastery.marks) and self.material_mastery_points == 0:
            self.material_mastery_points = mastery.value
            return True
        return False

    def check_energy_mastery(self, mastery):
        if (self.energy_mastery_marks >= mastery.marks) and self.energy_mastery_points == 0:
            self.energy_mastery_points = mastery.value
            return True
        return False

    def check_order_completion(self, order):
        for affix in order.affixes:
            if affix in self.completed_affixes:
                continue
            else:
                return False
        self.complete_order(order)
        return True

    def check_adventurer_orders(self):
        # jaaaaanky, try to improve
        completions = 0
        if self.sponsored_adventurer.enchantment in self.completed_affixes:
            for item in self.sponsored_adventurer.items:
                if item in self.completed_affixes:
                    # self.sponsored_adventurer.items.pop(item)
                    completions += 1
        new_completions = completions - self.adventurer_orders_completed
        while new_completions < 0:
            if (new_completions + self.adventurer_orders_completed) == 1:
                self.potions += 3
            elif (new_completions + self.adventurer_orders_completed) == 2:
                # player gets to mark 1 circle
                pass
            elif (new_completions + self.adventurer_orders_completed) == 3:
                # player gets to mak 1 circle
                self.adventurer_points = self.sponsored_adventurer.value
            new_completions -= 1
        self.adventurer_orders_completed = completions

    def mark_requirement_complete(self, die, affix_name, material):
        if self.affixes[affix_name].affix_type == 'physical':
            if die.value >= self.affixes[affix_name].requirements[material]['target']:
                self.affixes[affix_name].requirements[material]['completed'] = True
        if self.affixes[affix_name].affix_type == 'magical':
            if die.value <= self.affixes[affix_name].requirements[material]['target']:
                self.affixes[affix_name].requirements[material]['completed'] = True
        completed = True
        for req in self.affixes[affix_name].requirements.values():
            if not req['completed']:
                completed = False
        self.affixes[affix_name].completed = completed
        if completed:
            self.completed_affixes.append(affix_name)


#constants

NOT_EVALUATED = -1; #DO NOT CHANGE, (only change after modifying how genes are created in optimEnDec)
NONE = -1; #DO NOT CHANGE, (only change after modifying analysisKit.py)
YES = 1;
NO = 1;

#-node types
INPUT_NODE = 1;
HIDDEN_NODE = 2;
OUTPUT_NODE = 3;

#connection types
RANDOM = 1;
FULLY_CONNECTED = 2;

#-problem types
REGRESSION = 1;
CLASSIFICATION = 2;

#-node functions
IDENTITY = 1;
SIGMOID = 2;
GAUSSIAN = 3;
TANH = 4;
GAUSSIAN_II = 5;
PROB_SIGMOID = 6;
THIN_PLATE_SPLINE =7;
ARC_TAN = 8;

#-weight functions
INNER_PROD = 1;
EUCLID_DIST = 2;
HIGHER_ORDER_PROD = 3;
HIGHER_ORDER_SUB = 4;
STD_DEV = 5;
MIN = 6;
MAX = 7;
MANHATTAN_DIST = 8;
MAX_DIST = 9;
MEAN = 10;

#-status
ACTIVE = 1;
INACTIVE = 0;

#-count of meta-information genes
AGE_GENE = 3;
COST_GENE = 0;
COST2_GENE = 1;
MISC_GENE = 2;
META_INFO_COUNT = 4;

#-count of transfer function details (per node)
TRANS_INFO_COUNT = 5; #weightfn, nodefn, and 3 fnparameters

#Ensembles Selection methods
TOP_N = 1;
NON_DOMINATED_FRONT = 2;
GREEDY_SELECT = 3;


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="cfcompare", # Replace with your own username
    version="1.2.0",
    author="Harry Singh",
    author_email="hsing247@uwo.ca",
    description="Tool to compare CF Standard Name versions.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    download_url="https://github.com/HarrySng/cfsntool/archive/v1.0.0.tar.gz",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.7',
)


# Users.urls
from django.urls import path

from users import views

urlpatterns = [

    path(
        route='login/',
        view=views.LoginView.as_view(),
        name='login'
    ),

    path(
        route='logout/',
        view=views.LogoutView.as_view(),
        name='logout'
    ),

    path(
        route='singup/',
        view=views.SingupView.as_view(),
        name='singup'
    ),

    path(
        route='me/profile/',
        view=views.UpdateProfileView.as_view(),
        name='update_profile'
    ),

    path(
        route='<str:username>/',
        view=views.UserDetailView.as_view(
            template_name='users/detail.html'
        ),
        name='detail'
    ),

]


"""
errors.py

This module defines custom error handling for the django-graphene-jwt-oauth2 library,
including a custom exception and an HttpResponse subclass to obscure error
details in non-debug mode.

Classes:
- ObscureException: Custom exception class with an optional message.

Functions:
- ObscureHttpResponse: Function to create an HTTP response with obscured error details.

Variables:
- DEFAULT_ERROR_MESSAGE: Default error message used in non-debug mode.
- DEFAULT_STATUS_CODE: Default status code for obscured HTTP responses.
"""

from django.conf import settings
from django.http import HttpResponse

# Constants for default error message and status code
DEFAULT_ERROR_MESSAGE: str = "Unauthorized"
DEFAULT_STATUS_CODE: int = 401

def ObscureHttpResponse(  # pylint: disable=C0103
    reason: str = DEFAULT_ERROR_MESSAGE, code: int = DEFAULT_STATUS_CODE
) -> HttpResponse:
    """
    Create an HTTP response with the provided reason and status code or defaults,
    obscuring the error details in non-debug mode.

    :param reason: The reason for the HTTP response.
    :param code: The HTTP status code.
    :return: The HTTP response with obscured details.
    """
    return HttpResponse(
        reason if settings.DEBUG else DEFAULT_ERROR_MESSAGE, status=code
    )

class ObscureException(Exception):
    """
    Custom exception class with an optional message.

    This exception class is designed to obscure error details in non-debug mode,
    providing a generic error message.

    :param message: The exception message.
    """

    def __init__(self, message: str = DEFAULT_ERROR_MESSAGE):
        super().__init__(message if settings.DEBUG else DEFAULT_ERROR_MESSAGE)


"""Convert length in ft to in, yards & miles"""

# This script will convert length in ft to inches, yards & miles.
# Length would be accepted from user.

# Import modules.

import sys
import clear_screen_module

# Define functions.

def convert_ft_to_mi(len_ft):
    """Function to convert length in ft to miles"""
    ans = len_ft * 0.000189
    # round() helps round off the ans to 2 decimal spaces only.
    return round(ans,2)

def convert_ft_to_yd(len_ft):
    """Function to convert length in ft to yards"""
    ans = len_ft * 0.33
    return round(ans,2)

def convert_ft_to_in(len_ft):
    """Function to convert length in ft to inches"""
    ans = len_ft * 12
    return round(ans,2)

def main():
    """First function to be called"""
    # Clear screen using module function.
    clear_screen_module.clear_screen()
    print("This script will convert length in ft to inches, yards & miles.\n")
    # Accept numeric value only for length from user using try-except.
    try:
        user_len = eval(input("Enter the length: "))
    except NameError:
        print("The script accepts only numeric value for length. Exiting script!")
        sys.exit(1)
    len_in = convert_ft_to_in(user_len)
    print(f"\n{user_len} ft of length = {len_in} inches")
    len_yd = convert_ft_to_yd(user_len)
    print(f"{user_len} ft of length = {len_yd} yards")
    len_mi = convert_ft_to_mi(user_len)
    print(f"{user_len} ft of length = {len_mi} miles\n")
    return None

# Call main function when this script is executed.

if __name__ == "__main__":
    main()


import os
import pandas as pd
from settings import PATH_SAVE_PREDICTIONS
PATH_CREDIT_CARD_DATASET = "dao/data/"

def load_creditcard_dataset():
    try:
        df = pd.read_csv(f'{PATH_CREDIT_CARD_DATASET}creditcard.csv')
        df_nonevent = df.loc[df['Class']==0].sample(1500)
        df_event = df.loc[df['Class']==1].sample(250)
        return df_nonevent.append(df_event)
    except Exception as e:
        raise Exception(e)

def write_predictions(df_predictions: pd.DataFrame):
    file = f'creditcard_predictions.csv'
    try:
        if os.path.exists(PATH_SAVE_PREDICTIONS) == False:
            os.mkdir(PATH_SAVE_PREDICTIONS)
            df_predictions.to_csv(
                f'{PATH_SAVE_PREDICTIONS}{file}',
                index=False,
                decimal='.',
                sep=',',
                mode='w'
            )
        else:
            df_predictions.to_csv(
                f'{PATH_SAVE_PREDICTIONS}{file}',
                index=False,
                decimal='.',
                sep=',',
                mode='a',
                header=False
            )

    except Exception as e:
        raise Exception(e)


import heapq

class Solution(object):
  def trapRainWater(self, heightMap):
    """
    :type heightMap: List[List[int]]
    :rtype: int
    """
    n = len(heightMap)
    if n < 3:
      return 0
    m = len(heightMap[0])
    if m < 3:
      return 0
    h = []
    s = set()
    for j in range(m):
      h.append((heightMap[0][j], 0, j))
      h.append((heightMap[n-1][j], n-1, j))
      s.add((0, j))
      s.add((n-1, j))
    for i in range(1, n - 1):
      h.append((heightMap[i][0], i, 0))
      h.append((heightMap[i][m-1], i, m-1))
      s.add((i, 0))
      s.add((i, m-1))
    heapq.heapify(h)
    ret = [0]
    while h:
      height, i, j = heapq.heappop(h)
      def foo(i, j):
        if not (0 <= i < n and 0 <= j < m and (i, j) not in s):
          return
        s.add((i, j))
        if heightMap[i][j] > height:
          heapq.heappush(h, (heightMap[i][j], i, j))
          return
        ret[0] += height - heightMap[i][j]
        foo(i+1, j)
        foo(i-1, j)
        foo(i, j+1)
        foo(i, j-1)
      foo(i+1, j)
      foo(i-1, j)
      foo(i, j+1)
      foo(i, j-1)
    return ret[0]


import os
import pygame
# Mac 에서는 pygame 1.9.6 으로 실행 시 창이 제대로 실행되지 않는 이슈가 있음
# 2.0.0.dev6 버전으로 설치해서 진행 함. (python3 -m pip install pygame==2.0.0.dev6)

# initialization (Must)
pygame.init()

# Resolution
screen_width = 640
screen_height = 480
screen = pygame.display.set_mode((screen_width, screen_height))

# Game Title
pygame.display.set_caption("Project Games")

# FPS
clock = pygame.time.Clock()

# Path Setting
current_path = os.path.dirname(__file__)
image_path = os.path.join(current_path, "images")

# background image loading
background = pygame.image.load(os.path.join(image_path, "background.png"))
stage = pygame.image.load(os.path.join(image_path, "stage.png"))
stage_size = stage.get_rect().size  # 가로, 세로 크기를 구함
stage_height = stage_size[1]        # 0번째는 가로, 1번째는 세로

# character loading (stripe)
character = pygame.image.load(os.path.join(image_path, "character.png"))
character_size = character.get_rect().size
character_width = character_size[0]
character_height = character_size[1]
character_x_pos = (screen_width / 2) - (character_width / 2)
character_y_pos = screen_height - stage_height - character_height

# character moving variables
character_speed = 0.3
character_to_x_LEFT = 0
character_to_x_RIGHT = 0
# character_to_y_UP = 0
# character_to_y_DOWN = 0

# weapon loading
weapon = pygame.image.load(os.path.join(image_path, "weapon.png"))
weapon_size = weapon.get_rect().size
weapon_width = weapon_size[0]

# weapon fires multiple
weapons = []
weapon_speed = 10

# character moving speed

# Enemy loading & moving

# set Text Font

# Game Time

# Calculate Times

# Event Loop
running = True
while running:
    # set FPS
    dt = clock.tick(30)
    # print("FPS= "+str(clock.get_fps()))     # print frames on running console

    for event in pygame.event.get():
        if event.type == pygame.QUIT:  # event = Windows Closed
            running = False

        # Keyboard event : key pressed
        if event.type == pygame.KEYDOWN:  # WHEN USER press KEYS,
            if event.key == pygame.K_LEFT:  # move character to left
                character_to_x_LEFT -= character_speed  # calculate x to left by 2
            elif event.key == pygame.K_RIGHT:  # move character to right
                character_to_x_RIGHT += character_speed
            # elif event.key == pygame.K_UP:              # move character to up
            #     character_to_y_UP -= character_speed
            # elif event.key == pygame.K_DOWN:            # move character to down
            #     character_to_y_DOWN += character_speed
            elif event.key == pygame.K_SPACE:
                weapon_x_pos = (character_x_pos + character_width / 2) - (weapon_width /2)
                weapon_y_pos = character_y_pos
                weapons.append([weapon_x_pos, weapon_y_pos])

        # Keyboard event : key Released
        if event.type == pygame.KEYUP:  # WHEN USER release KEYS,
            if event.key == pygame.K_LEFT:
                character_to_x_LEFT = 0
            if event.key == pygame.K_RIGHT:
                character_to_x_RIGHT = 0
            # elif event.key == pygame.K_UP or pygame.K_DOWN:
            #   to_y = 0
            # 이 경우에는 up/down 을 누른 후 해제가 되지 않는다
            # 왜냐하면, up/down을 떼었을 때, 이미 left/right도 떼어진 상태이므로
            # 위의 left/right의 if문을 받지, elif로 내려오지 않기 때문이다. 그래서 아래처럼 별개로 만들어줘야 한다.
        # if event.type == pygame.KEYUP:
        #     if event.key == pygame.K_UP:
        #         character_to_y_UP = 0
        #     if event.key == pygame.K_DOWN:
        #         character_to_y_DOWN = 0
        if event.type == pygame.KEYUP:
            if event.key == pygame.K_SPACE:
                pass

    # set character moving coordinate * frame rate
    character_x_pos += (character_to_x_LEFT + character_to_x_RIGHT) * dt
    # character_y_pos += (character_to_y_UP + character_to_y_DOWN) * dt

    # weapon moving coordinate
    weapons = [[w[0], w[1] - weapon_speed] for w in weapons]
    # weapon remove when y < 0:
    weapons = [[w[0], w[1]] for w in weapons if w[1] > 0]
    # = if y < 0 = DO NOT into w[1] = no list data = remove

    # set character moving Limit on screen
    if character_x_pos < 0:
        character_x_pos = 0
    elif character_x_pos > screen_width - character_width:
        character_x_pos = screen_width - character_width
    # if character_y_pos < 0:
    #     character_y_pos = 0
    # elif character_y_pos > screen_height - stage_height - character_height:
    #     character_y_pos = screen_height - stage_height - character_height

    # Collision coordinate update
    character_rect = character.get_rect()
    character_rect.left = character_x_pos
    character_rect.top = character_y_pos

    # enemy_rect = enemy.get_rect()
    # enemy_rect.left = enemy_x_pos
    # enemy_rect.top = enemy_y_pos

    # Check Collision Coordinate
    # if character_rect.colliderect(enemy_rect):
    #     print("Game Over")
    #     running = False

    # blit background and character
    screen.blit(background, (0, 0))
    for weapon_x_pos, weapon_y_pos in weapons:
        screen.blit(weapon, (weapon_x_pos, weapon_y_pos))
    screen.blit(stage, (0, screen_height - stage_height))
    screen.blit(character, (character_x_pos, character_y_pos))

    # Timer insert by text
    # elapsed_time = (pygame.time.get_ticks() - start_ticks) / 1000
    # timer = game_font.render(str(int(total_time - elapsed_time)), True, (255, 255, 0))
    # screen.blit(timer, (15, 15))

    # if total_time - elapsed_time < 0:
    #     running = False
    #     print("Game Over")

    # screen update every frame (MUST)
    pygame.display.update()
# pygame End
pygame.quit()


def find_nth(n, d):
    # Edge Case(s)
    if n > len(d):
        return -1
    count = 1
    cur = float("-inf")
    for k, v in sorted(d.items(), key = lambda x: x[1]):
        if count == n:
            return k
        if v > cur:
            count += 1
            cur = v
    return -1


import re

from streamlink.plugin import Plugin
from streamlink.plugin.api import useragents, validate
from streamlink.stream import HLSStream
from streamlink.utils import parse_json, update_scheme

class IDF1(Plugin):
    DACAST_API_URL = 'https://json.dacast.com/b/{}/{}/{}'
    DACAST_TOKEN_URL = 'https://services.dacast.com/token/i/b/{}/{}/{}'

    _url_re = re.compile(r'https?://www\.idf1\.fr/(videos/[^/]+/[^/]+\.html|live\b)')
    _video_id_re = re.compile(r"""
            dacast\('(?P<broadcaster_id>\d+)_(?P<video_type>[a-z]+)_(?P<video_id>\d+)',\s*'replay_content',\s*data\);
        """, re.VERBOSE)
    _video_id_alt_re = re.compile(r'''
            <script\s+
                src="https://player.dacast.com/js/player.js"\s+
                id="(?P<broadcaster_id>\d+)_(?P<video_type>[cf])_(?P<video_id>\d+)"
        ''', re.VERBOSE)
    _player_url = 'http://ssl.p.jwpcdn.com/player/v/7.12.6/jwplayer.flash.swf'

    _api_schema = validate.Schema(
        validate.transform(parse_json),
        {
            validate.optional('html5'): validate.all(
                [
                    {
                        'src': validate.url()
                    },
                ],
            ),
            'hls': validate.url(),
            'hds': validate.url()
        },
        validate.transform(
            lambda x: [update_scheme(IDF1.DACAST_API_URL, x['hls']), x['hds']] + [y['src'] for y in x.get('html5', [])]
        )
    )

    _token_schema = validate.Schema(
        validate.transform(parse_json),
        {'token': validate.text},
        validate.get('token')
    )

    _user_agent = useragents.IE_11

    @classmethod
    def can_handle_url(cls, url):
        return IDF1._url_re.match(url)

    def _get_streams(self):
        res = self.session.http.get(self.url)
        match = self._video_id_re.search(res.text) or self._video_id_alt_re.search(res.text)
        if match is None:
            return
        broadcaster_id = match.group('broadcaster_id')
        video_type = match.group('video_type')
        video_id = match.group('video_id')

        videos = self.session.http.get(
            self.DACAST_API_URL.format(broadcaster_id, video_type, video_id),
            schema=self._api_schema
        )
        token = self.session.http.get(
            self.DACAST_TOKEN_URL.format(broadcaster_id, video_type, video_id),
            schema=self._token_schema,
            headers={'referer': self.url}
        )
        parsed = []

        for video_url in videos:
            video_url += token

            # Ignore duplicate video URLs
            if video_url in parsed:
                continue
            parsed.append(video_url)

            # Ignore HDS streams (broken)
            if '.m3u8' in video_url:
                yield from HLSStream.parse_variant_playlist(self.session, video_url).items()

__plugin__ = IDF1


from errno import EHOSTUNREACH
from scripts.helpful_scripts import LOCAL_BLOCKCHAIN_ENVIRONMENTS, get_account

#Brownie interacts with the solidity files
from brownie import (
    GovernorContract,
    GovernanceToken,
    GovernanceTimeLock,
    Treasury,
    Contract,
    config,
    network,
    accounts,
    chain,
)
from web3 import Web3, constants
import time

def transfer(amount, account, account_to_receive):
    token = GovernanceToken.deploy({'from': account})
    token.transfer(account_to_receive, amount, {'from': account})

def mint_and_give(amount, account_to_give, treasury):
    #mint the token by the amount
    token = GovernanceToken.deploy("MICASA", "MCS", amount, {'from': treasury})

    token.transfer(account_to_give, amount, {'from': treasury})

def main():
    ACCOUNT = get_account()

    TREASURY = Treasury.address

    #In gwei
    DEPOSIT_AMOUNT = int(input("Enter Deposit Amount [In Gwei]: "))

    #For now, we do a one-to-one convertion with our MICASA token

    #Transfer Gwei to Treasury
    transfer(DEPOSIT_AMOUNT, ACCOUNT, TREASURY)

    #Give MICASA token to account
    mint_and_give(DEPOSIT_AMOUNT, ACCOUNT, TREASURY)
    print(f"received {DEPOSIT_AMOUNT} MICASA")