# encoding: utf-8

import argparse
import os
import sys
import torch

from torch.backends import cudnn

sys.path.append('.')
from config import cfg
from data import make_data_loader
from engine.trainer import do_train, do_train_with_center
from modeling import build_model
from layers import make_loss, make_loss_with_center
from solver import make_optimizer, make_optimizer_with_center, WarmupMultiStepLR

from utils.logger import setup_logger

def train(cfg):
    # prepare dataset
    train_loader, val_loader, num_query, num_classes, clustering_loader = make_data_loader(cfg)

    # prepare model
    model = build_model(cfg, num_classes)

    if cfg.MODEL.IF_WITH_CENTER == 'on':
        loss_func, center_criterion_part, center_criterion_global, center_criterion_fore = make_loss_with_center(cfg, num_classes)
        optimizer, optimizer_center = make_optimizer_with_center(cfg, model, center_criterion_part, center_criterion_global, center_criterion_fore)
    else:
        loss_func = make_loss(cfg, num_classes)
        optimizer = make_optimizer(cfg, model)

    # Add for using self trained model
    if cfg.MODEL.PRETRAIN_CHOICE == 'imagenet':
        start_epoch = 0
        scheduler = WarmupMultiStepLR(optimizer, cfg.SOLVER.STEPS, cfg.SOLVER.GAMMA, cfg.SOLVER.WARMUP_FACTOR,
                                      cfg.SOLVER.WARMUP_ITERS, cfg.SOLVER.WARMUP_METHOD)
    else:
        print('Only support pretrain_choice for imagenet, but got {}'.format(cfg.MODEL.PRETRAIN_CHOICE))

    if cfg.MODEL.IF_WITH_CENTER == 'on':
        do_train_with_center(
            cfg,
            model,
            center_criterion_part,
            center_criterion_global,
            center_criterion_fore,
            train_loader,
            val_loader,
            optimizer,
            optimizer_center,
            scheduler,      # modify for using self trained model
            loss_func,
            num_query,
            start_epoch,     # add for using self trained model
            clustering_loader
        )
    else:
        do_train(
            cfg,
            model,
            train_loader,
            val_loader,
            optimizer,
            scheduler,      # modify for using self trained model
            loss_func,
            num_query,
            start_epoch,     # add for using self trained model
            clustering_loader
        )

def main():
    parser = argparse.ArgumentParser(description="ReID Baseline Training")
    parser.add_argument(
        "--config_file", default="", help="path to config file", type=str
    )
    parser.add_argument("opts", help="Modify config options using the command-line", default=None,
                        nargs=argparse.REMAINDER)

    args = parser.parse_args()

    num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1

    if args.config_file != "":
        cfg.merge_from_file(args.config_file)
    cfg.merge_from_list(args.opts)
    cfg.freeze()

    output_dir = cfg.OUTPUT_DIR
    if output_dir and not os.path.exists(output_dir):
        os.makedirs(output_dir)

    logger = setup_logger("reid_baseline", output_dir, 0)
    logger.info("Using {} GPUS".format(num_gpus))
    logger.info(args)

    if args.config_file != "":
        logger.info("Loaded configuration file {}".format(args.config_file))
        with open(args.config_file, 'r') as cf:
            config_str = "\n" + cf.read()
            logger.info(config_str)
    logger.info("Running with config:\n{}".format(cfg))

    if cfg.MODEL.DEVICE == "cuda":
        os.environ['CUDA_VISIBLE_DEVICES'] = cfg.MODEL.DEVICE_ID    # new add by gu
    cudnn.benchmark = True
    train(cfg)

if __name__ == '__main__':
    main()


import cv2
from cv2 import HOUGH_GRADIENT
import numpy as np
from matplotlib import pyplot as plt

sct_img = cv2.imread('coins.png')

def nothing(x):
    pass

H = 103
S = 255
V = 255
Hl = 0
Sl = 191
Vl = 119

while True:

    hsv = cv2.cvtColor(sct_img, cv2.COLOR_RGB2HSV)

    cv2.imshow('image',hsv)

    lower_blue = np.array([Hl,Sl,Vl])
    upper_blue = np.array([H,S,V])

    mask = cv2.inRange(hsv, lower_blue, upper_blue)

    # Find the circle blobs on the binary mask:
    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    # Use a list to store the center and radius of the target circles:
    detectedCircles = []

    for i, c in enumerate(contours):

        # Approximate the contour to a circle:
        (x, y), radius = cv2.minEnclosingCircle(c)

        # Compute the center and radius:
        center = (int(x), int(y))
        radius = int(radius)

        if radius > 20 and radius < 40:
            # Draw the circles:
            cv2.circle(sct_img, center, radius, (0, 0, 255), 2)
            cv2.rectangle(sct_img, (center[0] - 5, center[1] - 5), (center[0] + 5, center[1] + 5), (0, 128, 255), -1)

            # Store the center and radius:

            detectedCircles.append([center, radius])


#Fonksiyon okur yazarlığı

# =============================================================================
# bir fonksiyon tanımlamak için "def" kullanılır
# =============================================================================

def kare_al(x) :
    print(x**2)

kare_al(5)

#Bilgi notu ile çıktı almak

def kare_al2(x):
    print( "girilen sayımız:" ,x, "\n"
        "girilen sayının karesi:", x**2)

kare_al2(4)

# İki argümanlı fonksiyon

def carpma(x,y) :
    a = (x**2) * y**(-3/5)
    return a  # Return kullanırken () içinde almaya gerek yok ama alınabilirde

carpma(5, 3)

#Argümanların sırası bilinmesede o argümanı atıyarak fonksiyon içinde işlem yapılabilir
carpma(y = 5, x = 3) # x ve y nin yeri kasıtlı değişti

# =============================================================================
# Local ve Global değişkenler
# Local değişkenler : fonksiyon(def) içindeki değişkenlerdir ve fonksiyon dışına etki etmezler
# Global değişkenler: bizim enviromentimizde bulunan atanmışlardır ve bunlarda fonksiyon(def) içine
# etki etmezler
#
# =============================================================================

# Local etki alanından Global etki alanını değiştirmek

x = []

def eleman_ekleme(y) :
    x.append(y)
    return("eklenen ifade", y,"\n",
           "yeni liste :", x)

eleman_ekleme("ali")

x

#True False sorguları

hudut = " edirne"
hudut == "kars"

sayı = 100000
sayı == 12345 # Çift eşittir(==) sorgu oluşturur

#İf

sinir = 5000
gelir = 40000

if gelir < sinir :
    print("Gelir sınırdan küçük")
    gelir = gelir*2

# Else

if gelir < sinir :
    gelir = gelir*2
else :
    gelir = gelir / 2

# Elif 2 den fazla durumlar için

gelir1 = 5000
gelir2 = 6000
gelir3 = 3500
sınır = 5000

if gelir1 > sinir :
    print("tebrikler başardınız")
elif gelir1 < sinir :
    print("ceza aldınız")
else :
    print("takibe devam")

# Mini uygulama

sınır = (10**4)*5
magza_adi = input("mağza adını giriniz:")
gelir = int( input("Geliri giriniz:"))

if gelir > sınır :
    print("Promosyon kazandın ",magza_adi,"hadi iyisin")
elif gelir < sınır :
    print("UYARI!")
else:
    print("takibe devamke")

# For loop :)

ogrenciler = ["ali","veli","ayse","beren"]

for i in ogrenciler:
     print(i)

print(ogrenciler[0:4])

# For döngüsü örnek
maaslar = [1000,2000,3000,4000,5000]

for i in range(len(maaslar)):
    if 4000 > maaslar[i]:
        maaslar[i] += 1500
    elif maaslar[i] > 4000:
        maaslar[i] += 750
    print(maaslar[i])

# daha basiti
maaslar = [1000,2000,3000,4000,5000]

for i in maaslar:
    if 4000 > i:
        i += 1500
    elif i > 4000:
        i += 750
    print(i)

# R gibi değil burda i in veri dediğimizde direkt i o elemanı temsil ediyor
# bundan dolayı R da veri[i] derken burda i dememiz yetiyor


#Условие задачи - автоматизирйте расчёт квадрата Пифагора по дате рождения.

#Алгоритм расчёта квадрата Пифагора на конкретном примере (если дата рождения - 16 октября 1991 года), выглядит следующим образом:
#1.Выпишите цифры дня и месяца рождения: 1610. Сложите цифры, получится первое число: 1+6+1+0 = 8.
#2.Точно также высчитайте сумму цифр года рождения: 1+9+9+1 = 20. Получили второе число.
#3.Рассчитайте сумму двух первых, получившихся в результате расчёта, чисел: 8+20 = 28. Это первое рабочее число.
#4.Далее найдите сумму цифр первого рабочего числа: 2+8 = 10. Это второе рабочее число.
#5.Из первого рабочего числа вычитайте умноженную вдвое первую цифру даты рождения: 28-2*1 = 26. Это третье рабочее число.
#6.И, наконец, сложите цифры третьего рабочего числа: 2+6=8. В итоге получаем четвёртое рабочее число.

#Решение задачи:

#Используем квадрат Пифагора для определения характера человека.
#Для начала узнаем дату рождения:
birth_day = input("Введите день своего рождения: ")
birth_month = input("Введите месяц своего рождения: ")
birth_year = input("Введите год своего рождения: ")

#Далее действуем согласно алгоритма: суммируем числа дня и месяца рождения.

x = 0
for i in birth_day:
      x += (int(i))
y = 0
for l in birth_month:
      y += int(l)

#Суммируем числа, входящие в год рождения:

z = 0
for k in birth_year:
      z += int(k)

#Выведем 1 (первое) число.
first_n = x + y

#Выведем 2 число.
second_n = z

#Получим первое рабочее число.
work_n1 = first_n + second_n
work_n1_test = work_n1 - 9
if work_n1_test <= 0:
    print("Первое число:", "0", work_n1)
else:
    print("Первое число:", work_n1)

#Выведем второе рабочее число.
sum = 0
work_n1 = str(work_n1)
for n in work_n1:
      sum += int(n)
sum_test = sum - 9
work_n2 = sum
if sum_test <= 0:
   print("Второе число:", "0", work_n2)
else:
    print("Второе число:", work_n2)

#Выведем третье рабочее число.
birth_day = int(birth_day)
add = birth_day // 10
birth_day = str(birth_day)
work_n1 = int(work_n1)
work_n3 = work_n1 - add * 2
work_n3_test = work_n3 - 9
if work_n3_test <= 0:
   print("Третье число:", "0", work_n3)
else:
    print("Третье число:", work_n3)

#Выведем четвертое рабочее число.
sum = 0
work_n3 = str(work_n3)
for p in work_n3:
      sum += int(p)
      work_n3 = work_n1 - add * 2
sum5_test = sum - 9
work_n4 = sum
if sum5_test <= 0:
    print("Четвертое число:", "0", work_n4)
else:
    print("Четвертое число:", work_n4)


from ExtendDT import ext_dt
from Sequence import *

#> Find the optimal sequence for a single node when its children are optimized
def PruneOptimalSingle(xdt, main_seq, root_id):
    current_seq = copy.copy(main_seq[root_id])
    #* if optimized is not available, make it
    if  not current_seq.optimized:
        #* Begin of optimization procedure ==============================
        print(f"[{root_id} (d:{xdt.node_depth[root_id]})] | Opting ===>")
        # * check this is a leaf
        if xdt.is_leaf[root_id]:
            combined_seq = PruneSequence()
            combined_seq.OptFinished()
            current_seq = combined_seq
        else:
            #* Find children IDs
            left_root = xdt.children[root_id][0]
            right_root = xdt.children[root_id][1]
            #* Get optimal sequence of children
            lef_seq, node_id  = PruneOptimalSingle(xdt, main_seq, left_root)
            right_seq, node_id  = PruneOptimalSingle(xdt, main_seq, right_root)
            #* Combine children sequences with each other
            combined_seq = CombineSequence(lef_seq, right_seq, root_id)
            # * Append combined seq to the sequence of root
            current_seq.AppendSequence(combined_seq)
            #+ Tuple of changing root into a leaf
            current_tuple = [xdt.abs_cost[root_id], xdt.miss_class[root_id], [root_id]]
            #* Append this to the sequence
            current_seq.AppendTuple(current_tuple)
            #* Set register to finished
            current_seq.OptFinished()
        #* End of optimization procedure ==============================
        print(f"[{root_id}] ===> Ready |")
    return current_seq, root_id

#> Function for finding the optimal sequence of a node recursively
def PruneOptimal(xdt, goal_node):
    #* Get depth list
    xdt.getNodeDepth()
    depth_list = xdt.nodes_in_depth
    #* get the cost factors
    xdt.getAbsCostFactor()
    #* Find the children relations
    xdt.getChildren()
    #* Make the empty sequence for all
    main_seq = [PruneSequence()] * xdt.n_all_nodes
    #* Go through all depth from bottom to top
    for depth_row in range(len(depth_list)-1, -1, -1):
        row_list = depth_list[depth_row]
        #* Go through nodes in this depth
        for node_in_list in row_list:
            current_seq, node_id = PruneOptimalSingle(xdt, main_seq, node_in_list)
            main_seq[node_in_list] = current_seq
            if node_in_list == goal_node:
                break
        if node_in_list == goal_node:
            break
    return main_seq[goal_node]

#> Function for finding the optimal sequence of a node recursively
def PruneOptimalParallel(xdt, goal_node):
    #* Load multi-processor pckg
    import multiprocessing as mp
    #* Get depth list
    xdt.getNodeDepth()
    depth_list = xdt.nodes_in_depth
    #* get the cost factors
    xdt.getAbsCostFactor()
    #* Find the children relations
    xdt.getChildren()
    #* Make the empty sequence for all
    main_seq = [PruneSequence()] * xdt.n_all_nodes
    #+ Is goal reached
    goal_reached = False
    #* Start the pool
    num_processor = mp.cpu_count()
    pool = mp.Pool(processes = int(num_processor/2))
    #* Go through all depth from bottom to top
    for depth_row in range(len(depth_list)-1, -1, -1):
        #* Go through nodes in this depth
        row_list = depth_list[depth_row]
        #* Run the pool in this depth
        result_async = [pool.apply_async(PruneOptimalSingle, args = (xdt, main_seq, node_in_list)) for node_in_list in row_list]
        #* Get the pool results
        for worker_idx in result_async:
            current_seq = worker_idx.get()[0]
            current_node = worker_idx.get()[1]
            main_seq[current_node] = current_seq
            if current_node == goal_node:
                goal_reached = True
        if goal_reached:
            break
    pool.close()
    # pool.join()
    return main_seq[goal_node]

#> A function to get the
def get_ORAP_sequence(dt_base, **kwargs):
    # parse optional input
    use_parallel = False # by default use a single core
    for key, value in kwargs.items():
        # if use_parallel passed in set its value
        if key == "use_parallel":
            use_parallel = value
    #* Build the extended tree object
    xdt = ext_dt(dt_base)
    #* Calculate the optimal sequence for the root of the whole DT
    goal_node = 0
    #* Decide about number of processors
    if use_parallel:
        goal_seq = PruneOptimalParallel(xdt, goal_node)
    else:
        goal_seq = PruneOptimal(xdt, goal_node)
    #* Get the sequence for root (complete sequence)
    ORAP_base_sequence = copy.copy(goal_seq)
    #* Sort sequence according to the cost
    sort_factor = "cost"
    ORAP_base_sequence.SortSequence(sort_factor)
    #* Add the original tree to the sequence
    ORAP_base_sequence.cost.insert(0, 0)
    ORAP_base_sequence.miss.insert(0, 0)
    ORAP_base_sequence.removed.insert(0, [])

    return ORAP_base_sequence

#>A function to remove a series of nodes from a base DT
def series_prune(base_dt, node_series):
    pruned_xdt = ext_dt(copy.deepcopy(base_dt))
    #* Step by step remove branches
    for prune_node in node_series:
        current_dt = pruned_xdt.removeBranch(prune_node)
        pruned_xdt = ext_dt(current_dt)
    return pruned_xdt.base_dt


#!/usr/bin/env python3
import pprint, argparse, pickle, json

import maze

def fail(message):
    return {
        'score'     : 0,
        'output'    : message,
        'visibility': 'visible',
    }

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description     = 'CS440 MP1 Autograder',
        formatter_class = argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--gradescope', default = False, action = 'store_true',
                        help = 'save output in gradescope-readable json file')

    arguments   = parser.parse_args()

    try:
        import search
    except ImportError:
        message = 'could not find module \'search\', did you upload `search.py`?'
        if arguments.gradescope:
            with open('results.json', 'w') as file:
                file.write(json.dumps(fail(message)))
        else:
            print(message)
        raise SystemExit

def generate_answer_key(path, mazes, solutions):
    key_instructor  = tuple({case: getattr(search, solution)(maze)
        for case, maze in mazes.items()}
        for mazes, solution in zip(mazes, solutions))
    key_student     = tuple({case: len(path) for case, path in part.items()}
        for part in key_instructor)
    pickle.dump(key_instructor, open(path['instructor'], 'wb'))
    pickle.dump(key_student,    open(path['student'],    'wb'))

def load_answer_key(path):
    try:
        return pickle.load(open(path['instructor'], 'rb'))
    except FileNotFoundError:
        print('running in student mode (instructor key unavailable)')
        return pickle.load(open(path['student'],    'rb'))

def grade_closed(name, key, mazes, solution, weight = 1):
    def grade(case, maze):
        y = key[case]
        z = getattr(search, solution)(maze)
        # check that the path is valid
        score_validity  = int(maze.validate_path(z) is None)
        # check that the length of the student’s path matches
        score_length    = int(len(z) == (y if type(y) is int else len(y)))
        # if instructor key available, check that path vertices match
        score_vertices  = int(type(y) is not int and z == y)

        return (
            {
                'name'      : '{0}: `validate_path(_:)` for \'{1}\' maze'.format(name, case),
                'score'     : 0.5 * weight * score_validity,
                'max_score' : 0.5 * weight,
                'visibility': 'visible'
            },
            {
                'name'      : '{0}: correct path length for \'{1}\' maze'.format(name, case),
                'score'     : 0.5 * weight * score_length,
                'max_score' : 0.5 * weight,
                'visibility': 'visible'
            },
            #{
            #    'name'      : '{0}: correct path vertices for \'{1}\' maze'.format(name, case),
            #    'score'     : 0.75 * weight * score_vertices,
            #    'max_score' : 0.75 * weight * (type(y) is not int),
            #    'visibility': 'visible'
            #},
        )

    return tuple(item for case, maze in mazes.items() for item in grade(case, maze))

def grade_open(name, key, mazes, solution, weight = 1):
    def grade(case, maze):
        y = key[case]
        z = getattr(search, solution)(maze)
        # check that the path is valid
        score_validity  = int(maze.validate_path(z) is None)
        # score student path by dividing the length of an MST-based solution by
        # the length of the student path
        score_length    = min((y if type(y) is int else len(y)) / max(len(z), 1), 1)

        return (
            {
                'name'      : '{0}: `validate_path(_:)` for \'{1}\' maze'.format(name, case),
                'score'     : 0.5 * weight * score_validity,
                'max_score' : 0.5 * weight,
                'visibility': 'visible'
            },
            {
                'name'      : '{0}: path length for \'{1}\' maze'.format(name, case),
                'score'     : 0.5 * weight * score_length,
                'max_score' : 0.5 * weight,
                'visibility': 'visible'
            },
        )

    return tuple(item for case, maze in mazes.items() for item in grade(case, maze))

def main():
    solutions = ('bfs', 'astar_single', 'astar_corner', 'astar_multiple', 'fast')
    for solution in solutions:
        if not hasattr(search, solution):
            return fail('module \'search\' is missing expected member \'{0}\''.format(solution))
        if not callable(getattr(search, solution)):
            return fail('member \'{0}\' in module \'search\' is not callable'.format(solution))

    mazes = (
        # part 1: 20 points total, 4 points per case
        {case: maze.maze('data/part-1/{0}'.format(case))
            for case in ('tiny', 'small', 'medium', 'large', 'open')},
        # part 2: 20 points total, 4 points per case
        {case: maze.maze('data/part-2/{0}'.format(case))
            for case in ('tiny', 'small', 'medium', 'large', 'open')},
        # part 3: 30 points total, 10 points per case
        {case: maze.maze('data/part-3/{0}'.format(case))
            for case in ('tiny', 'medium', 'large')},
        # part 4: 30 points total, 10 points per case
        {case: maze.maze('data/part-4/{0}'.format(case))
            for case in ('tiny', 'small', 'medium')},
        # part 5: 22 points total, 22 points per case
        #{case: maze.maze('data/part-5/{0}'.format(case))
        #    for case in ('large',)},
    )

    #generate_answer_key({'instructor': 'key', 'student': 'key-student'}, mazes, solutions)
    key             = load_answer_key({'instructor': 'key', 'student': 'key-student'})
    parts_closed    = tuple(item for i, points in zip(range(0, 4), (4, 4, 10, 10))
        for item in grade_closed('part-{0}'.format(i + 1), key[i], mazes[i], solutions[i],
            weight = points))
    #parts_open      = tuple(item for i, points in zip(range(4, 5), (22,))
    #    for item in grade_open(  'part-{0}'.format(i + 1), key[i], mazes[i], solutions[i],
    #        weight = points))

    # construct grade dictionary for gradescope
    return {
        'visibility': 'visible',
        'tests': parts_closed
        #'tests': parts_closed + parts_open
    }

if __name__ == "__main__":
    results     = main()
    if arguments.gradescope:
        with open('results.json', 'w') as file:
            file.write(json.dumps(results))
    else:
        pprint.pp(results)


"""
Contains functions for common support adjustments.

Created on Thu May 11 16:30:11 2023

@author: MLechner
# -*- coding: utf-8 -*-
"""
from copy import deepcopy

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split

from mcf import mcf_data_functions as mcf_data
from mcf import mcf_general as gp
from mcf import mcf_general_sys as mcf_sys
from mcf import mcf_print_stats_functions as ps
from mcf import mcf_variable_importance_functions as vi

def common_support(mcf_, tree_df, fill_y_df, train=True):
    """Remove observations from data files that are off-support."""
    gen_dic = mcf_.gen_dict
    lc_dic, var_x_type, cs_dic = mcf_.lc_dict, mcf_.var_x_type, mcf_.cs_dict
    data_train_dic = mcf_.data_train_dict
    d_name, _, no_of_treat = mcf_data.get_treat_info(mcf_)
    x_name, x_type = gp.get_key_values_in_list(var_x_type)
    names_unordered = [x_name[j] for j, val in enumerate(x_type) if val > 0]
    if gen_dic['with_output'] and gen_dic['verbose']:
        ps.print_mcf(gen_dic, '\n' + '=' * 100 + '\nCommon support analysis',
                     summary=True)
    if train:
        if lc_dic['cs_cv']:   # Crossvalidate ... only tree data is used
            tree_mcf_df, fill_y_mcf_df = tree_df.copy(), fill_y_df.copy()
            d_tree_mcf_np = tree_mcf_df[d_name].to_numpy().ravel()
        else:  # Use lc_dic['cs_share'] of data for common support estim. only
            # Take the same share of obs. from both input samples
            tree_mcf_df, tree_cs_df = train_test_split(
                tree_df, test_size=lc_dic['cs_share'], random_state=42)
            fill_y_mcf_df, fill_y_cs_df = train_test_split(
                fill_y_df, test_size=lc_dic['cs_share'], random_state=42)
            data_cs_df = pd.concat([tree_cs_df, fill_y_cs_df], axis=0)
            x_cs_df, _ = mcf_data.get_x_data(data_cs_df, x_name)
            d_cs_np = data_cs_df[d_name].to_numpy().ravel()
        x_fy_df, _ = mcf_data.get_x_data(fill_y_mcf_df, x_name)
    else:
        tree_mcf_df, fill_y_mcf_df = tree_df, None
    x_mcf_df, obs_mcf = mcf_data.get_x_data(tree_mcf_df, x_name)  # train,adj.
    txt = ''
    if train:
        if names_unordered:  # List is not empty
            x_fy_df, _ = mcf_data.dummies_for_unord(
                x_fy_df, names_unordered, data_train_dict=data_train_dic)
            x_mcf_df, dummy_names = mcf_data.dummies_for_unord(
                x_mcf_df, names_unordered, data_train_dict=data_train_dic)
            if not lc_dic['cs_cv']:
                x_cs_df, _ = mcf_data.dummies_for_unord(
                    x_cs_df, names_unordered, data_train_dict=data_train_dic)
        else:
            dummy_names = None
        if not lc_dic['cs_cv']:
            x_cs_np = x_cs_df.to_numpy()
        x_fy_np, x_mcf_np = x_fy_df.to_numpy(), x_mcf_df.to_numpy()
        if gen_dic['with_output'] and gen_dic['verbose']:
            txt += '\n' + '-' * 100 + '\n'
            txt += 'Computing random forest based common support\n'
            ps.print_mcf(gen_dic, txt, summary=False)
        max_workers = 1 if gen_dic['replication'] else gen_dic['mp_parallel']
        classif = RandomForestClassifier(
            n_estimators=mcf_.cf_dict['boot'], max_features='sqrt',
            bootstrap=True, oob_score=False, n_jobs=max_workers,
            random_state=42, verbose=False, min_samples_split=5)
        if lc_dic['cs_cv']:   # Crossvalidate
            index = np.arange(obs_mcf)       # indices
            rng = np.random.default_rng(seed=9324561)
            rng.shuffle(index)
            index_folds = np.array_split(index, lc_dic['cs_cv_k'])
            pred_mcf_np = np.empty((len(index), no_of_treat))
            pred_fy_np_fold = np.zeros((len(x_fy_np), no_of_treat))
            forests = []
            for fold_pred in range(lc_dic['cs_cv_k']):
                fold_train = [x for idx, x in enumerate(index_folds)
                              if idx != fold_pred]
                index_train = np.hstack(fold_train)
                index_pred = index_folds[fold_pred]
                x_pred, x_train = x_mcf_np[index_pred], x_mcf_np[index_train]
                d_train = d_tree_mcf_np[index_train]
                classif.fit(x_train, d_train)
                forests.append(deepcopy(classif))
                pred_mcf_np[index_pred, :] = classif.predict_proba(x_pred)
                pred_fy_np_fold += classif.predict_proba(x_fy_np)
            pred_cs_np, d_cs_np = pred_mcf_np, d_tree_mcf_np  # To get cut-offs
            pred_fy_np = pred_fy_np_fold / lc_dic['cs_cv_k']
        else:
            x_train, x_test, d_train, d_test = train_test_split(
                x_cs_np, d_cs_np, test_size=0.25, random_state=42)
            classif.fit(x_train, d_train)
            pred_cs_np = classif.predict_proba(x_test)  # -> determine cut-offs
            d_cs_np = d_test
            pred_mcf_np = classif.predict_proba(x_mcf_np)   # cut and return
            pred_fy_np = classif.predict_proba(x_fy_np)     # cut and return
            forests = [classif]
        cs_dic['forests'] = forests
        if gen_dic['with_output']:
            vi.print_variable_importance(
                deepcopy(classif), x_mcf_df, tree_mcf_df[d_name], x_name,
                names_unordered, dummy_names, gen_dic, summary=False)
        # Normalize estimated probabilities to add up to 1
        pred_cs_np_sum = pred_cs_np.sum(axis=1, keepdims=True)
        pred_mcf_np_sum = pred_mcf_np.sum(axis=1, keepdims=True)
        pred_fy_np_sum = pred_fy_np.sum(axis=1, keepdims=True)
        pred_cs_np /= pred_cs_np_sum
        pred_mcf_np /= pred_mcf_np_sum
        pred_fy_np /= pred_fy_np_sum
        # Determine cut-offs nased on pred_cs_np
        cs_dic['cut_offs'] = get_cut_off_probs(mcf_, pred_cs_np, d_cs_np)
        mcf_.cs_dict = cs_dic   # Update instance with cut-off prob's
        # Descriptive stats
        if gen_dic['with_output']:
            plot_support(mcf_, pred_cs_np, d_cs_np)
            descriptive_stats_on_off_support(mcf_, pred_fy_np, fill_y_mcf_df,
                                             'Training - fill mcf with y data')
        # Reduce samples
        fill_y_mcf_df, _ = on_off_support_df(mcf_, pred_fy_np, fill_y_mcf_df)
    else:  # Reduce prediction sample
        # Predict treatment probabilities
        if names_unordered:  # List is not empty
            x_mcf_df, _ = mcf_data.dummies_for_unord(
                x_mcf_df, names_unordered, data_train_dict=data_train_dic)
        pred_mcf_np = np.zeros((len(x_mcf_df), no_of_treat))
        # If cross-validation, take average of forests in folds
        for forest in cs_dic['forests']:
            pred_mcf_np += forest.predict_proba(x_mcf_df.to_numpy())
        pred_mcf_np /= len(cs_dic['forests'])
    # Normalize estimated probabilities to add up to 1
    pred_mcf_np /= pred_mcf_np.sum(axis=1, keepdims=True)
    # Delete observation off support
    if gen_dic['with_output']:
        titel = 'Training - build mcf data' if train else 'Prediction data'
        descriptive_stats_on_off_support(mcf_, pred_mcf_np, tree_mcf_df, titel)
    tree_mcf_df, _ = on_off_support_df(mcf_, pred_mcf_np, tree_mcf_df)
    return tree_mcf_df, fill_y_mcf_df

def check_if_too_many_deleted(mcf_, obs_keep, obs_del):
    """Check if too many obs are deleted and raise Exception if so."""
    max_del_train = mcf_.cs_dict['max_del_train']
    share_del = obs_del / (obs_keep + obs_del)
    if share_del > max_del_train:
        err_str = (
            f'{share_del:3.1%} observation deleted in common support, but only'
            f' {max_del_train:3.1%} observations of training data are allowed'
            ' to be deleted in support check. Programme is terminated. Improve'
            ' balance of input data or change share allowed to be deleted.')
        raise ValueError(err_str)

def descriptive_stats_on_off_support(mcf_, probs_np, data_df, titel=''):
    """Compute descriptive stats for deleted and retained observations."""
    keep_df, delete_df = on_off_support_df(mcf_, probs_np, data_df)
    gen_dic, var_dic = mcf_.gen_dict, mcf_.var_dict
    if delete_df.empty:
        txt = (f'\n\nData investigated for common support: {titel}\n'
               + '-' * 100)
        ps.print_mcf(gen_dic, '\nNo observations deleted in common support '
                     'check', summary=True)
    else:
        d_name, _, _ = mcf_data.get_treat_info(mcf_)
        x_name = var_dic['x_name']
        obs_del, obs_keep = len(delete_df), len(keep_df)
        obs = obs_del + obs_keep
        txt = '\n' + '-' * 100
        txt += f'\nData investigated for common support: {titel}\n' + '-' * 100
        txt += f'\nObservations deleted: {obs_del:4} ({obs_del/obs:.2%})'
        txt += '\n' + '-' * 100
        ps.print_mcf(gen_dic, txt, summary=True)
        txt = ''
        with pd.option_context(
                'display.max_rows', 500, 'display.max_columns', 500,
                'display.expand_frame_repr', True, 'display.width', 150,
                'chop_threshold', 1e-13):
            all_var_names = [name.upper() for name in data_df.columns]
            if d_name[0].upper() in all_var_names:
                d_keep = keep_df[d_name]
                d_delete = delete_df[d_name]
                d_keep_count = d_keep.value_counts(sort=False)
                d_delete_count = d_delete.value_counts(sort=False)
                d_keep_count = pd.concat(
                    [d_keep_count, np.round(d_keep_count / obs_keep * 100, 2)],
                    axis=1)
                d_delete_count = pd.concat(
                    [d_delete_count,
                     np.round(d_delete_count / obs_del * 100, 2)], axis=1)
                d_keep_count.columns = ['Obs.', 'Share in %']
                d_delete_count.columns = ['Obs.', 'Share in %']
                if gen_dic['panel_data']:
                    cluster_id = data_df[var_dic['cluster_name']].squeeze()
                    cluster_keep = keep_df[var_dic['cluster_name']].squeeze()
                    cluster_delete = delete_df[var_dic['cluster_name']
                                               ].squeeze()
                k_str = '\nObservations kept, by treatment\n    '
                d_str = '\nObservations deleted, by treatment\n '
                k_str += d_keep_count.to_string()
                d_str += d_delete_count.to_string()
                txt += k_str + '\n' + '-   ' * 20 + d_str
                if gen_dic['panel_data']:
                    txt += '-   ' * 20
                    txt += '\nTotal number of panel units:'
                    txt += f'{len(cluster_id.unique())}'
                    txt += '\nObservations belonging to '
                    txt += f'{len(cluster_keep.unique())} panel units that are'
                    txt += '  ON support\nObservations belonging to '
                    txt += f'{len(cluster_delete.unique())} panel units are'
                    txt += ' OFF support'
                ps.print_mcf(gen_dic, txt, summary=True)
            else:
                txt = f'\nData investigated for common support: {titel}\n'
                txt += '-' * 100 + '\nTreatment not in prediction data.\n'
                txt += '-' * 100
                ps.print_mcf(gen_dic, txt, summary=False)
            txt = '\n' + '-' * 100
            txt += '\nFull sample (Data ON and OFF support)' + '\n' + '-' * 100
            ps.print_mcf(gen_dic, txt, summary=False)
            ps.print_mcf(gen_dic, data_df[x_name].describe().transpose(),
                         summary=False)
            if d_name[0].upper() in all_var_names:
                mean_by_treatment(data_df[d_name], data_df[x_name], gen_dic,
                                  summary=False)
            txt = '\n' + '-' * 100 + '\nData ON support' + '\n' + '-' * 100
            ps.print_mcf(gen_dic, txt, summary=False)
            ps.print_mcf(gen_dic, keep_df[x_name].describe().transpose(),
                         summary=False)
            if d_name[0].upper() in all_var_names and len(keep_df) > 5:
                mean_by_treatment(keep_df[d_name], keep_df[x_name], gen_dic,
                                  summary=False)
            txt = '\n' + '-' * 100 + '\nData OFF support' + '\n' + '-' * 100
            ps.print_mcf(gen_dic, txt, summary=False)
            ps.print_mcf(gen_dic, delete_df[x_name].describe().transpose(),
                         summary=False)
            if d_name[0].upper() in all_var_names and len(delete_df) > 5:
                mean_by_treatment(delete_df[d_name], delete_df[x_name],
                                  gen_dic, summary=False)
        check_if_too_many_deleted(mcf_, obs_keep, obs_del)

def mean_by_treatment(treat_df, data_df, gen_dic, summary=False):
    """Compute mean by treatment status."""
    treat_df = treat_df.squeeze()
    treat_vals = pd.unique(treat_df)
    txt = '\n------------------ Mean by treatment status ---------------------'
    ps.print_mcf(gen_dic, txt, summary=summary)
    if len(treat_vals) > 0:
        mean = data_df.groupby(treat_df).mean(numeric_only=True)
        ps.print_mcf(gen_dic, mean.transpose(), summary=summary)
    else:
        txt = f'\nAll obs have same treatment: {treat_vals}'
        ps.print_mcf(gen_dic, txt, summary=summary)

def on_off_support_df(mcf_, probs_np, data_df):
    """Split DataFrame into retained and deleted part."""
    cs_dic = mcf_.cs_dict
    _, _, no_of_treat = mcf_data.get_treat_info(mcf_)
    lower, upper = cs_dic['cut_offs']['lower'], cs_dic['cut_offs']['upper']
    obs = len(probs_np)
    off_support = np.empty(obs, dtype=bool)
    # off_upper = np.empty(no_of_treat, dtype=bool)
    # off_lower = np.empty_like(off_upper)
    for i in range(obs):
        off_upper = np.any(probs_np[i, :] > upper)
        off_lower = np.any(probs_np[i, :] < lower)
        off_support[i] = off_upper or off_lower
    data_on_df = data_df[~off_support].copy()
    data_off_df = data_df[off_support].copy()
    return data_on_df, data_off_df

def plot_support(mcf_, probs_np, d_np):
    """Histogrammes for distribution of treatment probabilities for overlap."""
    cs_dic, int_dic = mcf_.cs_dict, mcf_.int_dict
    lower, upper = cs_dic['cut_offs']['lower'], cs_dic['cut_offs']['upper']
    _, d_values, _ = mcf_data.get_treat_info(mcf_)
    color_list = ['red', 'blue', 'green', 'violet', 'magenta', 'crimson',
                  'yellow', 'darkorange', 'khaki', 'skyblue', 'darkgreen',
                  'olive', 'greenyellow',  'aguamarine', 'deeppink',
                  'royalblue', 'navy', 'blueviolet', 'purple']
    if len(color_list) < len(d_values):
        color_list = color_list * len(d_values)
    color_list = color_list[:len(d_values)]
    for idx_p, ival_p in enumerate(d_values):  # iterate treatment probs
        treat_prob = probs_np[:, idx_p]
        titel = f'Probability of treatment {ival_p} in different subsamples'
        f_titel = f'common_support_pr_treat{ival_p}'
        file_name_csv = (cs_dic['common_support_fig_pfad_csv']
                         + '/' + f_titel + '.csv')
        file_name_jpeg = (cs_dic['common_support_fig_pfad_jpeg']
                          + '/' + f_titel + '.jpeg')
        file_name_pdf = (cs_dic['common_support_fig_pfad_pdf']
                         + '/' + f_titel + '.pdf')
        file_name_csv_d = (cs_dic['common_support_fig_pfad_csv']
                           + '/' + f_titel + '_d.csv')
        file_name_jpeg_d = (cs_dic['common_support_fig_pfad_jpeg']
                            + '/' + f_titel + '_d.jpeg')
        file_name_pdf_d = (cs_dic['common_support_fig_pfad_pdf']
                           + '/' + f_titel + '_d.pdf')
        data_hist = [treat_prob[d_np == val] for val in d_values]
        fig, axs = plt.subplots()
        fig_d, axs_d = plt.subplots()
        labels = ['Treat ' + str(d) for d in d_values]
        fit_line_all, bins_all = [], []
        for idx, dat in enumerate(data_hist):
            axs.hist(dat, bins='auto', histtype='bar', label=labels[idx],
                     color=color_list[idx], alpha=0.5, density=False)
            _, bins, _ = axs_d.hist(dat, bins='auto', histtype='bar',
                                    label=labels[idx], color=color_list[idx],
                                    alpha=0.5, density=True)
            bins_all.append(bins)
            sigma = np.std(dat)
            fit_line = ((1 / (np.sqrt(2 * np.pi) * sigma))
                        * np.exp(-0.5 * (1 / sigma
                                         * (bins - np.mean(dat)))**2))
            axs_d.plot(bins, fit_line, '--', color=color_list[idx],
                       label='Smoothed ' + labels[idx])
            fit_line_all.append(fit_line)
        axs.set_title(titel)
        axs.set_xlabel('Treatment probability')
        axs.set_ylabel('Observations')
        axs.set_xlim([0, 1])
        axs.axvline(lower[idx_p], color='blue', linewidth=0.7,
                    linestyle="--", label='min')
        axs.axvline(upper[idx_p], color='black', linewidth=0.7,
                    linestyle="--", label='max')
        axs.legend(loc=int_dic['legend_loc'], shadow=True,
                   fontsize=int_dic['fontsize'])
        mcf_sys.delete_file_if_exists(file_name_jpeg)
        mcf_sys.delete_file_if_exists(file_name_pdf)
        mcf_sys.delete_file_if_exists(file_name_csv)
        fig.savefig(file_name_jpeg, dpi=int_dic['dpi'])
        fig.savefig(file_name_pdf, dpi=int_dic['dpi'])
        save_list = [data_hist, labels]
        save_df = pd.DataFrame(save_list)
        save_df = save_df.fillna(value='NaN')
        save_df.to_csv(file_name_csv, index=False)
        axs_d.set_title(titel)
        axs_d.set_xlabel('Treatment probability')
        axs_d.set_ylabel('Density')
        axs_d.set_xlim([0, 1])
        axs_d.axvline(lower[idx_p], color='blue', linewidth=0.7,
                      linestyle="--", label='min')
        axs_d.axvline(upper[idx_p], color='black', linewidth=0.7,
                      linestyle="--", label='max')
        axs_d.legend(loc=int_dic['legend_loc'], shadow=True,
                     fontsize=int_dic['fontsize'])
        mcf_sys.delete_file_if_exists(file_name_jpeg_d)
        mcf_sys.delete_file_if_exists(file_name_pdf_d)
        mcf_sys.delete_file_if_exists(file_name_csv_d)
        save_list = [fit_line_all, bins_all, labels]
        save_df = pd.DataFrame(save_list)
        save_df = save_df.fillna(value='NaN')
        save_df.to_csv(file_name_csv_d, index=False)
        fig_d.savefig(file_name_jpeg_d, dpi=int_dic['dpi'])
        fig_d.savefig(file_name_pdf_d, dpi=int_dic['dpi'])
        if int_dic['show_plots']:
            plt.show()
        else:
            plt.close()

def get_cut_off_probs(mcf_, probs_np, d_np):
    """Compute the cut-offs for common support for training only."""
    cs_dic, gen_dic = mcf_.cs_dict, mcf_.gen_dict
    _, d_values, no_of_treat = mcf_data.get_treat_info(mcf_)
    if cs_dic['type'] == 1:
        q_s = cs_dic['quantil']
        upper_limit = np.empty((no_of_treat, no_of_treat))
        lower_limit = np.empty_like(upper_limit)
        for idx, ival in enumerate(d_values):
            probs = probs_np[d_np == ival]
            if q_s == 1:
                upper_limit[idx, :] = np.max(probs, axis=0)
                lower_limit[idx, :] = np.min(probs, axis=0)
            else:
                upper_limit[idx, :] = np.quantile(probs, q_s, axis=0)
                lower_limit[idx, :] = np.quantile(probs, 1 - q_s, axis=0)
            upper_limit[idx, 0] = 1
            lower_limit[idx, 0] = 0
        txt = ''
        if cs_dic['adjust_limits'] != 0:
            upper_limit *= 1 + cs_dic['adjust_limits']
            lower_limit *= 1 - cs_dic['adjust_limits']
            lower_limit = np.clip(lower_limit, a_min=0, a_max=1)
            upper_limit = np.clip(upper_limit, a_min=0, a_max=1)
            if gen_dic['with_output']:
                txt += '\n' + '-' * 100 + '\nCommon support bounds adjusted by'
                txt += f' {cs_dic["adjust_limits"]:5.2%}-points\n' + '-' * 100
        if gen_dic['with_output']:
            txt += '\nTreatment sample     Treatment probabilities in %'
            txt += '\n--------------------- Upper limits ----------------'
            for idx, ival in enumerate(d_values):
                txt += f'\nD = {ival:9}          '
                for jdx in range(no_of_treat):
                    txt += f'{upper_limit[idx, jdx]:>7.2%}  '
            txt += '\n--------------------- Lower limits ----------------'
            for idx, ival in enumerate(d_values):
                txt += f'\nD = {ival:9}          '
                for jdx in range(no_of_treat):
                    txt += f'{lower_limit[idx, jdx]:>7.2%}  '
            txt += '\n' + 100 * '-'
            txt += '\nFirst treatment is set to 1 and 0 (ignored) due to'
            txt += ' additivity.' + '\n' + 100 * '-'
        upper, lower = np.min(upper_limit, axis=0), np.max(lower_limit, axis=0)
        if gen_dic['with_output']:
            upper_str = [f'{x:>7.2%}' for x in upper]
            lower_str = [f'{x:>7.2%}' for x in lower]
            txt += '\nUpper limits used: ' + ' '.join(upper_str)
            txt += '\nLower limits used: ' + ' '.join(lower_str)
            txt += '\n' + 100 * '-'
            ps.print_mcf(gen_dic, txt, summary=True)
    else:
        # Normalize such that probabilities add up to 1
        upper = np.ones(no_of_treat) * (1 - cs_dic['min_p'])
        lower = np.ones(no_of_treat) * cs_dic['min_p']
    cut_offs = {'upper': upper, 'lower': lower}
    return cut_offs


#!/usr/bin/python3
#
# Script for dumping/programming SPI flash chips with Hydrabus.
# Based on HydraBus BBIO documentation: https://github.com/hydrabus/hydrafw/wiki/HydraFW-Binary-SPI-mode-guide
#
# Author: MrKalach [https://github.com/MrKalach]
# License: GPLv3 (https://choosealicense.com/licenses/gpl-3.0/)
#

import hexdump
import serial
import sys
import argparse

class Params():
    serial_port     = 'COM4'
    serial_speed    = 115200

    block_size      = 0x1000      # max_buffer (HydraBus limitation)
    spi             = 1
    polarity        = 0
    clock_phase     = 0

    SPI_SPEED_TABLE = {
        "SPI1" : {
            "320k"  : 0b01100000,
            "650k"  : 0b01100001,
            "1.31m" : 0b01100010,
            "2.62m" : 0b01100011,
            "5.25m" : 0b01100100,
            "10.5m" : 0b01100101,
            "21m"   : 0b01100110,
            "42m"   : 0b01100111,
        },
        "SPI2" : {
            "160k"  : 0b01100000,
            "320k"  : 0b01100001,
            "650k"  : 0b01100010,
            "1.31m" : 0b01100011,
            "2.62m" : 0b01100100,
            "5.25m" : 0b01100101,
            "10.5m" : 0b01100110,
            "21m"   : 0b01100111,
        }
    }

    SPI_speed = 0b01100110

def check_result(result, expected, error_str):
    assert expected in result, f'{error_str}: expected={expected} != result={result}'

class SPI_hydra:
    def __init__(self, params):
        self.serial = None
        self.params = params

    def __enter__(self):
        self.serial = serial.Serial(self.params.serial_port, self.params.serial_speed)

        # Switching HydraBus to binary mode
        for _ in range(20):
            self.serial.write(b'\x00')
        check_result(b"BBIO1", self.serial.read(5), 'Could not switch into binary mode!')

        self.serial.reset_input_buffer()

        # Sitching to SPI mode
        self.write_bytes(0b00000001)
        check_result(b"SPI1", self.serial.read(4), 'Could not switch to SPI mode')

        # Configuring SPI
        cfg = 0b10000000 | self.params.polarity << 3 | self.params.clock_phase << 2 | self.params.spi
        self.write_bytes(cfg) #polarity => 0, phase => 0, SPI1
        check_result(b'\x01', self.serial.read(1),'Could not setup SPI port!')

        # Setting up SPI speed
        self.write_bytes(self.params.SPI_speed)
        check_result(b'\x01', self.serial.read(1), 'Could not setup SPI speed!')

        self.cs_off()

        return self

    def __exit__(self, type, value, traceback):
        # Switch HydraBus back to terminal mode
        self.serial.write(b'\x00')
        self.serial.write(b'\x0F')

    def write_bytes(self, data):
        if not isinstance(data, list) and not isinstance(data, bytes):
            data = [data]
        self.serial.write(data)

    def read(self, num):
        return self.serial.read(num)

    def cs_on(self):
        self.write_bytes(0b00000011)
        check_result(b'\x01', self.serial.read(1), 'Cannot switch CS to on!')

    def cs_off(self):
        self.write_bytes(0b00000010)
        check_result(b'\x01', self.serial.read(1), 'Cannot switch CS to off!')

##############################################################################################################################

def num_to_bytes(num, padding):
    return num.to_bytes(padding, byteorder='big')

def dump_chip_id(params):
    with SPI_hydra(params) as hydra:
        print("Getting chip manufacturer id ...")

        # 0b00000100 - write then read (with CS pinning)
        hydra.write_bytes([0b00000100, 0x00, 0x01, 0x00, 0x03])
        # send RDID command
        hydra.write_bytes(0x9f)

        check_result(b'\x01', hydra.read(1), 'Error occuried while getting chip id!')
        result = hydra.read(3)

        print(f'Chip manufacturer id: {hexdump.dump(result)}')

def dump_flash(params, filename, blocks, start_addr_hex):
    start_addr = int(start_addr_hex, 16)

    with SPI_hydra(params) as hydra:
        print('Start dumping ...')
        print(f'Reading blocks: ', end='')

        block = 0
        buf = bytearray()

        while block < blocks:
            # write-then-read: \x00\x04 - input size
            hydra.write_bytes(b'\x04\x00\x04' + num_to_bytes(params.block_size, 2))

            # read command (\x03) and 3 bytes of address
            hydra.write_bytes(b'\x03' + num_to_bytes(start_addr, 3))

            check_result(b'\x01', hydra.read(1), 'Error occurred while dumping!')

            buf += hydra.read(params.block_size)

            if (block % 10) == 0:
                print(block, end='')
            else:
                print('.', end='')

            block += 1
            start_addr += params.block_size

        with open(filename, 'wb+') as f:
            f.write(buf)

        print('\nDone')

# This code has been proofed && worked with Macronix MX25L12845E chip
# But somekind of generic commands is used, so it should work for most of other chips
def program_flash(params, filename, start_addr_hex, erase_before_program):
    MAX_ATTEMPTS = 30

    def enable_write():
        # Enabling write
        attempts = 0
        status = 0
        while status & 0b11 != 0b10:
            # WREN
            hydra.write_bytes(b'\x04\x00\x01\x00\x00')
            hydra.write_bytes(0x06)
            check_result(b'\x01', hydra.read(1), 'Error occurred while sending WREN!')

            # RDSR
            hydra.write_bytes(b'\x04\x00\x01\x00\x01')
            hydra.write_bytes(0x05)
            check_result(b'\x01', hydra.read(1), 'Error occurred while sending RDSR!')

            status = hydra.read(1)[0]

            if status & (1 << 7):
                assert 'Chip has a write protection!'

            attempts += 1
            if attempts > MAX_ATTEMPTS:
                assert 'Cannot set WREN!'

    def check_fail():
        # RDSCUR to check P_FAIL and E_FAIL
        hydra.write_bytes(b'\x04\x00\x01\x00\x01')
        hydra.write_bytes(0x28)
        check_result(b'\x01', hydra.read(1), 'Error occurred while sending RDSCUR!')

        status = hydra.read(1)[0]
        if status & 0x01100000 != 0: # check P_FAIL && E_FAIL
            # CLSR - clear SR Fail Register
            hydra.write_bytes(b'\x04\x00\x01\x00\x01')
            hydra.write_bytes(0x30)
            check_result(b'\x01', hydra.read(1), 'Error occurred while sending RDSCUR!')

            assert f'P_FAIL or E_FAIL: register={bin(status)}'

    def erase_sector_4k(addr):
        enable_write()

        # Erase sector/block
        # SE - sector erase, 0x20 + 3 bytes of address
        hydra.write_bytes(b'\x04\x00\x04\x00\x00')
        hydra.write_bytes(b'\x20' + num_to_bytes(start_addr, 3)) # CE - chip erase, can be used here
        check_result(b'\x01', hydra.read(1), 'Error occurred while sending SE!')

        # Check status
        status = 1
        attempts = 0
        while status & 0b1 != 0: #write in progress bit check
            # RDSR
            hydra.write_bytes(b'\x04\x00\x01\x00\x01')
            hydra.write_bytes(0x05)
            check_result(b'\x01', hydra.read(1), 'Error occurred while sending RDSR!')

            status = hydra.read(1)[0]

            attempts += 1
            if attempts > MAX_ATTEMPTS:
                assert 'Cannot read status RDSR!'

        check_fail()

    def write_page(page, data):
        if len(data) != 0x100:
            data = data + b'\x00' * (0x100 - len(data))
        assert len(data) == 0x100, f'Somthing goes wrong, data size is not 0x100 => {len(data)}'

        enable_write()

        # Program data
        # PP - page program, 0x02 + 3 bytes of address + data
        # low byte of address is always 0x00
        hydra.write_bytes(b'\x04\x01\x04\x00\x00')
        hydra.write_bytes(b'\x02' + num_to_bytes(page, 2) + b'\x00')
        hydra.write_bytes(data)
        check_result(b'\x01', hydra.read(1), 'Error occurred while sending PP!')

        # Check status
        status = 1
        attempts = 0
        while status & 0b1 != 0: #write in progress bit checking
            # RDSR
            hydra.write_bytes(b'\x04\x00\x01\x00\x01')
            hydra.write_bytes(0x05)
            check_result(b'\x01', hydra.read(1), 'Error occurred while sending RDSR!')

            status = hydra.read(1)[0]

            attempts += 1
            if attempts > MAX_ATTEMPTS:
                assert 'Cannot read status RDSR!'

        check_fail()

    start_addr = int(start_addr_hex, 16)
    assert start_addr & 0x1FF == 0, f'In program mode start address has to be aligned (0x1FF bytes boundary). start address = {start_addr}'

    print('Starting programming...')
    print('Blocks:', end='')
    page = 0
    with SPI_hydra(params) as hydra:
        with open(filename, 'rb') as f:
            while True:
                data = f.read(0x100)
                if len(data) == 0:
                    break

                if page % 0xF == 0:
                    if erase_before_program:
                        erase_sector_4k(start_addr)
                    start_addr += 0x1000
                    block_number = int(start_addr / 0x1000)
                    if block_number % 10 == 0:
                        print(block_number, end='')
                    else:
                        print('.', end='')

                write_page(page, data)
                page += 1

    print('\nDone.')

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '-hsp', '--hydra-serial-port',
        type=str, default=Params.serial_port,
        help='Hydra COM port (eg. COM4 or /dev/ttyS0)'
    )
    parser.add_argument(
        '-hss', '--hydra-serial-speed',
        type=int, default=Params.serial_speed,
        help='Hydra COM port speed'
    )
    parser.add_argument(
        '-bs', '--block-size',
        type=int, default=Params.block_size,
        help='Max size of data block for SPI, value should be in range (0, 4096]'
    )

    parser.add_argument(
        '-spi', '--spi-port',
        type=str, default='SPI2' if Params.spi == 0 else 'SPI1',
        help='SPI port',
        choices=['SPI1', 'SPI2']
    )

    parser.add_argument(
        '-ss', '--spi-speed',
        type=str, default='21m',
        help=f'''
            SPI speed (
                SPI1: {",".join(Params.SPI_SPEED_TABLE["SPI1"].keys())}
                SPI2: {",".join(Params.SPI_SPEED_TABLE["SPI2"].keys())}
            )
        ''',
    )

    parser.add_argument(
        '-sp', '--spi-polarity',
        type=int, default=Params.polarity,
        help=f'SPI polarity',
        choices=[0, 1]
    )

    parser.add_argument(
        '-scph', '--spi-clock-phase',
        type=int, default=Params.clock_phase,
        help=f'SPI clock phase',
        choices=[0, 1]
    )

    subparsers = parser.add_subparsers(dest='action', required=True)

    subparsers.add_parser('chip-id', help='Get manufacturer chip id')

    dump_parser = subparsers.add_parser('dump', help='Dump flash into file')
    dump_parser.add_argument('filename', type=str, help='File to store dump')
    dump_parser.add_argument('blocks', type=int, help='Blocks to read')
    dump_parser.add_argument('start_address', type=str, help='Start address (HEX)')

    burn_parser = subparsers.add_parser('program', help='Program img into flash')
    burn_parser.add_argument('filename', type=str, help='File to program')
    burn_parser.add_argument('start_address', type=str, help='Start address (HEX)')
    burn_parser.add_argument('-ebw', '--erase_before_program', action='store_true', help='Erase data before program')

    try:
        options = parser.parse_args()
        params = Params()
        params.serial_port = options.hydra_serial_port
        params.serial_speed = options.hydra_serial_speed
        if options.block_size < 1 or options.block_size > 0x1000:
            print(f'Invalid block size {options.block_size}! Value should be in range (1, 4096]')
            exit()
        params.block_size = options.block_size
        params.spi = {"SPI1" : 1, "SPI2" : 0}[options.spi_port]
        if options.spi_speed not in Params.SPI_SPEED_TABLE[options.spi_port]:
            print(f'Wrong speed({options.spi_speed}) for this({options.spi_port}) port!')
            exit()

        params.SPI_speed = Params.SPI_SPEED_TABLE[options.spi_port][options.spi_speed]
        params.polarity = options.spi_polarity
        params.clock_phase = options.spi_clock_phase

        if options.action == 'chip-id':
            dump_chip_id(params)
        elif options.action == 'dump':
            dump_flash(params, options.filename, options.blocks, options.start_address)
        elif options.action == 'program':
            program_flash(params, options.filename, options.start_address, options.erase_before_program)

    except SystemExit:
        print()
        parser.print_help()
        sys.exit(0)

if __name__ == '__main__':
    main()


import subprocess
from itertools import product, combinations

import numpy as np

def runner():
    SEQ = 'MQYKLILNGKTLKGETTTEAVDAATAEKVFKQYANDNGVDGEWTYDDATKTFTVTE'
    AA = ['A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'L',
          'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'Y']
    AA_dict = dict((k,v) for v,k in enumerate(AA))
    SEQ = list(map(AA_dict.get, list(SEQ)))

    # Test
    mutations = []
    for pos1 in range(1):
        for mut1 in range(4):
            if mut1!=SEQ[pos1]:
                mutations.append([pos1, mut1, -1, -1])
    mutations = np.array(mutations)

    np.savetxt("./assets/mutation_input/test.csv", mutations, delimiter=",")
    run_process("test")

    # Single mutation
    mutations = []
    for pos1 in range(1,len(SEQ)):
        for mut1 in range(len(AA)):
            if mut1!=SEQ[pos1]:
                mutations.append([pos1, mut1, -1, -1])
    mutations = np.array(mutations)
    fnm = 'single_mutations'
    np.savetxt('./assets/mutation_input/'+fnm+'.csv', mutations, delimiter=',')
    run_process(fnm)

    # Double mutation
    residues_list = combinations(list(range(1,len(SEQ))),2)
    for cnt, residues in enumerate(residues_list):
        mutations = []
        for mut1 in range(20):
            for mut2 in range(20):
                if mut1!=SEQ[residues[0]] and mut2!=SEQ[residues[1]]:
                    mutations.append([residues[0],mut1,residues[1],mut2])
        mutations = np.array(mutations)
        fnm = "double_mutations_{0:d}_{1:d}".format(residues[0],residues[1])
        np.savetxt('./assets/mutation_input/'+fnm+'.csv', mutations, delimiter=',')
        run_process(fnm)
        write_files_to_run(fnm, cnt)

def write_files_to_run(fnm, cnt):
    fnm_list = ['run_files_{0:d}'.format(i) for i in range(1,4)]
    index = cnt//495
    with open(fnm_list[index], 'a+') as f:
        f.write('sbatch ./assets/sbatch_scripts/'+fnm+'.sh \n')

def run_process(fnm):
    with open('./assets/sbatch_scripts/'+fnm+'.sh', 'w+') as f:
        partitions = ['sched_mit_arupc_long',
                    'sched_mit_arupc',
                    'sched_any',
                    'sched_mit_hill',
                    'newnodes']
        f.write('#!/bin/bash \n')
        f.write('#SBATCH --job-name='+fnm+' \n')
        f.write('#SBATCH --nodes=1 \n')
        f.write('#SBATCH --cpus-per-task=2 \n')
        f.write('#SBATCH --time=12:00:00 \n')
        f.write('#SBATCH --partition='+','.join(partitions)+' \n')
        f.write('#SBATCH --mem-per-cpu=2000 \n')
        f.write('#SBATCH -o /nobackup1c/users/leerang/UniRep/output/output_%j.txt \n')
        f.write('#SBATCH -e /nobackup1c/users/leerang/UniRep/error/error_%j.txt \n\n')
        f.write('cd /nobackup1c/users/leerang/UniRep')
        f.write('module add /home/software/modulefiles/singularity/3.7.0 \n')
        f.write('singularity exec -B /nobackup1c/users/leerang/UniRep ./UniRep.sif python3 embedding_getter.py '+fnm)

if __name__ == '__main__':
    runner()
    #result = subprocess.run(["python", "embedding_getter.py", "test.csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    #print(result.stdout)
    #print(result.stderr)
    #print(result.returncode)


# 一些常量
target = 'nt'

# 取3000个nt词和7000个非nt值
Length_nt = 3000
Length_other = 7000
Length = Length_nt + Length_other

# 文件位置
train_set = 'Train.txt'
train_out = 'train_pair.txt'

test_set = 'Test.txt'
test_out = 'test_pair.txt'

Validation_set = 'Validation.txt'
Validation_out = 'validation_pair.txt'

# 学习率和迭代次数
alpha = 1e-2
max_lte = 200


import os
from os import listdir
from zipfile import ZipFile
import img2pdf

#Get Current Working Directory
zipfolder = os.getcwd()

# read all zip files in folder
for zip_files in os.listdir(zipfolder):# 這三行可以用來避免'.DS_Store' problem in Mac
    if not zip_files.endswith(".zip"):
        if not zip_files.endswith(".rar"):
            continue

    clean_name = os.path.splitext(zip_files)[0]
    dirname = zipfolder+'/'+clean_name

# create a folder by zip_files's name
    if zip_files.endswith('.zip'):
        os.makedirs(clean_name)
# 把zip files的圖片unzip到上面建立的folder內
    if zip_files.endswith('.zip'):
        with ZipFile(zip_files, 'r') as zipObj:
            zipObj.extractall(dirname)

# 把PDF放在當前folder內，二選一
    with open(f"{clean_name}.pdf", "wb") as f:
        imgs = []
        for pdf_files in os.listdir(dirname):
            if not pdf_files.endswith(".jpg"):
                if not pdf_files.endswith(".png"):
                    continue
            path = os.path.join(dirname, pdf_files)
            if os.path.isdir(path):
                continue
            imgs.append(path)
            imgs.sort()
        f.write(img2pdf.convert(imgs))

# 把PDF放在圖片新開的folder內，二選一
    # pdf_path = zipfolder+'/'+'/'+clean_name+'/'+clean_name+'.pdf'
    # with open(pdf_path, "wb") as f:
    #     imgs = []
    #     for pdf_files in os.listdir(dirname):
    #         if not pdf_files.endswith(".jpg"):
    #             if not pdf_files.endswith(".png"):
    #                 continue
    #         path = os.path.join(dirname, pdf_files)
    #         if os.path.isdir(path):
    #             continue
    #         imgs.append(path)
    #         imgs.sort()
    #     f.write(img2pdf.convert(imgs))

# can using `rm .DS_Store` to detele this file first then run these code


'''
LeetCode #701 - Insert Into a Binary Search Tree prompt:

You are given the root node of a binary search tree (BST) and a
value to insert into the tree. Return the root node of the BST
after the insertion. It is guaranteed that the new value does
not exist in the original BST.

Notice that there may exist multiple valid ways for the
insertion, as long as the tree remains a BST after insertion.
You can return any of them.

Example 1:
Input: root = [4,2,7,1,3], val = 5
Output: [4,2,7,1,3,5]
Explanation: Another accepted tree is:

Example 2:
Input: root = [40,20,60,10,30,50,70], val = 25
Output: [40,20,60,10,30,50,70,null,null,25]

Example 3:
Input: root = [4,2,7,1,3,null,null,null,null,null,null], val = 5
Output: [4,2,7,1,3,5]

Constraints:
The number of nodes in the tree will be in the range [0, 10^4].
-10^8 <= Node.val <= 10^8
All the values Node.val are unique.
-10^8 <= val <= 10^8
It's guaranteed that val does not exist in the original BST.
'''
from typing import Optional
# Definition for a binary tree node.
class TreeNode:
	def __init__(self, val: int = 0, left = None, right = None):
		self.val = val
		self.left = left        # left < Parent Node
		self.right = right      # right > Parent Node

class Solution:
	# Time complexity:  O(log n) = O(h)
	# Space complexity: O(log n)
	def insertIntoBST(self, root: Optional[TreeNode], val: int) -> Optional[TreeNode]:
		if not root:    # if tree doesnt exist insert single node tree
			return TreeNode(val) # & if parent node points to NULL insert node

		if val < root.val:  # iterate left
			root.left = self.insertIntoBST(root.left, val)
		else:           # iterate right
			root.right = self.insertIntoBST(root.right, val)

		return root

# Ex 1:
# Input: root = [4,2,7,1,3], val = 5
# Output: [4,2,7,1,3,5]
root = TreeNode(4)
root.left = TreeNode(2)
root.left.left = TreeNode(1)
root.left.right = TreeNode(3)
root.right = TreeNode(7)
val = 5
result = Solution().insertIntoBST(root, val)
assert result.val == 4
assert result.left.val == 2
assert result.left.left.val == 1
assert result.left.right.val == 3
assert result.right.val == 7
assert result.right.left.val == 5

# Ex 2:
# Input: root = [40,20,60,10,30,50,70], val = 25
# Output: [40,20,60,10,30,50,70,null,null,25]
root = TreeNode(40)
root.left = TreeNode(20)
root.left.left = TreeNode(10)
root.left.right = TreeNode(30)
root.right = TreeNode(60)
root.right.left = TreeNode(50)
root.right.right = TreeNode(70)
val = 25
result = Solution().insertIntoBST(root, val)
assert result.val == 40
assert result.left.val == 20
assert result.left.left.val == 10
assert result.left.right.val == 30
assert result.right.val == 60
assert result.right.left.val == 50
assert result.right.right.val == 70
assert result.left.right.left.val == 25

# Ex 3:
# Input: root = [4,2,7,1,3,null,null,null,null,null,null], val = 5
# Output: [4,2,7,1,3,5]
root = TreeNode(4)
root.left = TreeNode(2)
root.left.left = TreeNode(1)
root.left.right = TreeNode(3)
root.right = TreeNode(7)
val = 5
result = Solution().insertIntoBST(root, val)
assert result.val == 4
assert result.left.val == 2
assert result.left.left.val == 1
assert result.left.right.val == 3
assert result.right.val == 7
assert result.right.left.val == 5

Trees/LeetCode #701 - Insert into a Binary Search Tree.py


import xml.etree.ElementTree as ET
import subprocess
import asyncio
import sys

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

async def execute11(line):
    cmd = "openssl s_client -connect %s < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin" % line
    fp_ = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
    stdout, stderr = await fp_.communicate()
    f[line] = stdout
    return f

print(f"""{bcolors.HEADER}
 _____ _           _ ____                        ____          _
 |  ___(_)_ __   __| / ___|  __ _ _ __ ___   ___ / ___|___ _ __| |_
 | |_  | | '_ \ / _` \___ \ / _` | '_ ` _ \ / _ \ |   / _ \ '__| __|
 |  _| | | | | | (_| |___) | (_| | | | | | |  __/ |__|  __/ |  | |_
 |_|   |_|_| |_|\__,_|____/ \__,_|_| |_| |_|\___|\____\___|_|   \__|
{bcolors.ENDC}"""
)

argc = len(sys.argv)
if argc == 1 or argc == 2:
	print('Usage:\n\tpython3 FindSameCert.py domain:port nmap.xml')
	print('For example:\n\tpython3 FindSameCert.py kkk.com:443 nmap.xml')
	sys.exit()
else:
	url = sys.argv[1]
	nmap = sys.argv[2]

tree = ET.parse(nmap)
root = tree.getroot()

SslPorts = []
ListDomCheck = {}
hosts = root.findall('host')

for host in hosts:
    try:
        HostNameElem = host.findall('hostnames')
        HostName = HostNameElem[0].findall('hostname')[0].attrib['name']
        PortElement = host.findall('ports')
        ports = PortElement[0].findall('port')
        for port in ports:
            service = port.findall('service')[0].attrib['name']
            if "tunnel" in port.findall('service')[0].attrib:
                _port = port.attrib['portid']
                SslPorts.append(_port)
            SslPortsCopy = SslPorts.copy()
            ListDomCheck[HostName] = SslPortsCopy
        SslPorts.clear()
    except:
        pass

DomainList = []
for i in ListDomCheck:
    for j in range(len(ListDomCheck[i])):
        line = i+":"+ListDomCheck[i][j]
        DomainList.append(line)

f = {}

cmd = "openssl s_client -connect %s < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin" %url
fp = subprocess.Popen([cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()[0]
print (f"{bcolors.HEADER}TLS certificate fingerprint for %s:{bcolors.ENDC}" %url)
print(fp.decode("utf-8"))

futures = [execute11(domain) for domain in DomainList]

loop = asyncio.get_event_loop()
fp_ = loop.run_until_complete(asyncio.wait(futures))

print(f"{bcolors.HEADER}The same TLS certificate is used on:{bcolors.ENDC}")
for k, v in f.items():
    if v == fp:
        print(f"{bcolors.BOLD}%s{bcolors.ENDC}" %k)


########################################
#Author: Gentry Atkinson
#Date: 05 Feb 2020
#Organization: Texas State University
#Description: randomly alter several labels in the MNIST CSVs
########################################

PLAIN_TRAIN_LABELS = "MNIST/Extracted Data/train_labels.csv"
PLAIN_TEST_LABELS = "MNIST/Extracted Data/test_labels.csv"
ALTERED_TRAIN_LABELS = "MNIST/Extracted Data/train_altered_labels.csv"
ALTERED_TRAIN_LABELS_INDEXES = "MNIST/Extracted Data/train_altered_label_indexes.csv"
ALTERED_TEST_LABELS = "MNIST/Extracted Data/test_altered_labels.csv"
ALTERED_TEST_LABELS_INDEXES = "MNIST/Extracted Data/test_altered_label_indexes.csv"

PERCENT_TO_ALTER = 1
NUM_TRAINS_SAMPLES = 60000
NUM_TEST_SAMPLES = 10000

import numpy as np
import random as rand
#from datetime import datetime

def main():
    train_labels = np.loadtxt(PLAIN_TRAIN_LABELS, delimiter=',')
    altered_labels = open(ALTERED_TRAIN_LABELS, "w+")
    altered_indexes = open(ALTERED_TRAIN_LABELS_INDEXES, "w+")
    #rand.seed(datetime.now())
    index = -1
    count = 0
    for label in train_labels:
        index = index + 1
        label = int(label)
        picker = rand.randint(0,100)
        if picker < PERCENT_TO_ALTER:
            newLabel = rand.randint(0,9)
            while newLabel == label:
                newLabel = rand.randint(0,9)
            altered_labels.write(str(newLabel))
            altered_labels.write("\n")
            altered_indexes.write(str(index))
            altered_indexes.write("\n")
            count += 1
            print("Index ", index, " changed from ", label, " to ", newLabel)
        else:
            altered_labels.write(str(label))
            altered_labels.write("\n")
    print(count, " total entries changed")

if __name__ == "__main__":
    main()


import scrapy
import re
from competition.items import CompetitionItem

class CiasiSpider(scrapy.Spider):
    name = 'ciasi'
    allowed_domains = ['ciasi.org.cn']
    # start_urls = ['http://www.ciasi.org.cn/Home/safety/index?sid=15&bid=&cid=&sss=1&year=51,50']
    start_urls = ['http://www.ciasi.org.cn/Home/safety/index?sid=15&bid=&cid=&sss=1&year=52,51,50,49']

    def parse(self, response):
        list_item = response.xpath("//div[@class='eval_by_item']")
        for item in list_item:
            items = CompetitionItem()
            next_url = item.xpath("./a/@href").extract_first()
            brand = item.xpath("./a/div[@class='ev_i_brand']/p/text()").extract_first()
            menu = item.xpath("./a/div[@class='ev_i_manu']/div/p/text()").extract_first()
            models = item.xpath("./a/div[@class='ev_i_models']/div/p/text()").extract_first()
            level = item.xpath("./a/div[@class='ev_i_level']/div/p/text()").extract_first()
            model = item.xpath("./a/div[@class='ev_i_model']/div/p/text()").extract_first()
            items['brand'] = brand
            items['menu'] = menu
            items['models'] = models
            items['level'] = level
            items['model'] = model
            items['l_safe'] = ''
            items['r_pro'] = ''
            items['b_eval'] = ''
            # yield items
            if next_url is not None:
                next_url = 'http://www.ciasi.org.cn/' + next_url
                yield scrapy.Request(
                    next_url,
                    callback= self.parse_detail,
                    meta = {'item': items}
                )

    def parse_detail(self,response):
        # TODO 主页数据
        main_item = response.meta['item']
        # TODO 左侧数据
        pur_list = response.xpath("//div[@class='pur_le_item']")
        safe_pur_list = {}
        for pur in pur_list:
            title_txt = pur.xpath("./div[1]/p/text()").extract_first()
            if title_txt is not None:
                img_src = pur.xpath("./div[2]/div/img/@src").extract_first()
                img_stat = str(img_src).split('/')[-1]
                if img_stat == 'icon-greenDi.png':
                    img_stat = '车辆标配'
                elif img_stat == 'icon-radCa.png':
                    img_stat = '车辆未配备'
                elif img_stat == 'icon-yellowQu.png':
                    img_stat = '车辆选配'
                safe_pur_list[title_txt] = img_stat
        main_item['l_safe'] = safe_pur_list

        # TODO 右侧数据
        pro_index_list = []
        main_list = response.xpath("//tr[contains(@class,'pur_hd_')]")
        for i in main_list:
            right_title = i.xpath(".//div[@class='pr_e_lt']/p/text()").extract_first()
            right_result = i.xpath(".//div[@class='ev_i_bs']/span/text()").extract_first()
            pro_index_list.append({right_title:right_result,"detail": {}})
        if len(pro_index_list) > 0:
            count = 0
            for cls in ["co_p_one","co_p_two","co_p_san","co_p_si"]:
                one_item = {}
                for one in response.xpath("//tr[@class='{}']".format(cls)):
                    one_title = one.xpath(".//div[@class='pr_t_xt']/p/text()").extract_first()
                    one_result = one.xpath(".//div[@class='co_pu_s']/span/text()").extract_first()
                    one_item[one_title] = one_result
                pro_index_list[count]["detail"] = one_item
                count += 1
        elif len(pro_index_list) == 0:
            main_title_list = response.xpath("//div[@class='pr_e_lt']/p/text()").extract()
            main_price_list = response.xpath("//div[@class='ev_i_bs']/span/text()").extract()
            item_title_list = response.xpath("//div[@class='pr_t_xt']/p/text()").extract()
            item_price_list = response.xpath("//div[@class='co_pu_s']/span/text()").extract()
            pro_list = []
            for ti,pr in zip(main_title_list,main_price_list):
                _item = {ti: pr,'detail':{}}
                pro_list.append(_item)

            pro_list[0]['detail'] = {item_title_list[0]:item_price_list[0]}
            pro_list[1]['detail'] = {t:p for t,p in zip(item_title_list[1:4],item_price_list[1:4])}
            pro_list[2]['detail'] = {item_title_list[5]:item_price_list[5]}
            pro_list[3]['detail'] = {item_title_list[6]:item_price_list[6]}
            pro_index_list += pro_list

        main_item['r_pro'] = pro_index_list

        #TODO 下层数据
        evaluation_details = response.xpath("//div[@class='pa_t_ds']/p/text() | //td[@class='p_h_les']/div//p/text()").extract()
        evaluation_price = response.xpath("//div[@class='pa_t_bz']/span/text() | //div[@class='pa_t_bz']/div[@class='pa_i_rg']/span/text()").extract()

        evaluation = [{de:pr} for de,pr in zip(evaluation_details,evaluation_price)]

        table_list = response.xpath("//div[@class='par_p_block']//table")
        eval_table_data = []
        price_image = table_list[0].xpath(".//div[@class='pa_i_rg']//img/@src").extract_first()
        for index,table in enumerate(table_list):
            title_list = table.xpath(".//div[@class='pa_i_le' or @class='pa_i_xl']/p/text()").extract()
            price_list = table.xpath(".//div[@class='pa_i_rg' or @class='co_pu_s']/span/text()").extract()
            price_image = str(price_image).split('/')[-1]
            if price_image == 'icon-greenDi.png':
                price_image = '气囊未起爆'
            elif price_image == 'icon-radCa.png':
                price_image = '气囊起爆'
            if title_list[0] == '正面碰撞得分（满分30分）':
                price_list.append(price_image)

            _table_data = [{title:price} for title,price in zip(title_list,price_list)]
            eval_table_data += _table_data
        price_car = response.xpath("//div[@class='pa_bt_le']/span/text()").extract_first()
        evaluation = evaluation + eval_table_data
        evaluation.append({'price':price_car})
        main_item['b_eval'] = evaluation

        price = response.xpath("//div[@class='pa_t_bt']/div[@class='pa_bt_le']/span/text()").extract_first()
        price = re.findall(r"\d+\.?\d*",price)
        main_item['price'] = price

        yield main_item
        # //div[@class="par_p_block"]//table//div[@class='pa_i_le' or @class='pa_i_xl']/p


from dotenv import load_dotenv
from logging import error
import telegram
import requests
import os
import logging
from telegram.ext import CommandHandler, Updater, Filters, MessageHandler
from telegram.message import Message
from pprint import pprint

load_dotenv()

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s, %(levelname)s, %(message)s, %(name)s'
)

TOKEN = os.getenv('TOKEN')
CHAT_ID = os.getenv('CHAT_ID')

bot = telegram.Bot(token=TOKEN)
updater = Updater(token=TOKEN)

URL = 'https://api.thecatapi.com/v1/images/search'

def start_work(update, context):
    chat = update.effective_chat
    name = update.message.chat.first_name
    button = telegram.ReplyKeyboardMarkup(
        [
            ['/newcat'],
            ['Выход']
        ], resize_keyboard=True
    )
    context.bot.send_message(
        chat_id=chat.id,
        text='спс, от души, брат-{}!'.format(name),
        reply_markup=button
    )
    context.bot.send_photo(chat.id, get_image())

def get_image():
    try:
        response = requests.get(URL)
    except Exception:
        logging.exception(f'Ошибка при запросе к основному API: {error}')
        NEW_URL = 'https://api.thedogapi.com/v1/images/search'
        response = requests.get(NEW_URL)
    rnd_cat = response.json()[0]['url']
    return rnd_cat

def put_image(update, context):
    chat = update.effective_chat
    context.bot.send_photo(chat.id, get_image())

def say_hi(update, context):
    chat = update.effective_chat
    context.bot.send_message(chat_id=chat.id, text='message')

def main():
    updater.dispatcher.add_handler(CommandHandler('start', start_work))
    updater.dispatcher.add_handler(CommandHandler('newcat', put_image))
    updater.dispatcher.add_handler(MessageHandler(Filters.text, say_hi))
    updater.start_polling(poll_interval=3.0)
    updater.idle()

if __name__ == '__main__':
    main()

def send_message(message):
    bot.send_message(chat_id=CHAT_ID, text=message)

text = 'Вам телеграмма!'

send_message(text)


import socket
import os
import multiprocessing

def send_message(s_socket, client_address):
    message = 'Hi ' + client_address[0] + ':' + str(client_address[1]) + '. This is server ' + str(
        os.getpid())
    s_socket.sendto(str.encode(message), client_address)
    print('Sent to client: ', message)

def test(x):
    return x * x

if __name__ == "__main__":

    number_processes = 4
    # Initialise the pool
    pool = multiprocessing.Pool(number_processes)

    server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    # Bind the socket to the port
    server_address = '127.0.0.1'
    server_port = 10001

    # Buffer size
    buffer_size = 1024

    server_socket.bind((server_address, server_port))
    print('Server up and running at {}:{}'.format(server_address, server_port))

    while True:
        data, address = server_socket.recvfrom(buffer_size)
        print('Received message \'{}\' at {}:{}'.format(data.decode(), address[0], address[1]))
        pool.apply_async(send_message, args=(server_socket, address,))


condicion = input('True or False: ')

if condicion == 'True':
    condicion = True
    print ('La condicion es Verdadera')
elif condicion == 'False':
    condicion = False
    print ('La condicion es Falsa')
else:
    print ('Parámetro no válido')
    condicion = input('Ingresa un parámetro válido (True/False): ')

print (f'Fin del programa')


"""
CP1404/CP5632 Practical
"""
import shutil
import os

def main():
    """Demo os module functions."""
    print("Starting directory is: {}".format(os.getcwd()))

    # Change to desired directory
    os.chdir('FilesToSort')

    # Print a list of all files in current directory
    print("Files in {}:\n{}\n".format(os.getcwd(), os.listdir('.')))

    extensions = []

    # Loop through each file in the (current) directory
    for filename in os.listdir('.'):
        # Ignore directories, just process files
        if os.path.isdir(filename):
            continue

        extension = filename.split(".", 1)[1]
        if extension not in extensions:
            os.mkdir(extension)
            extensions.append(extension)
        shutil.move(filename, extension + "/" + filename)
        print("Moved {} to {}".format(filename, extension))

main()


from rest_framework import serializers
from . import twitterhelper
from .register import register_social_user
import os
from rest_framework.exceptions import AuthenticationFailed

class TwitterAuthSerializer(serializers.Serializer):
    """Handles serialization of twitter related data"""
    access_token_key = serializers.CharField()
    access_token_secret = serializers.CharField()

    def validate(self, attrs):

        access_token_key = attrs.get('access_token_key')
        access_token_secret = attrs.get('access_token_secret')

        user_info = twitterhelper.TwitterAuthTokenVerification.validate_twitter_auth_tokens(
            access_token_key, access_token_secret)

        try:
            user_id = user_info['id_str']
            email = user_info['email']
            name = user_info['name']
            provider = 'twitter'
            first_token = attrs.get('access_token_key')
            second_toekn = attrs.get('access_token_secret')
        except:
            raise serializers.ValidationError(
                'The tokens are invalid or expired. Please login again.'
            )

        return register_social_user(
            provider=provider, user_id=user_id, email=email, name=name, first_token = first_token, second_token = second_token)


#!/usr/bin/env python3

# Created by Marwan Mashaly
# Created on October 2019
# This program tells user the month afte typing the number

def mailing(name, last_name, st_address, city, province, postal_code,
            apt_number=None):
    # This function organizes the mailing address to standardized

    name_address = name + " " + last_name
    name_address = name_address
    if apt_number != None:
        address = apt_number + " - " + st_address
    else:
        address = st_address
    address2 = city + " " + province + "  " + postal_code

    mailing_address = name_address.upper() + "\n" + address.upper() + "\n" + address2.upper()

    return mailing_address

def main():
    # This function gets input from user and calls other functions
    apt_number = None

    name = input("Enter your first name: ")
    last_name = input("Enter your last name: ")
    print("")
    question = input("Do you have an apartment number? (y/n): ")
    if question.upper() == "Y" or question.upper() == "YES":
        apt_number = input("Enter the apartment number : ")
    st_address = input("Enter the street address: ")
    city = input("Enter the city: ")
    province = input("Enter The province: ")
    postal_code = input("Enter the postal code: ")
    print("")

    if apt_number != None:
        address = mailing(name, last_name, st_address, city, province,
                          postal_code, apt_number)
    else:
        address = mailing(name, last_name, st_address, city, province,
                          postal_code)
    print(address)

if __name__ == "__main__":
    main()


from flask import request
from flask.json import jsonify
from . import habits
from .models import Habit
from .dbmodel import Habitdb
import json
from bson import json_util
from flask_jwt_extended import get_jwt_identity,jwt_required

@habits.route('/',methods=['POST'])
@jwt_required()
def create():
    current_user = get_jwt_identity()
    habit = Habit(**request.json)
    new_habit=Habitdb().create_habit(habit,current_user)
    if(new_habit):
        return jsonify(request.json),200
    return 'Error',409

@habits.route('/',methods=['GET'])
@jwt_required()
def get():
    current_user = get_jwt_identity()
    habit_db=Habitdb().get_all_habits(current_user)
    response=json.loads(json_util.dumps(habit_db))
    return jsonify(response),200

@habits.route('/<string:_id>',methods=['DELETE'])
@jwt_required()
def delete(_id):
    habit_db=Habitdb().delete_habit(_id)
    if(habit_db):
        return 'Sucess',200
    return 'Cant delete it',409

@habits.route('/<string:_id>',methods=['PUT'])
@jwt_required()
def edit(_id):
    habit=Habitdb().get_habit_Object(_id)
    habit.changes(**request.json)
    habit_db=Habitdb().update_habit(habit)
    if(habit_db):
        return jsonify(request.json),200
    return 'Cant edit anything',405

@habits.route('/today/<string:_id>',methods=['POST'])
@jwt_required()
def marktoday(_id):
    r=request.json
    habit=Habitdb().get_habit_Object(_id)
    try:
        if(r['mark']):
            habit.date.mark_today()
        else:
            habit.date.unmark_today()
        update_habit=Habitdb().update_habit(habit)
        if(update_habit):
            return 'Sucess',200
    except KeyError:
        return 'error in date',405


from PIL import Image

im1 = Image.open("lena.png")
im2 = Image.open("lena_modified.png")
#im.show()

width, height = im1.size
print(width)
print(height)

for y in range(height):
    for x in range(width):
        rgba1 = im1.getpixel( (x, y) )
        rgba2 = im2.getpixel( (x, y) )
        #print(rgba)
        if rgba1 == rgba2 :
            rgba2 = (rgba2[0] - rgba1[0],  # R
                     rgba2[1] - rgba1[1],  # G
                     rgba2[2] - rgba1[2],  # B
                     rgba2[3] - rgba1[3]);  # A
        im1.putpixel((x, y), rgba2)

im1.show()
im1.save("new.png")


from typing import Annotated, List, Optional

from mypy_extensions import TypedDict

RaiderDictionary = TypedDict("RaiderDictionary", {"address": str, "ranking": int})
RaiderWithAgentDictionary = TypedDict(
    "RaiderWithAgentDictionary", {"address": str, "ranking": int, "agent_address": str}
)
CurrencyDictionary = TypedDict(
    "CurrencyDictionary",
    {"ticker": str, "decimalPlaces": int, "minters": Optional[List[str]]},
)
RewardDictionary = TypedDict(
    "RewardDictionary", {"quantity": str, "currency": CurrencyDictionary}
)
RankingRewardDictionary = TypedDict(
    "RankingRewardDictionary",
    {"raider": RaiderDictionary, "rewards": List[RewardDictionary]},
)
RankingRewardWithAgentDictionary = TypedDict(
    "RankingRewardWithAgentDictionary",
    {"raider": RaiderWithAgentDictionary, "rewards": List[RewardDictionary]},
)
RecipientRow = Annotated[List[str], 8]
AmountDictionary = TypedDict(
    "AmountDictionary",
    {
        "ticker": str,
        "decimalPlaces": int,
        "quantity": int,
    },
)
Recipient = TypedDict("Recipient", {"recipient": str, "amount": AmountDictionary})


import spacy
import re
import pickle
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import classification_report
from sklearn.pipeline import Pipeline
from sklearn.decomposition import TruncatedSVD
from gensim.models import Word2Vec
import numpy as np

nlp = spacy.load("en_core_web_sm")

def lematizar_documento(ruta_doc, nombre_archivo_salida):
    try:
        archivo_df = open(ruta_doc, 'rb')
        df = pickle.load(archivo_df)

        for i in range(len(df)):
            tokens = nlp(df.loc[i, 'doc'])
            lista_lemmas = []
            for t in tokens:
                lista_lemmas.append(t.lemma_)
            df.loc[i, 'doc'] = " ".join(lista_lemmas)

        salida = open(nombre_archivo_salida, 'wb')
        pickle.dump(df, salida)

        archivo_df.close()
        salida.close()
    except FileNotFoundError:
        print("No se encontro el archivo")

def quitar_sw_documento(ruta_doc, nombre_archivo_salida):
    try:
        archivo_df = open(ruta_doc, 'rb')
        df = pickle.load(archivo_df)

        for i in range(len(df)):
            tokens = nlp(df.loc[i, 'doc'])
            lista_palabras = []
            for t in tokens:
                if not t.is_stop:  # and t.pos_ != "DET" and t.pos_ != "ADP" and t.pos_ != "CONJ" and t.pos_ != "PRON":
                    lista_palabras.append(t.text)
            texto = " ".join(lista_palabras)
            texto = re.sub('\s+', ' ', texto)
            texto = texto.strip()
            df.loc[i, 'doc'] = texto

        salida = open(nombre_archivo_salida, 'wb')
        pickle.dump(df, salida)
        print(df.loc[0, 'doc'])
        archivo_df.close()
        salida.close()
    except FileNotFoundError:
        print("No se encontro el archivo")

def limpia_texto(ruta_doc, nombre_archivo_salida):
    try:
        archivo_df = open(ruta_doc, 'rb')
        df = pickle.load(archivo_df)

        for i in range(len(df)):
            tokens = nlp(df.loc[i, 'doc'])
            lista_palabras = []
            for t in tokens:
                if not t.is_punct:
                    lista_palabras.append(t.text)
            texto = " ".join(lista_palabras)
            texto = re.sub('\s+', ' ', texto)
            texto = texto.strip()
            df.loc[i, 'doc'] = texto

        salida = open(nombre_archivo_salida, 'wb')
        pickle.dump(df, salida)
        print(df.loc[0, 'doc'])
        archivo_df.close()
        salida.close()
    except FileNotFoundError:
        print("No se encontro el archivo")

# Normalizacion combinaciones :
# quitar_sw_documento("test.pkl", "test_nsw.pkl")
# quitar_sw_documento("train.pkl", "train_nsw.pkl")

# lematizar_documento("test.pkl", "test_lemma.pkl")
# lematizar_documento("train.pkl", "train_lemma.pkl")

# limpia_texto('test.pkl','test_limp.pkl')
# limpia_texto('train.pkl','train_limp.pkl')

# lematizar_documento("test_nsw.pkl","test_nsw_lemma.pkl")
# lematizar_documento("train_nsw.pkl","train_nsw_lemma.pkl")

# quitar_sw_documento("test_lemma.pkl","test_lemma_nsw.pkl")
# quitar_sw_documento("train_lemma.pkl","train_lemma_nsw.pkl")

# limpia_texto('test_nsw.pkl','test_nsw_limp.pkl')
# limpia_texto('train_nsw.pkl','train_nsw_limp.pkl')

# lematizar_documento("test_limp.pkl","test_limp_lemma.pkl")
# lematizar_documento("train_limp.pkl","train_limp_lemma.pkl")

# lematizar_documento("test_nsw_limp.pkl","test_nsw_limp_lemma.pkl")
# lematizar_documento("train_nsw_limp.pkl","train_nsw_limp_lemma.pkl")

def get_x_y_data(ruta_doc):
    try:
        archivo_df = open(ruta_doc, 'rb')
        df = pickle.load(archivo_df)
        data = df['doc']
        target = df['label']
        archivo_df.close()

        return data, target
    except FileNotFoundError:
        print("No se encontro el archivo")

def rep_vec(data_train, data_test, vectorizer):
    vectors_train = vectorizer.fit_transform(data_train)
    vectors_test = vectorizer.transform(data_test)
    print(vectorizer.get_feature_names_out())
    print(len(vectorizer.get_feature_names_out()))
    return vectors_train, vectors_test

def get_model(route_train):
    X_train, y_train = get_x_y_data(route_train)
    sentences = [sentence.lower().split() for sentence in X_train]
    model = Word2Vec(sentences, vector_size = 300, window = 16, min_count = 5, workers = 4)
    # workers = number of threads

    return model

def vectorize_embedding(sentence, w2v_model):
    words = sentence.split()
    words_vecs = [w2v_model.wv[word] for word in words if word in w2v_model.wv]
    if len(words_vecs) == 0:
        return np.zeros(300)
    words_vecs = np.array(words_vecs)
    return words_vecs.mean(axis=0)

def classify(clf, route_train, route_test, vectorizer=None):
    X_train, y_train = get_x_y_data(route_train)
    X_test, y_test = get_x_y_data(route_test)

    if vectorizer != None:
        vectors_train, vectors_test = rep_vec(X_train, X_test, vectorizer)
        clf.fit(vectors_train, y_train)
        y_pred = clf.predict(vectors_test)
    else:
        model = get_model(route_train)
        X_train = np.array([vectorize_embedding(sentence.lower(), model) for sentence in X_train])
        X_test = np.array([vectorize_embedding(sentence.lower(), model) for sentence in X_test])
        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
    print(y_pred)
    print(classification_report(y_test, y_pred))

def classify_svd(clf, route_train, route_test, vectorizer):
    X_train, y_train = get_x_y_data(route_train)
    X_test, y_test = get_x_y_data(route_test)
    pipe = Pipeline([('text_representation', vectorizer), ('dimensionality_reduction', TruncatedSVD(300)),
                     ('classifier', clf)])
    pipe.set_params(dimensionality_reduction__n_components = 1000)
    print(pipe)
    pipe.fit(X_train, y_train)
    y_pred = pipe.predict(X_test)
    print(classification_report(y_test, y_pred))

vectorizer_bin = CountVectorizer(binary=True)
vectorizer_freq = CountVectorizer()
vectorizer_tfidf = TfidfVectorizer()

clf_NB = MultinomialNB()
clf_LR = LogisticRegression(max_iter = 300)
clf_MLPC = MLPClassifier()
clf_SGD = SGDClassifier()

# Pruebas con los clasificadores:

print()
print()
print("No Stop Words + lemmatization : tfidf , SGD")
classify(clf_SGD, 'train_nsw_lemma.pkl', 'test_nsw_lemma.pkl', vectorizer_tfidf)
print()
print("No Stop Words + clean text + lemmatization : tfidf , SGD")
classify(clf_SGD, 'train_nsw_limp_lemma.pkl', 'test_nsw_limp_lemma.pkl', vectorizer_tfidf)

print()
print()
print("No Stop Words + lemmatization : tfidf , SGD , SVD")
classify_svd(clf_SGD, 'train_nsw_lemma.pkl', 'test_nsw_lemma.pkl', vectorizer_tfidf)
print()
print("No Stop Words + clean text + lemmatization : tfidf , SGD , SVD")
classify_svd(clf_SGD, 'train_nsw_limp_lemma.pkl', 'test_nsw_limp_lemma.pkl', vectorizer_tfidf)


import stripe
from stripe.error import AuthenticationError, InvalidRequestError
from django.conf import settings

# from .models import StripeConfig, SillyStripeConfig

# color parameters: style;background (30 is none);foreground
color = {
    "end": "\x1b[0m",
    "info": "\x1b[0;30;36m",
    "success": "\x1b[0;30;32m",
    "warning": "\x1b[0;30;33m",
    "danger": "\x1b[0;30;31m",
}

DSS_CONFIG_ERROR = (
    f"{color['warning']}DJANGO-SILLY-STRIPE IS NOT CONFIGURED PROPERLY."
    "\nCheck the configuration in the admin panel."
    f"{color['end']}"

    )

SILLY_STRIPE = {
    # Basic settings
    'AUTO_SET': 'CLASSIC',  # 'SPA' or 'CLASSIC'
    'DSS_SECRET_KEY': 'sk_xxxxxx',
    'DSS_PUBLIC_KEY': 'pk_xxxxxx',
    'DSS_RESTRICTED_KEY': 'rk_xxxxxx',  # optionnal
    'DSS_WEBHOOK_SECRET': 'wk_xxxxxx',
    'DSS_PREFIX': 'dss/',
    # Django Silly Stripe Endpoints
    'USE_CHECKOUT': True,
    'USE_SUBSCRIPTIONS_CANCEL': True,
    'USE_WEBHOOK': True,
    'USE_PORTAL': True,
    # Checkout settings
    'SUCCESS_URL': 'https://example.com/checkout_success',
    'CANCEL_URL': 'https://example.com/checkout_cancel',
    # Subscriptions settings
    'SUBSCRIPTION_CANCEL': 'PERIOD',  # 'PERIOD' or 'NOW' (beware with 'NOW': no refund)
    'SUBSCRIBE_ONLY_ONCE': True,
    # Portal settings
    'PORTAL_BACK_URL': 'https://example.com/back_from_portal',
    # Misc
    'PRINT_DEV_LOGS': False,
}

for key in settings.SILLY_STRIPE:
    SILLY_STRIPE[key] = settings.SILLY_STRIPE[key]


# Project I. Drawing turtle
# Problem 3. Draw a 5 x 8 dot grid with a nested loop.

import turtle

# create a turtle
bob = turtle.Turtle()

# define the shape of the grid
dot_distance = 20
width = 5
height = 8

# pen up so we don't draw when moving between dots
bob.penup()

for row in range(8):
    row_width = 0
    for col in range(5):
        bob.dot()
        bob.forward(dot_distance)
        row_width += dot_distance

    # go to the starting position of the current row
    bob.backward(row_width)

    # move to the beginning of the next row
    bob.right(90)
    bob.forward(dot_distance)
    bob.left(90)

# We're done!
turtle.done()