import numpy as np
import pandas as pd
from sklearn import tree
from sklearn.ensemble import (AdaBoostClassifier, GradientBoostingClassifier,
                              RandomForestClassifier)
from sklearn.feature_extraction.text import (CountVectorizer,
                                             HashingVectorizer,
                                             TfidfVectorizer)
from sklearn.linear_model import (LogisticRegression,
                                  PassiveAggressiveClassifier)
from sklearn.metrics import (accuracy_score, f1_score, precision_score,
                             recall_score)
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
"""
@author: yaoyue123
2019/11/26
社会计算（跨）第一次作业demo
"""

def read_train_valid(filename):
    """
    读取训练或者验证文件
    :param filename: 训练集/验证集的文件名字
    :return:
    返回训练集的文本和标签
    其中文本是一个list, 标签是一个list(每个元素为int)
    返回示例：['我很开心', '你不是真正的快乐', '一切都是假的], [1, 0, 0]
    """

    fp = pd.read_table(filename, sep='\t', error_bad_lines=False)

    return fp['review'].tolist(), list(map(int, fp['sentiment'].tolist()))

def read_test(filename):
    """
    读取测试文件
    :param filname: 测试集文件名字
    :return:
    返回测试集文本和对应的id编号
    其中文本是一个list, id就是文件中的id
    返回示例：['我很开心', '你不是真正的快乐', '一切都是假的], [1,2,3]
    """

    fp = pd.read_table(filename, sep='\t', error_bad_lines=False)

    return fp['review'].tolist(), fp['id'].tolist()

def split_text(text_data):
    """
    将原始文本分词
    例如：输入['今天星期四', '明天有雨'],
             返回[['今天','星期四'], ['明天','有','雨']]

    或者例如：输入['I don't want to go', ''Who are you?’],
            返回[['I', 'don't', 'want', 'to', 'go',],['Who', ' are', 'you',' ?']]
        中文推荐jieba分词(请pip install jieba)，英文直接用空格切分句子即可
    :param text_data:
    :return:
    """
    #英文文本无需分词，直接返回即可
    return text_data

#此特征提取方法最好
def vectorizer(train_data, valid_data, test_data):
    """
    将原始文本转化为向量
    可选方法
    1. one hot 方法, 2. tf-idf方法，3. 词向量或者其他方法
    推荐使用1或者2的方法，两种方法都可以选择调用第三方包或者自己实现
    可用的包：
    CountVectorizer, TfidfVectorizer可以实现one-hot和tfd-idf编码
    使用tf-idf比onehot有加分; 自己实现比调用第三方包有额外加分
    愿意使用词向量也可
    返回示例：[[0,0,1,2], [2,3,3,4]]
                   [[0,0,1,2], [2,3,3,4]]
                    [[0,0,1,2], [2,3,3,4]]
    分别对应train, valid, test的向量化结果，每一个结果都是一个二维的list, 每一个元素都是int or float
    :param text:  一个list, 每个元素是一句话
    :return: 3个二维list
    """
    #经测试启用stop_words反而准确率会下降， ngram_range=(1, 3)的时候准确零最高
    count_vectorizer = CountVectorizer(stop_words=None, ngram_range=(1, 3))
    count_train = count_vectorizer.fit_transform(train_data)
    count_valid = count_vectorizer.transform(valid_data)
    count_test = count_vectorizer.transform(test_data)
    return count_train, count_valid, count_test

#评分函数
def score(valid_label, pred):
    return accuracy_score(valid_label, pred), precision_score(
        valid_label, pred), recall_score(valid_label,
                                         pred), f1_score(valid_label, pred)

def train_valid(train_x, train_label, valid_x, valid_label):
    """
    开始训练模型并使用验证集验证效果
    推荐方法
    1. 朴素贝叶斯: 可用包 BernoulliNB,
    2. LogisticRegression 可用包 sklearn中的LogisticsRegression
    3. 其他方法不做强制性要求
    以上方法，自己实现比调包加分，手动实现 LR比手动实现朴素贝叶斯加分
    调包示例：
        model = one_model()
        model.some_func(train_data, train_label)
        score = model.score_function(valid_data, valid_label)
        print("valid accuracy score is {:.4f}".format(score}

        以上只计算了accuracy score，如果能打印验证集precision_score, recall_score, F1_score， 加分
        可以调用 sklearn.metric 中的precision_score, recall_score, F1_score
        请试着调节模型的参数，将你的各项score尽量提高，验证集分数越高，测试集的效果也可能越好
        也推荐同学们尝试交叉验证方法，提升模型的泛化能力
        最终返回训练完成的模型 model
    :param train_data:
    :param train_label:
    :param valid_data:
    :param valid_label:
    :return:
    """
    """
    # 定义模型
    model = LogisticRegression()
    # 开始训练,使用训练集的train_x 和train_y
    model.fit(train_x, train_label)
    # 训练完毕，预测一下验证集
    prediction = model.predict(valid_x)
    # 打印预测的结果和得分
    print(prediction)
    print('Accuracy score: {:.4f}'.format(
        accuracy_score(valid_label, prediction)))
    print('Precision score: {:.4f}'.format(
        precision_score(valid_label, prediction)))
    print('Recall score: {:.4f}'.format(recall_score(valid_label, prediction)))
    print('F1 score: {:.4f}'.format(f1_score(valid_label, prediction)))

    return model
    """

    # 逻辑回归算法
    LR_model = LogisticRegression()
    LR_model.fit(train_x, train_label)
    pred = LR_model.predict(valid_x)
    print("\nLogisticRegression-score: \nAc\tPr\tRe\tF1\n",
          score(valid_label, pred))

    return LR_model

def predict(model, test_data):
    """
    使用训练好的模型预测测试数据
    返回预测的标签，为list
    :param mode:
    :param test_data:
    :return:
    """
    result = model.predict(test_data)
    print("test result", result)
    return result

def run_step():
    """
        选择相应的任务和文件
        读文件，train_data, train_label = some_function(filename='')
                    valid_data, validlabel = some_function(filename='')
                   test_data, test_ids = some_function(filename='')
        将原始文本分词：
                  train_data = split_function(train_data)
                  valid_data = split_function(valid_data)
                  test_data = split_function(test_data)
        将分词后的文本变成向量：
                  train_vec, valid_vec, test_vec = vectorizer(train_data, valid_data, test_data)
        开始训练模型：
                model = train_function(train_vec, train_label, valid_vec, valid_label)
        开始预测结果：
                 test_result = predict(mode, test_data)
        写入文件：
                将test和对应的test写入test_result, 结果样本如submit所示
        请大家尽量按照步骤来，顺利完成整个流程。有余力的同学可以试试提高预测的精度
    :return:
    """
    train_sentences, train_label = read_train_valid(filename=r'training.tsv')
    test_sentences, test_id = read_test(filename=r'test.tsv')
    valid_sentences, valid_label = read_train_valid(filename=r'validation.tsv')

    train_sentences_split = split_text(train_sentences)
    test_sentences_split = split_text(test_sentences)
    valid_sentences_split = split_text(valid_sentences)
    """
    # 遍历所有的句子，并打印编号和句子内容
    print(1, train_sentences, train_label)
    print(2, test_sentences, test_id)
    print(3, valid_sentences, valid_label)
    print(4, train_sentences_split)
    print(
        5,
        vectorizer(train_sentences_split, valid_sentences_split,
                   test_sentences_split))
    """
    train_x, valid_x, test_x = vectorizer(train_sentences_split,
                                          valid_sentences_split,
                                          test_sentences_split)

    model = train_valid(train_x, train_label, valid_x, valid_label)

    test_result = predict(model, test_x)

    with open("submit.tsv", "w", encoding='utf-8') as fp:
        fp.write('id' + '\t' + 'sentiment' + '\n')
        for i in range(len(test_result)):
            sentiment = str(test_result[i])
            id = str(test_id[i])
            fp.write(id + '\t' + sentiment + '\n')

if __name__ == '__main__':
    run_step()


import cv2
from addPadding import *
from dividingImage import *
from pickleAndUnpickle import *
from testingAndTrainingPaths import *
from settings import *
from printDataBase import *
from skimage.io import sift
from matplotlib import pyplot as plt

def constructTrainingArray(fontsList, keyPointsFileName, partitioningType, constantWindowParameters, slidingWindowParameters):

    print("Constructing the database...")
    trainingDataPathsForAllFonts = []
    for font in fontsList:
        trainingDataPathsForOneFont = getTrainingImagesPaths(Font=font)
        trainingDataPathsForAllFonts.append(trainingDataPathsForOneFont)
    listForAllFonts=[]
    i = 0
    for eachFont in trainingDataPathsForAllFonts:
        listForFont = []
        for eachChar in eachFont:
            j = 0
            listForChar = []
            for eachShape in eachChar:
                listForShape = []
                img = cv2.imread(eachShape, 0)
                # cv2.imshow('img'+str(i),img)
                # cv2.waitKey(0)
                paddedImage = addPadding(img, horizontalPadding, verticalPadding)
                # cv2.imshow('img'+str(i+1),paddedImage)
                # cv2.waitKey(0)
                if partitioningType == "constant":
                    parts = divideImage(paddedImage, constantWindowParameters["n"], constantWindowParameters["m"])
                elif partitioningType == "sliding":
                    parts = slidingWindow(paddedImage, slidingWindowParameters["winWidth"]
                                          , slidingWindowParameters["winHeight"],
                                          slidingWindowParameters["shift"],
                                          slidingWindowParameters["shiftDirection"])

                ImagePartsKeyPoints_array = []
                for part in parts:
                    subImage = getSubImageData(paddedImage, part)
                    # Initiate SIFT detector
                    sift = cv2.xfeatures2d.SIFT_create()
                    kp1, des1 = sift.detectAndCompute(subImage, None)
                    temp =  pickle_keypoints(kp1, des1)
                    listForShape.append(temp)
                    # listForShape.append([kp1,des1])
                    # Initiate ORB detector
                    # orb = cv2.ORB_create()
                    # find the keypoints and descriptors with ORB
                    # kp1, des1 = orb.detectAndCompute(subImage, None)
                char, pos = getCharWithPositionFromPath(eachShape)
                listForChar.append((int(pos)-1,listForShape))
            listForFont.append(listForChar)
        listForAllFonts.append(listForFont)
    # print(listForAllFonts[0][1])
    storeToFile(databaseFolderName+keyPointsFileName, listForAllFonts)
    print("Database is successfully constructed.")


import pandas as pd
import os
import yaml
from util import Path, run_command

class Sanity_check(object):
    def __init__(self, config) -> None:
        self.samples = config['samples']
        self.path_to_dreem_output_files = config['path_to_dreem_output_files']
        self.sample_file = self.path_to_dreem_output_files+'samples.csv'
        self.library_file = self.path_to_dreem_output_files+'library.csv'
        self.path_to_rnastructure = config['rnastructure']['path']
        self.verbose = config['verbose']
        self.path = Path()
        self.config = config

    def files(self):
        # check that every file is there
        for s in self.samples:
            assert os.path.exists(self.path_to_dreem_output_files+s+'.csv'), f"{self.path_to_dreem_output_files+s+'.csv'} not found"
        if self.config['use_library']:
            assert os.path.exists(self.library_file), f"{self.library_file} not found"
        if self.config['use_samples']:
            assert os.path.exists(self.sample_file), f"{self.sample_file} not found"
        if self.config['use_rnastructure']:
            assert os.path.exists(self.path_to_rnastructure), f"{self.path_to_rnastructure} not found"

    def check_samples(self):
        if self.verbose: print(f"Checking {self.sample_file}")
        # check the sanity of samples.csv
        df = pd.read_csv(self.sample_file)
        assert len(df['sample']) == len(df['sample'].unique()), "Every line isn't unique in samples.csv"

        # check that every sample as a corresponding line of samples.csv
        df['sample'] = df['sample'].astype(str)

        for s in self.samples:
            assert s in list(df['sample']), f"{s, type(s)} doesn't have a corresponding line in samples.csv"

        with open(self.path.sample_attribute_path, 'r') as f:
            sample_attributes = yaml.safe_load(f)

        assert len(df['exp_env'].unique()) == 1, "exp_env is not unique in samples.csv"
        exp_env = df['exp_env'].unique()[0]

        # check that every column of samples.csv is in sample_attributes.yml
        assert exp_env in ['in_vivo','in_vitro'], f"{exp_env} is not a valid value for exp_env. Should be in_vivo or in_vitro"
        # Check that you have all mandatory columns

        for mand in sample_attributes['mandatory']['all'] + sample_attributes['mandatory'][exp_env]:
            assert mand in list(df.columns), f"{mand} is not in samples.csv"

        # check that every mandatory column of samples.csv is not empty for every sample
        for mand in sample_attributes['mandatory']['all'] + sample_attributes['mandatory'][exp_env]:
            for s in self.samples:
                assert df[df['sample']==s][mand].isnull().sum() == 0, f"{mand} is empty in samples.csv for sample {s}"

        df.to_csv(self.config['temp_folder']+'/samples.csv', index=False)
        if self.verbose: print('Checking samples.csv done\n')
        return 1

    def check_library(self):
        # check the sanity of libraries.csv
        if self.verbose: print(f"Checking library.csv")
        assert os.path.exists(self.library_file), f"{self.library_file} not found"
        df = pd.read_csv(self.library_file)
        assert 'construct' in list(df.columns), "construct is not in library.csv"
        df.to_csv(f"{self.config['temp_folder']}/library.csv", index=False)
        if self.verbose: print(f"Checking library.csv done\n")

    def make_folder(self):
       if not os.path.exists( self.config['temp_folder']):
            os.makedirs( self.config['temp_folder'])

    def run(self):
        self.make_folder()
        if self.verbose: print("Checking files")
        self.files()
        if self.config['use_library']:
            self.check_library()
        if self.config['use_samples']:
            self.check_samples()
        if self.verbose: print("Checking files done\n")
        return 1


# Standard Packages
import json
import os

# Custom Packages
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import streamlit as st

# Modules
import dataset

st.title("NLP Dataset Streamlit")

# Select an existing/supported datasets
if dataset.supported[0]["id"] is not None:
    dataset.supported.insert(0, {"id": None, "name": ""})  # add a blank entry

selected_dataset = st.sidebar.selectbox(
    "Choose a file", dataset.supported, format_func=lambda x: x["name"]
)

# Read the dataset as a DataFrame
# Or upload a file
st.sidebar.write("or")

uploaded_file = st.sidebar.file_uploader("Upload a file")

if uploaded_file is not None:
    print(upload_file)

# Load dataset
data = []

if selected_dataset["id"] is not None:
    # List available files
    files = dataset.list_files(selected_dataset["directory"])
    selected_files = st.sidebar.multiselect(
        "Select set",
        files,
        default=None if len(files) > 1 else files[0],
        format_func=os.path.basename,
    )

    data_load_state = st.text("Loading data...")
    data = dataset.load_files(selected_files)
    data_load_state.text("Loading data... done!")

# Options
if data:
    # Raw data
    raw_data = st.sidebar.checkbox("Raw data", value=True)
    if raw_data:
        n_rows = st.sidebar.number_input("# of rows", min_value=1, value=10)

    # Sequence length
    sequence_length = st.sidebar.checkbox("Sequence length", value=True)
    if sequence_length:
        cols = zip(data[0]["df"].columns, data[0]["df"].dtypes)
        cols = [col[0] for col in cols if col[1] is np.dtype("O")]
        sequence_length_columns = st.sidebar.multiselect("Select columns", cols)
        n_bins = st.sidebar.number_input("# bins", min_value=1, value=20)

for entry in data:
    st.header(entry["name"])
    df = entry["df"]

    # Simple stats
    st.write(f"Number of elements: {len(df)}")

    # Display raw data
    if raw_data:
        st.subheader("Raw data")
        st.write(df.sample(min(n_rows, len(df))))

    # Histogram of sequence length
    if sequence_length:
        fig, ax = plt.subplots(1, 1)

        for col in sequence_length_columns:
            ax.hist(
                [
                    len(elem.split() if hasattr(elem, "split") else elem)
                    for elem in df[col]
                ],
                bins=n_bins,
                histtype="bar",
                alpha=0.5,
            )

        ax.set_xlabel("Length")
        ax.set_ylabel("Frequency")
        ax.grid()
        ax.legend(sequence_length_columns)

        st.pyplot(fig)


import numpy as np
import random
import operator

from approaches.abstract_approach import AbstractApproach
from api.instance import Instance
from api.insertion import Insertion

class Random(AbstractApproach):

    def __init__(self, instance, seed = None):
        self.instance = instance
        self.seed = seed
        self.examples_num_first = np.ceil(instance.matrix.shape[0] / 2)
        self.examples_num_second = instance.matrix.shape[0] - self.examples_num_first
        self.algorithm = 'Random Cycle'
        self._first_solution = list()
        self._second_solution = list()

    def solve(self, first_start_node, second_start_node):
        random.seed(self.seed)
        self._first_solution, self._second_solution = self._find_solution(first_start_node, second_start_node)

    def _find_solution(self, first_start_node, second_start_node):
        first_cycle = [first_start_node]
        second_cycle = [second_start_node]

        while len(set(first_cycle)) < self.examples_num_first or len(set(second_cycle)) < self.examples_num_second:
            if len(set(first_cycle)) < self.examples_num_first:
                nodes_not_in_cycle = list(set(range(self.instance.matrix.shape[0])) - set(first_cycle) - set(second_cycle))
                first_cycle += [random.choice(nodes_not_in_cycle)]

            if len(set(second_cycle)) < self.examples_num_second:
                nodes_not_in_cycle = list(set(range(self.instance.matrix.shape[0])) - set(first_cycle) - set(second_cycle))
                second_cycle += [random.choice(nodes_not_in_cycle)]

        first_cycle += [first_start_node]
        second_cycle += [second_start_node]

        return first_cycle, second_cycle


from tkinter import *
import mysql.connector
root=Tk()
root.geometry("600x400")
root.title("Registration Form")

def register():
    print("Name :",first.get())
    print("Father :",father.get())
    print("Surname :",surname.get())
    print("Roll :",roll.get())
    print("Gender :",rad.get())
    print("Stadard :",std.get())
    print("Contact :",contact.get())
    connection_db=mysql.connector.connect(host="localhost",user="root",password="",database="tkinter")

    cursor_b=connection_db.cursor()

    sql = "INSERT INTO `registration` (`First Name`, `Father Name`, `Surname`, `Roll No`, `Standard`, `Gender`, `Contact No`) VALUES ( %s, %s, %s, %s, %s, %s, %s)"

    val = (first.get(), father.get(), surname.get(),roll.get(), std.get(), rad.get(),contact.get(), )

    cursor_b.execute(sql, val)

    connection_db.commit()

f1=Frame(root,borderwidth=2,relief=SOLID)
f1.pack()

l0=Label(text="Fill the Details to Register for Dance Competition",fg='white',bg='black',padx=10,pady=20,font='arial 24 bold')
l0.pack(fill='x',padx=10,pady=10)

f2=Frame(root,borderwidth=2,relief=SOLID,padx=10,pady=20)
f2.pack(padx=20,pady=20)

first=StringVar()
father=StringVar()
surname=StringVar()
roll=StringVar()
rad=StringVar()
std=StringVar()
contact=StringVar()
rad.set('none')

Label(f2,text='First Name :',font='arial 18',fg='gray',pady=3,).grid(row=0,column=0)
Entry(f2,textvariable=first,font='arial 18',fg='gray',).grid(row=0,column=1,pady=3)

Label(f2,text='Father Name :',font='arial 18',fg='gray',pady=3,).grid(row=1,column=0)
Entry(f2,textvariable=father,font='arial 18',fg='gray',).grid(row=1,column=1,pady=3)

Label(f2,text='Surname  :',font='arial 18',fg='gray',pady=3,).grid(row=2,column=0)
Entry(f2,textvariable=surname,font='arial 18',fg='gray',).grid(row=2,column=1,pady=3)

Label(f2,text='Roll No. :',font='arial 18',fg='gray',pady=3,).grid(row=3,column=0)
Entry(f2,textvariable=roll,font='arial 18',fg='gray',).grid(row=3,column=1,pady=3)

Label(f2,text='Standard :',font='arial 18',fg='gray',pady=3,).grid(row=4,column=0)
Entry(f2,textvariable=std,font='arial 18',fg='gray',).grid(row=4,column=1,pady=3)

li=['Male', 'Female', 'Other']

Label(f2,text='Gender :',font='arial 18',fg='gray',pady=3,).grid(row=5,column=0)
Radiobutton(f2,text=li[0],variable=rad,value=li[0],font="arial 12",fg='gray').grid(row=6,column=0)

Radiobutton(f2,text=li[1],variable=rad,value=li[1],font="arial 12",fg='gray').grid(row=6,column=1)

Radiobutton(f2,text=li[2],variable=rad,value=li[2],font="arial 12",fg='gray').grid(row=6,column=2)

Label(f2,text='Contact :',font='arial 18',fg='gray',pady=3,).grid(row=7,column=0)
Entry(f2,textvariable=contact,font='arial 18',fg='gray',).grid(row=7,column=1,pady=3)

Button(f2,text="Register Now",font='arial 18',fg='gray',pady=5,command=register).grid(row=8,columnspan=3,pady=15)

root.mainloop()


f1 = f = open("1.txt", "w", encoding="utf-8")

def write_array(array):
    ans = array[0].split("(")[-1][:-1]
    f1.write(array[0] + "@")
    for c in ans:
        f1.write(array[["A", "B", "C", "D", "E"].index(c) + 1] + "  ")
    f1.write("\n")

f = open("卫生法.txt", "r", encoding="utf-8")
lines = f.readlines()
f.close()
array = []

for line in lines:
    line = line.strip()

    if line == "":
        write_array(array)
        array = []
    else:
        array.append(line)

f1.close()
print("Done.")


import glob
import matplotlib.pyplot as plt
import numpy as np
import random
import pickle

from confusionMetrics import plot_confusion_matrix
from ffmpeginput import input
from hmmlearn import hmm
from pprint import pprint
from sklearn.metrics import precision_recall_fscore_support, recall_score, precision_score, classification_report, confusion_matrix, f1_score, accuracy_score
from sklearn.model_selection import LeaveOneOut
from seqlearn.hmm import MultinomialHMM
from seqlearn.perceptron import StructuredPerceptron
from timeRemarks import outputSteps

# module pickle for loading data from text file
with open("data_random_heuristic.txt", 'rb') as data:
    ground_truth_sampled = pickle.load(data)

#----------------seperate random, heuristic sequence for evaluation-------------
y_random = ground_truth_sampled[-2]
y_heuristic = ground_truth_sampled[-1]

# training data
ground_truth_dim = ground_truth_sampled[:13]

# common action set
step_set = set.intersection(*map(set, ground_truth_dim))

# number of streams
number_files = len(ground_truth_dim)

# array for average of confusion matrix
average_cm = np.zeros((len(step_set),len(step_set)))

#----------get the verbs (low-level activity) of the actions in streams----------
observation, len_observation = [], []
# Function to split the step names and get the verbs
def get_actions(streams):
    verb = []
    for i in streams:
        oldKey = i.split('-')
        verb.append(oldKey[0])
    return verb

for i in ground_truth_sampled:
    observation.append(list(get_actions(i)))
    len_observation.append(len(i))

# action set of observations,
action_set = set.intersection(*map(set, observation))

#-----------------------------------HMM------------------------------------------
# do one-hot encoding for the action_set = {'twist_on', 'switch_on', 'put', 'pour', 'stir', 'take', 'open', 'none'}
O =  []
for x in observation:
    o = []
    for i in x:
        if i == 'take':
            o.append([1,0,0,0,0,0,0,0])
        elif i == 'open':
            o.append([0,1,0,0,0,0,0,0])
        elif i == 'stir':
            o.append([0,0,1,0,0,0,0,0])
        elif i == 'pour':
            o.append([0,0,0,1,0,0,0,0])
        elif i == 'put':
            o.append([0,0,0,0,1,0,0,0])
        elif i == 'switch_on':
            o.append([0,0,0,0,0,1,0,0])
        elif i == 'twist_on':
            o.append([0,0,0,0,0,0,1,0])
        elif i == 'none':
            o.append([0,0,0,0,0,0,0,1])
    O.append(np.array(o))

# array mit encoded action streams
X = np.array(O)
X_random = X[-2]
X_heuristic = X[-1]
X = X[:13]

#-----------------------leave one out cross validation--------------------------
# sciki-learn fct for LeaveOneOut
loo = LeaveOneOut()

len_train, len_test, y_pred_hmmlearn, y_pred_seqlearn, y_seqlearn_de = [],[],[],[],[]
y_ground_truth_seqlearn = []

# lists for evaluation
precision_seqlearn, recall_seqlearn, accuracy_seqlearn, f1_seqlearn = [], [], [], []
prob_results_seqlearn, prob_results_random_seq_seqlearn, prob_results_heuristic_seqlearn = [], [], []

# splitting the data into testset and trainingset
for train, test in loo.split(X):
    print("%s %s" % (train, test))
    # crossvalidation test und training set
    X_train, X_test = X[train], X[test]
    # ground_truth for evaluation
    y_ground_truth  = ground_truth_sampled[np.asscalar(np.array([test]))]

    # for seqlearn
    for i in train:
        y_ground_truth_seqlearn.append(ground_truth_sampled[np.asscalar(np.array([i]))])

    [len_train.append(len(i)) for i in X_train]
    [len_test.append(len(i)) for i in X_test]

    # flatten X_train for HMM function
    X_train_flatten = [item for sublist in X_train for item in sublist]
    X_test_flatten = [item for sublist in X_test for item in sublist]

    # flatten X_train for HMM function
    y_ground_truth_flatten= [item for sublist in y_ground_truth_seqlearn for item in sublist]

    # change type to array
    seqlearn_X_train = np.array(X_train_flatten)
    seqlearn_y_ground_truth = np.array(y_ground_truth_flatten)

    # HMM seqlearn MultimodalHMM
    model_seqlearn = MultinomialHMM()

    # training
    model_seqlearn.fit(seqlearn_X_train, seqlearn_y_ground_truth, len_train)

     # state prediction
    y_pred_seqlearn = model_seqlearn.predict(X_test_flatten)

    # print output time remarks
    outputSteps(y_pred_seqlearn)

    #state prediction for random sequence
    y_pred_seqlearn_random = model_seqlearn.predict(X_random)

    # state prediction for heuristic sequence
    y_pred_seqlearn_random = model_seqlearn.predict(X_heuristic)

    # target names for evaluation
    target_names = list(step_set)
    target_names = sorted(target_names)

    # confusion matrix
    cnf_matrix = confusion_matrix(y_ground_truth, y_pred_seqlearn)
    normalize  = cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]
    average_cm = average_cm + normalize

    y_ground_truth_seqlearn, y_seqlearn_de = [], []
    len_train, len_test = [], []

    #------------------------------evaluation HMM-------------------------------
    precision_seqlearn.append(precision_score(y_ground_truth, y_pred_seqlearn, average='weighted'))
    recall_seqlearn.append(recall_score(y_ground_truth, y_pred_seqlearn, average='weighted'))
    f1_seqlearn.append(f1_score(y_ground_truth, y_pred_seqlearn, average='weighted'))
    accuracy_seqlearn.append(accuracy_score(y_ground_truth, y_pred_seqlearn))

# print perfomance scores
print('Precision')
pprint(np.mean(precision_seqlearn))
print('Recall')
pprint(np.mean(recall_seqlearn))
print('F1-score')
pprint(np.mean(f1_seqlearn))
print('Accuracy')
pprint(np.mean(accuracy_seqlearn))

# calculate and print avergae confusion matrix
average_cm = average_cm / number_files
plt.figure()
plot_confusion_matrix(average_cm, classes=target_names, normalize=True,
                  title='Normalized confusion matrix')

plt.show()


import sys, os, pdb
import json
import shutil, errno
from tqdm import tqdm
import pandas as pd
from utils.constant import *

class PreProcessData(object):
    """docstring for PreProcessData"""
    def __init__(self):
        super(PreProcessData, self).__init__()
        self.data_dir = "/path/to/where/the/raw/dataset/is"
        self.save_dir = "/path/to/store/the/processed/dataset/" # e.g. ./data/processed/Dialogue-Summarization

    def _load_json(self, path=None):
        if path is None or not os.path.exists(path):
            raise IOError('File does not exist: %s' % path)
            # return None
        with open(path) as df:
            data = json.loads(df.read())
        return data

    def _load_txt(self, path=None, split_tok="\n"):
        if path is None or not os.path.exists(path):
            raise IOError('File does not exist: %s' % path)
        with open(path) as df:
            data = df.read().strip().split(split_tok)
        return data

    def _load_csv(self, path=None, sep="\t"):
        if path is None or not os.path.exists(path):
            raise IOError('File does not exist: %s' % path)
        with open(path) as df:
            data = pd.read_csv(df, sep=sep)
        return data

    def _load_jsonl(self, path=None):
        if path is None or not os.path.exists(path):
            raise IOError('File does not exist: %s' % path)
        data = []
        with open(path) as df:
            for line in df.readlines():
                data.append(json.loads(line))
        return data

    def _load_dir_json(self, dir_path=None):
        if dir_path is None or not os.path.exists(dir_path): return None
        total_data = [] # assume data is a list of dialogs
        for filename in sorted(os.listdir(dir_path)):
            if filename in ["schema.json"]: continue
            if not filename.endswith(".json"): continue
            file_path = os.path.join(dir_path, filename)
            data = self._load_json(path=file_path)
            if type(data) == list:
                total_data.extend(data)
            else:
                total_data.append(data)
        return total_data

    def _load_dir_txt(self, dir_path=None, file_type="txt"):
        if dir_path is None or not os.path.exists(dir_path): return None
        total_data = [] # assume data is a list of dialogs
        for filename in sorted(os.listdir(dir_path)):
            if not filename.endswith(file_type): continue
            file_path = os.path.join(dir_path, filename)
            data = self._load_txt(path=file_path)
            if type(data) == list:
                total_data.extend(data)
            else:
                total_data.append(data)
        return total_data

    def _load_dir_tsv(self, dir_path=None, sep="\t"):
        if dir_path is None or not os.path.exists(dir_path): return None
        total_data = None
        for filename in sorted(os.listdir(dir_path)):
            file_path = os.path.join(dir_path, filename)
            data = self._load_csv(path=file_path, sep=sep)
            total_data = pd.concat([total_data, data], ignore_index=True)
        return total_data

    def _save_json(self, data, path):
        with open(path, "w") as tf:
            json.dump(data, tf, indent=4)

    def init_dial(self, dial_idx=0, ori_dial_id=""):
        dial = {
            ORI_DIAL_ID: ori_dial_id,
            DIAL_IDX: dial_idx,
            ORI_DIAL_INFO: {},
            LOG: [],
            PROMPT: [],
        }
        return dial

    def init_turn(self, turn_id=0, dial_hist=[]):
        turn = {
            TURN_ID: turn_id,
            USR_UTT: "",
            SYS_UTT: "",
            DIAL_HIST: " ".join(dial_hist),
            ORI_USR_ANN: {},
            ORI_SYS_ANN: {},
        }
        return turn

    def save_dial(self, data, data_name="", file_idx=0, mode="train"):
        save_name = f"dialogues_{file_idx}.json"
        folder_path = os.path.join(self.save_dir, data_name, mode)
        if not os.path.exists(folder_path): os.makedirs(folder_path)
        path = os.path.join(folder_path, save_name)
        self._save_json(data, path)

    def copy_general(self, src, dst):
        try:
            shutil.copytree(src, dst, dirs_exist_ok=True)
        except OSError as exc: # python >2.5
            if exc.errno in (errno.ENOTDIR, errno.EINVAL):
                shutil.copy(src, dst)
            else: raise

    def copy_related_files(self, data_name, exp_list=[], extra_dir=""):
        source_dir = os.path.join(self.data_dir, data_name, extra_dir)
        target_dir = os.path.join(self.save_dir, data_name)
        for filename in os.listdir(source_dir):
            if filename.startswith("."): continue # ignore hidden files
            if filename.startswith("__"): continue # ignore hidden files
            if filename in exp_list: continue
            if filename.endswith(".py"): continue
            source_path = os.path.join(source_dir, filename)
            target_path = os.path.join(target_dir, filename)
            self.copy_general(source_path, target_path)

    def save_original_examples(self, examples, data_name):
        """
        save 5 original data points just for reference and check
        data would be a list of length 5, each entry is a dialog
        in the form of dictionary
        """
        path = os.path.join(self.save_dir, data_name, "original_examples.json")
        self._save_json(examples, path)
        print("original examples saved")

    def save_converted_examples(self, data_name):
        """
        extract the first 5 examples from the train set of the
        already processed data, just for reference and check
        """
        data = self._load_json(os.path.join(self.save_dir, data_name, "train/dialogues_1.json"))
        examples = {key: data[key] for key in list(data.keys())[:5]}
        self._save_json(examples, os.path.join(self.save_dir, data_name, "converted_examples.json"))
        print("converted examples saved")

    def _import_system_file(self, filename="", module_name=""):
        import importlib, sys
        spec = importlib.util.spec_from_file_location(module_name, filename)
        module = importlib.util.module_from_spec(spec)
        sys.modules[module_name] = module
        spec.loader.exec_module(module)
        return module

    def tweetsum(self):
        """
        real data store in kaggle, need to download and preprocess first
        """
        data_name = "TweetSumm"
        # prepare data
        Modules = self._import_system_file(os.path.join(self.data_dir, data_name, "tweet_sum_processor.py"), "TweetSumProcessor")
        processor = Modules.TweetSumProcessor(os.path.join(self.data_dir, data_name, "archive/twcs/twcs.csv"))
        exp_list = ["tweet_sum_data_files", "archive", "tweet_sum_processor.py"]
        for mode in ["train", "val", "test"]:
            real_name = f"final_{mode}_tweetsum.jsonl" if mode != "val" else "final_valid_tweetsum.jsonl"
            path = os.path.join(self.data_dir, data_name, "tweet_sum_data_files", real_name)

            # split = self._load_jsonl(path)
            new_data = {}
            file_idx = 1
            original_data_sample = []

            with open(path) as f:
                dialog_with_summaries = processor.get_dialog_with_summaries(f.readlines())
                for dial_idx, dialog_with_summary in tqdm(enumerate(dialog_with_summaries)):
                    new_dial_id = f"{data_name}--{mode}--{dial_idx+1}"

                    json_format = dialog_with_summary.get_json()
                    dial = json.loads(json_format)
                    if mode == "train" and dial_idx < 5:
                        original_data_sample.append(dial)

                    new_dial = self.init_dial(dial_idx=dial_idx+1, ori_dial_id=dial["dialog"]["dialog_id"]) # idx starts from 1
                    new_dial[ORI_DIAL_INFO] = {
                        "summaries" : dial["summaries"]
                    }
                    turn_id, dial_hist = 1, []
                    new_turn = self.init_turn(turn_id=turn_id)
                    for idx, turn in enumerate(dial["dialog"]["turns"]):
                        utt = " ".join(turn["sentences"])
                        if turn["is_agent"]:
                            new_turn[SYS_UTT] += f" {utt}"
                            new_turn[SYS_UTT] = new_turn[SYS_UTT].strip()
                            if idx == len(dial["dialog"]["turns"]) - 1 or \
                                not dial["dialog"]["turns"][idx+1]["is_agent"]:

                                new_dial[LOG].append(new_turn)
                                turn_id += 1
                                if new_turn[USR_UTT]:
                                    dial_hist.append("<USER> " + new_turn[USR_UTT])
                                dial_hist.append("<SYSTEM> " + new_turn[SYS_UTT])
                                new_turn = self.init_turn(turn_id=turn_id)
                                new_turn[DIAL_HIST] = " ".join(dial_hist)
                        else:
                            new_turn[USR_UTT] += f" {utt}"
                            new_turn[USR_UTT] = new_turn[USR_UTT].strip()

                    new_data[new_dial_id] = new_dial
                    if (dial_idx+1) % 1000 == 0 or dial_idx+1 == len(dialog_with_summaries):
                        self.save_dial(new_data, data_name=data_name, file_idx=file_idx, mode=mode)
                        new_data = {} # reset
                        file_idx += 1

            if mode == "train": self.save_original_examples(original_data_sample, data_name)
        self.save_converted_examples(data_name)
        self.copy_related_files(data_name, exp_list)
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def samsum(self):
        """
        1. achieved from HF datasets "samsum"
        2. no sys/user, but two human being, assuming the first utterance comes from user, ignore residual
        """
        data_name = "SAMSum"
        # prepare data
        from datasets import load_dataset
        data = load_dataset("samsum")
        for mode in ["train", "val", "test"]:
            real_name = mode if mode != "val" else "validation"
            new_data, file_idx = {}, 1

            for dial_idx, dial in tqdm(enumerate(data[real_name])):
                new_dial_id = f"{data_name}--{mode}--{dial_idx+1}"
                new_dial = self.init_dial(dial_idx=dial_idx+1, ori_dial_id=dial["id"]) # idx starts from 1
                new_dial[ORI_DIAL_INFO] = {
                    "summary" : dial["summary"]
                }
                dial_hist = []
                sep = "\r\n" if "\r\n" in dial["dialogue"] else "\n"
                for turn_idx, turn in enumerate(dial["dialogue"].split(sep)):
                    speaker, utt = turn.split(": ")[0], ": ".join(turn.split(": ")[1:])
                    if turn_idx % 2 == 0:
                        new_turn = self.init_turn(turn_id=turn_idx//2+1)
                        new_turn[DIAL_HIST] = " ".join(dial_hist)
                        new_turn[USR_UTT] = utt.strip().replace("  ", " ")
                        new_turn[ORI_USR_ANN]['speaker'] = speaker
                    else:
                        new_turn[SYS_UTT] = utt.strip().replace("  ", " ")
                        new_turn[ORI_SYS_ANN]['speaker'] = speaker
                        dial_hist.append("<USER> " + new_turn[USR_UTT])
                        dial_hist.append("<SYSTEM> " + new_turn[SYS_UTT])
                        new_dial[LOG].append(new_turn)

                new_data[new_dial_id] = new_dial
                if (dial_idx+1) % 1000 == 0 or dial_idx+1 == len(data[real_name]):
                    self.save_dial(new_data, data_name=data_name, file_idx=file_idx, mode=mode)
                    new_data = {} # reset
                    file_idx += 1

        self.save_original_examples(data["train"][:5], data_name)
        self.save_converted_examples(data_name)
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def dialogsum(self):
        """
        1. we use the data from github: https://github.com/cylnlp/dialogsum/tree/main/DialogSum_Data
            but, it is also available from HF datasets "knkarthick/dialogsum"
        2. no sys/user, but two human being, assuming the first utterance comes from user, ignore residual
        """
        data_name = "DialogSum"

        for mode in ["train", "val", "test"]:
            real_name = mode if mode != "val" else "dev"
            path = os.path.join(self.data_dir, data_name, f"DialogSum_Data/dialogsum.{real_name}.jsonl")
            data = self._load_jsonl(path)
            new_data, file_idx = {}, 1

            for dial_idx, dial in tqdm(enumerate(data)):
                new_dial_id = f"{data_name}--{mode}--{dial_idx+1}"
                new_dial = self.init_dial(dial_idx=dial_idx+1, ori_dial_id=dial["fname"]) # idx starts from 1
                for key in dial:
                    if key in ["fname", "dialogue"]: continue
                    new_dial[ORI_DIAL_INFO][key] = dial[key]

                dial_hist = []
                turns = dial["dialogue"].replace("PErson","Person").split("#Person")[1:]
                for turn_idx, turn in enumerate(turns):
                    speaker, utt = turn.split("#:")
                    speaker = "Person" + speaker
                    utt = utt.replace("\n","").strip()

                    if turn_idx % 2 == 0:
                        new_turn = self.init_turn(turn_id=turn_idx//2+1)
                        new_turn[DIAL_HIST] = " ".join(dial_hist)
                        new_turn[USR_UTT] = utt.strip()
                        new_turn[ORI_USR_ANN]['speaker'] = speaker.replace("#","")
                    else:
                        new_turn[SYS_UTT] = utt.strip()
                        new_turn[ORI_SYS_ANN]['speaker'] = speaker.replace("#","")
                        dial_hist.append("<USER> " + new_turn[USR_UTT])
                        dial_hist.append("<SYSTEM> " + new_turn[SYS_UTT])
                        new_dial[LOG].append(new_turn)

                new_data[new_dial_id] = new_dial
                if (dial_idx+1) % 1000 == 0 or dial_idx+1 == len(data):
                    self.save_dial(new_data, data_name=data_name, file_idx=file_idx, mode=mode)
                    new_data = {} # reset
                    file_idx += 1

            if mode == "train": self.save_original_examples(data[:5], data_name)
        self.save_converted_examples(data_name)
        self.copy_related_files(data_name, ['Baseline'])
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def ami(self):
        """
        download processed data from https://drive.google.com/drive/folders/1BbmaZnzG9WrqOO-D3h211NOJePotqwQJ
        the data is separated into 6 files based on annotation
        here we extract the dialog context based on file "dialogueActs"
        no train/val/test split, consider all as train
        no readme file needs to be copied
        we use ABCD instead of USR_UTT/SYS_UTT

        1. each dialog contains more than 2 speaker? yes A,B,C,D
        2. speaking in any order? yes A->B->C->D
        """
        data_name = "AMI"
        mode = "train"
        data_dir = os.path.join(self.data_dir, data_name, "dialogueActs")
        new_data, dial_idx = {}, 1

        for filename in os.listdir(data_dir):
            dial = self._load_json(os.path.join(data_dir, filename))
            new_dial = self.init_dial(dial_idx=dial_idx) # idx starts from 1
            # # # save dialog log
            new_dial[ORI_DIAL_INFO]["dialog history"] = []
            for turn in dial:
                new_dial[ORI_DIAL_INFO]["dialog history"].append(turn["speaker"] + " : " + turn["text"])

            # # # save abstractive summary
            if os.path.exists(os.path.join(self.data_dir, data_name, "abstractive", filename)):
                abs_sum = self._load_json(os.path.join(self.data_dir, data_name, "abstractive", filename))
                new_dial[ORI_DIAL_INFO]["abstractive summary"] = abs_sum
            # # # save extractive summary
            if os.path.exists(os.path.join(self.data_dir, data_name, "extractive", filename)):
                ext_sum = self._load_json(os.path.join(self.data_dir, data_name, "extractive", filename))
                new_dial[ORI_DIAL_INFO]["extractive summary"] = []
                for ext_turn in ext_sum:
                    new_dial[ORI_DIAL_INFO]["extractive summary"].append(ext_turn["speaker"] + " : " + ext_turn["text"])

            new_dial_id = f"{data_name}--{mode}--{dial_idx}"
            new_dial[ORI_DIAL_ID] = filename
            new_data[new_dial_id] = new_dial
            dial_idx += 1
            if dial_idx == 2:
                self.save_original_examples(dial, data_name)

        self.save_dial(new_data, data_name=data_name, file_idx=1, mode=mode)
        self.save_converted_examples(data_name)
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def icsi(self):
        """
        similar as AMI
        speak can last to A->J
        """
        data_name = "ICSI"
        mode = "train"
        data_dir = os.path.join(self.data_dir, data_name, "dialogueActs")
        new_data, dial_idx = {}, 1

        for filename in os.listdir(data_dir):
            dial = self._load_json(os.path.join(data_dir, filename))
            new_dial = self.init_dial(dial_idx=dial_idx) # idx starts from 1
            # # # save dialog log
            new_dial[ORI_DIAL_INFO]["dialog history"] = []
            for turn in dial:
                new_dial[ORI_DIAL_INFO]["dialog history"].append(turn["speaker"] + " : " + turn["text"])

            # # # save abstractive summary
            if os.path.exists(os.path.join(self.data_dir, data_name, "abstractive", filename)):
                abs_sum = self._load_json(os.path.join(self.data_dir, data_name, "abstractive", filename))
                new_dial[ORI_DIAL_INFO]["abstractive summary"] = abs_sum
            # # # save extractive summary
            if os.path.exists(os.path.join(self.data_dir, data_name, "extractive", filename)):
                ext_sum = self._load_json(os.path.join(self.data_dir, data_name, "extractive", filename))
                new_dial[ORI_DIAL_INFO]["extractive summary"] = []
                for ext_turn in ext_sum:
                    new_dial[ORI_DIAL_INFO]["extractive summary"].append(ext_turn["speaker"] + " : " + ext_turn["text"])

            new_dial_id = f"{data_name}--{mode}--{dial_idx}"
            new_dial[ORI_DIAL_ID] = filename
            new_data[new_dial_id] = new_dial
            dial_idx += 1
            if dial_idx == 2:
                self.save_original_examples(dial, data_name)

        self.save_dial(new_data, data_name=data_name, file_idx=1, mode=mode)
        self.save_converted_examples(data_name)
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def qmsum(self):
        data_name = "QMSum"
        for mode in ["train", "val", "test"]:
            path = os.path.join(self.data_dir, data_name, f"data/ALL/{mode}")
            data = self._load_dir_json(path)
            new_data, file_idx = {}, 1
            for dial_idx, dial in tqdm(enumerate(data)):
                new_dial_id = f"{data_name}--{mode}--{dial_idx+1}"
                new_dial = self.init_dial(dial_idx=dial_idx+1)
                for key_ in dial:
                    if key_ == "meeting_transcripts": continue
                    new_dial[ORI_DIAL_INFO][key_] = dial[key_]

                new_dial[ORI_DIAL_INFO]["dialog history"] = []
                for turn in dial["meeting_transcripts"]:
                    new_dial[ORI_DIAL_INFO]["dialog history"].append(turn["speaker"] + " : " + turn["content"])

                new_data[new_dial_id] = new_dial
                if (dial_idx+1) % 1000 == 0 or dial_idx+1 == len(data):
                    self.save_dial(new_data, data_name=data_name, file_idx=file_idx, mode=mode)
                    new_data = {} # reset
                    file_idx += 1

            if mode == "train": self.save_original_examples(data[:5], data_name)
        self.save_converted_examples(data_name)
        self.copy_related_files(data_name, ['Baseline'])
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def mediasum(self):
        data_name = "MediaSum"
        split_id = self._load_json(os.path.join(self.data_dir, data_name, "data/train_val_test_split.json"))
        data = self._load_json(os.path.join(self.data_dir, data_name, "data/news_dialogue.json"))

        split_id2mode, new_data, file_idx, dial_idx = {}, {}, {}, {}
        for mode in ["train", "val", "test"]:
            for dial_id in split_id[mode]:
                split_id2mode[dial_id] = mode
            new_data[mode], file_idx[mode], dial_idx[mode] = {}, 1, 1

        for dial in tqdm(data):
            new_dial = self.init_dial() # idx starts from 1
            new_dial[ORI_DIAL_ID] = dial['id']
            for key_ in dial:
                if key_ in ["id", "utt", "speaker"]: continue
                new_dial[ORI_DIAL_INFO][key_] = dial[key_]
            dialog_log = []
            for idx in range(len(dial["utt"])):
                dialog_log.append(dial["speaker"][idx] + " : " + dial["utt"][idx])
            new_dial[ORI_DIAL_INFO]["dialog history"] = dialog_log

            mode = split_id2mode.get(dial["id"], "train")
            new_dial_id = f"{data_name}--{mode}--{dial_idx[mode]}"
            new_dial[DIAL_IDX] = dial_idx[mode]
            new_data[mode][new_dial_id] = new_dial
            dial_idx[mode] += 1

            if len(new_data[mode]) == 1000:
                self.save_dial(new_data[mode], data_name=data_name, file_idx=file_idx[mode], mode=mode)
                new_data[mode] = {} # reset
                file_idx[mode] += 1

        # if there are some unsaved dialogs left, save it now
        for mode in ["train", "val", "test"]:
            if new_data[mode]:
                self.save_dial(new_data[mode], data_name=data_name, file_idx=file_idx[mode], mode=mode)

        self.save_original_examples(data[:5], data_name)
        self.save_converted_examples(data_name)
        self.copy_related_files(data_name, ["data"])
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def crd3(self):
        """
        For this dataset, we choose present only chunk_size=2 offset=0
        some file are missing for chunk size = 2
        """
        data_name = "CRD3"
        exp_list = []
        for filename in os.listdir(os.path.join(self.data_dir, data_name)):
            if filename == "readme.txt": continue
            if filename == "LICENSE": continue
            exp_list.append(filename)
        for mode in ["train", "val", "test"]:
            new_data, file_idx, dial_idx = {}, 1, 1
            for file_name in self._load_txt(os.path.join(self.data_dir, data_name, f"data/aligned data/{mode}_files")):
                file_path = os.path.join(self.data_dir, data_name, f"data/aligned data/c=2/{file_name}_2_0.json")
                if not os.path.exists(file_path): continue
                data = self._load_json(file_path)

                new_dial_id = f"{data_name}--{mode}--{dial_idx}"
                new_dial = self.init_dial(dial_idx=dial_idx)
                new_dial[ORI_DIAL_ID] = file_name
                new_dial[ORI_DIAL_INFO] = data
                new_data[new_dial_id] = new_dial
                dial_idx += 1

                if (dial_idx) % 1000 == 0:
                    self.save_dial(new_data, data_name=data_name, file_idx=file_idx, mode=mode)
                    new_data = {} # reset
                    file_idx += 1
            if new_data: self.save_dial(new_data, data_name=data_name, file_idx=file_idx, mode=mode)
            if mode == "train": self.save_original_examples([new_dial[ORI_DIAL_INFO]], data_name)
        self.save_converted_examples(data_name)
        self.copy_related_files(data_name, exp_list)
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def ectsum(self):
        data_name = "ECTSum"
        for mode in ["train", "val", "test"]:
            new_data, file_idx, dial_idx = {}, 1, 1
            data_dir = os.path.join(self.data_dir, data_name, "data/final", mode)
            for file_name in os.listdir(os.path.join(data_dir, "ects")):
                if not file_name.endswith("txt"): pdb.set_trace()
                ect_data = self._load_txt(os.path.join(data_dir, "ects", file_name))
                sum_data = self._load_txt(os.path.join(data_dir, "gt_summaries", file_name))

                new_dial_id = f"{data_name}--{mode}--{dial_idx}"
                new_dial = self.init_dial(dial_idx=dial_idx)
                new_dial[ORI_DIAL_INFO]["file_name"] = file_name
                new_dial[ORI_DIAL_INFO]["ect"] = ect_data
                new_dial[ORI_DIAL_INFO]["summary"] = sum_data
                new_data[new_dial_id] = new_dial
                dial_idx += 1

                if (dial_idx) % 1000 == 0:
                    self.save_dial(new_data, data_name=data_name, file_idx=file_idx, mode=mode)
                    new_data = {} # reset
                    file_idx += 1
            if new_data: self.save_dial(new_data, data_name=data_name, file_idx=file_idx, mode=mode)
            if mode == "train": self.save_original_examples([new_dial[ORI_DIAL_INFO]], data_name)
        self.save_converted_examples(data_name)
        self.copy_related_files(data_name, ['codes', 'data'])
        print("*"*10, f"finishing processing dataset {data_name}", "*"*10)

    def run_all(self):
        # self.todsum()
        # self.tweetsum()
        # self.samsum()
        # self.dialogsum()
        # self.ami()
        # self.icsi()
        # self.qmsum()
        self.mediasum()
        # self.crd3()
        # self.ectsum()
        pass

    def copy_example(self):
        source_dir = self.save_dir
        target_dir = "/home/qkun/projs/TOD-Project/Datasets/Dialogue-Summarization_PROCESSED/"
        file_list = ["converted_examples.json", "original_examples.json", "readme.txt", "LICENSE"]
        for dir_name in sorted(os.listdir(source_dir)):
            if os.path.isfile(os.path.join(source_dir, dir_name)): continue
            if not os.path.exists(os.path.join(target_dir, dir_name)): os.makedirs(os.path.join(target_dir, dir_name))
            for filename in file_list:
                source_path = os.path.join(source_dir, dir_name, filename)
                target_path = os.path.join(target_dir, dir_name, filename)
                if not os.path.exists(source_path): continue
                shutil.copy(source_path, target_path)

def main():
    preprocess = PreProcessData()
    preprocess.run_all()
    preprocess.copy_example()

if __name__ == '__main__':
    main()


acidez = float(input("Ingresar acidez: "))
materiaOrganica = float(input("Ingresar materia organica: "))
catAcidez = ""
catMateria = ""

if 5.6 <= acidez <= 6.5:
    catAcidez = "SUMAMENTE APTO"
elif 6.6 <= acidez <= 7.0 or 5.1 <= acidez <= 5.5:
    catAcidez = "MODERADAMENTE APTO"
elif 7.1 <= acidez <= 8.0 or 4.5 <= acidez <= 5.0:
    catAcidez = "MARGINALMENTE APTO"
elif acidez > 8.0 or acidez < 4.5:
    catAcidez = "NO APTO"
# categoria materia organica
if 0 <= materiaOrganica <= 100:
    if materiaOrganica > 5:
        catMateria = "SUMAMENTE APTO"
    elif 4.1 <= materiaOrganica <= 5:
        catMateria = "MODERADAMENTE APTO"
    elif 3 <= materiaOrganica <= 4:
        catMateria = "MARGINALMENTE APTO"
    elif materiaOrganica < 3:
        catMateria = "NO APTO"
else:
    print("porcentaje no valido")
# Salida
if catMateria == catAcidez:
    print(catMateria)
else:
    if acidez < materiaOrganica:
        print(catAcidez)
    else:
        print(catMateria)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0020_schedule_scheduletime'),
    ]

    operations = [
        migrations.AlterField(
            model_name='schedule',
            name='day',
            field=models.CharField(max_length=3, choices=[('MON', 'Monday'), ('TUE', 'Tuesday'), ('WED', 'Wednesday'), ('THU', 'Thursday'), ('FRI', 'Friday'), ('SAT', 'Saturday'), ('SUN', 'Sunday')]),
        ),
    ]


'''
for this task, we want to ensure that the program follows the instructions of a BASIC program. in this subtask, a sample input would be "findLine(['10 GOTO 20','20 END'], '10')".
the output should be '0' because the second parameter, 10, is found in the 0th index of the list (the first parameter).
'''

def findLine(prog, target):
   for i in prog:
      x=i.split()    #here we're making a new array from a string so we can access the 0th element of the string
      if x[0] == target:  #for each string in the array, check the 0th element. if it equals our target, tell us the index of the string it was found in
         return prog.index(i)

'''
old explanation: (i do like the visual though!)

translation:
-so first we set up the function with two parameters, prog and target. target is what we want to find in prog.
-notice in the example input how each index in the list is a phrase with spaces, i.e. '10 GOTO 20' and '20 END'.
-in order to locate where in 'prog' the 'target' shows up, we need to make a list out of each element in 'prog' to separate out each word or number.
-then we'll get '10 GOTO 20' => ['10','GOTO','20'] and now we can try matching each of those elements to our 'target'.
-to do this, a for loop works. for each element (i) in prog:...
-assign a variable to hold each split element from prog (i.e. 1 split element would be ['10','GOTO','20']. another would be ['20,'END']. i used 'x' as my variable.
-now we have to set a condition. if the first element in our new list, 'x', matches our target, then...
-this gets a bit confusing (at least for me), so I'll try visualizing it:

                      prog => ['10 GOTO 20', '20 END']    this is one list
                      index:       0             1
                                  /                 \
                                 /                   \
                      x=> ['10','GOTO','20'], ['20','END']   this is now two lists
                   index:   0     1      2       0    1
                      answer =>  ah! our target, '10', is the 0th element in that first list of x, which is in the 0th element of prog!

- now using our code, 'return prog.index(i)', we're saying, "after you find our target, tell me where the target is relative to the index in prog. in this case, 10 is in the 0th element of prog."
- notice how we picked the 0th element for x. This is because we want to ensure that the program can move line to line, and so it's the first number that matters.
'''


from distutils.core import setup

DESCRIPTION = "Javascript Animation exporter for matplotlib and IPython"
LONG_DESCRIPTION = DESCRIPTION
NAME = "JSAnimation"
AUTHOR = "Jake VanderPlas"
AUTHOR_EMAIL = "jakevdp@cs.washington.edu"
MAINTAINER = "Jake VanderPlas"
MAINTAINER_EMAIL = "jakevdp@cs.washington.edu"
DOWNLOAD_URL = 'http://github.com/jakevdp/JSAnimation'
LICENSE = 'BSD'

VERSION = '0.1'

setup(name=NAME,
      version=VERSION,
      description=DESCRIPTION,
      long_description=LONG_DESCRIPTION,
      author=AUTHOR,
      author_email=AUTHOR_EMAIL,
      maintainer=MAINTAINER,
      maintainer_email=MAINTAINER_EMAIL,
      url=DOWNLOAD_URL,
      download_url=DOWNLOAD_URL,
      license=LICENSE,
      packages=['JSAnimation'],
      package_data={'JSAnimation': ['icons/*.png']}
     )


import asyncio
from asyncio.events import AbstractEventLoop
from asyncio.futures import Future
from types import FunctionType
from typing import Union
from wmp.handler import parse

class Transport(asyncio.Protocol):
    def __init__(self, loop: AbstractEventLoop, callback: FunctionType = None):
        self.transport = None
        self.loop = loop
        self.callback: Union[None, FunctionType] = callback
        self.next = None
        self.disconnection_callback = None

    def connection_made(self, transport):
        self.transport = transport

    def connection_lost(self, exception):
        self.transport = None
        if self.disconnection_callback is not None:
            self.disconnection_callback()

    def data_received(self, data):
        lines = data.decode("utf-8").strip().split("\r\n")

        # If the command looks like a configuration change, let the callback handler
        # deal with that, other wise ship it off to the
        results = list(map(lambda line: parse(line), lines))

        if self.next is not None:
            if len(results) == 1:
                self.next.set_result(results[0])
            else:
                self.next.set_result(results)

            self.next = None
        else:
            if self.callback:
                for result in results:
                    self.callback(result)

    def send(self, message: str) -> Future:
        if self.transport is not None:
            self.next = self.loop.create_future()
            self.transport.write(bytes(message + "\r\n", "UTF-8"))
            return self.next
        else:
            future = self.loop.create_future()
            future.set_result(None)
            return future


# Lista Duplamente Encadeada com uso de Sentinelas

class Dnodo:
    def __init__(self, dado = None):
        self.dado = dado
        self.anterior = None
        self.proximo = None

    def __str__(self):
        return str(self.dado)

class LDE:
    def __init__(self):
        self.header = Dnodo()   # <<< Sentinela Cabeça
        self.trailer = Dnodo()  # <<< Sentinela Cauda
        self.tam = 0            # qtde de itens na lista

    def is_empty(self):
        if self.header.proximo is None and self.trailer.anterior is None:
            return True
        return False

    def __inserir(self, novo):
        self.header.proximo = novo    # header -> novo
        novo.anterior = self.header   # header <- novo
        novo.proximo = self.trailer   # novo -> trailer
        self.trailer.anterior = novo  # novo <- trailer

    def inserirInicio(self, novo):
        self.tam += 1
        if self.is_empty():
            self.__inserir(novo)
        else:
           antigo_primeiro = self.header.proximo ## item que deixara de ser o primeiro

           novo.anterior = self.header     ##  HEADER <- NOVO
           novo.proximo = antigo_primeiro  ##  NOVO -> AntigoPrimeiro
           antigo_primeiro.anterior = novo ##  NOVO <- AntigoPrimeiro
           self.header.proximo = novo      ##  HEADER -> NOVO

    def inserirFim(self, novo):
        self.tam += 1
        if self.is_empty():
            self.__inserir(novo)
        else:
            antigo_ultimo = self.trailer.anterior
            novo.proximo = self.trailer
            novo.anterior = antigo_ultimo
            antigo_ultimo.proximo = novo
            self.trailer.anterior = novo

    def removerInicio(self): # esse metodo retorna o nodo para o usuario
        if self.is_empty():
            print('Lista Vazia!')
            return
        else:
            # lista possui + de 1 item
            removido = self.header.proximo
            novo_head = removido.proximo
            self.header.proximo = novo_head
            novo_head.anterior = self.header
            removido.proximo = None
            removido.anterior = None

        self.tam -= 1
        return removido

    def removerFim(self):
        if self.is_empty():
            print('Lista Vazia!')
            return

        else:
            # refazer os apontamentos
            pass

        self.tam -= 1
        return removido

    def imprimir(self, reverso=False):
        if self.is_empty():
            print('Lista Vazia')
            return

        if not reverso:
            item = self.header.proximo # devolve o 1o da lista
            while (item.proximo is not None):
                print(item)
                item = item.proximo
        else:
            item = self.trailer.anterior # devolve o ultimo da lista
            while (item.anterior is not None):
                print(item)
                item = item.anterior

    def buscar(self, alvo): # retorna a 1a ocorrencia
        if alvo.dado == self.header.proximo.dado:
            return self.header.proximo

        if alvo.dado == self.trailer.anterior.dado:
            return self.trailer.anterior

        # laço de busca do 2o elemento até o penultimo
        # faça seu codigo

    def get(self, indice):
        pass

    def removerAntes(self, alvo):
        # header <-> [] <-> [] <-> trailer

        nodo_atual = self.buscar(alvo)
        nodo_anterior = nodo_atual.anterior

        if (nodo_anterior != self.header):
            aux = nodo_anterior.anterior
            aux.proximo = nodo_atual
            nodo_atual.anterior = aux

            nodo_anterior.anterior = None
            nodo_anterior.proximo = None

        self.tam -= 1

    def removerApos(self, alvo):
        pass

    def substituir(self, alvo, valor):
        pass

    def buscarTodos(self, alvo): # retorna uma list com resultados
        lista_todos = []
        ##
        ## sua logica aqui
        ##
        return lista_todos

    def primeiro(self):
        if self.is_empty():
            print('Lista Vazia')
            return
        else:
            return self.header.proximo

    def ultimo(self):
        pass

    def __len__(self):
        return self.tam

## TESTES ##

lista = LDE()
lista.inserirInicio(Dnodo('abc'))
lista.inserirInicio(Dnodo(9999))
lista.inserirInicio(Dnodo('001'))
lista.inserirInicio(Dnodo('xyz'))
lista.inserirFim(Dnodo('www'))
lista.imprimir()
print('Removido o nodo:', lista.removerInicio() )

print()
#lista.removerInicio()
#lista.removerInicio()
#lista.removerInicio()
lista.imprimir()
print(len(lista))


from tkinter import *
import math

# ---------------------------- CONSTANTS ------------------------------- #
PINK = "#e2979c"
RED = "#e7305b"
GREEN = "#9bdeac"
YELLOW = "#f7f5dd"
FONT_NAME = "Courier"
WORK_MIN = 25
SHORT_BREAK_MIN = 5
LONG_BREAK_MIN = 20
reps = 0
timer = None

# ---------------------------- TIMER RESET ------------------------------- #
def reset_timer():
    window.after_cancel(timer)
    canvas.itemconfig(timer_text, text=f"00:00")
    timer_label.config(text="Timer", fg=GREEN)
    check_labels.config(text="")
    global reps
    reps = 0

# ---------------------------- TIMER MECHANISM ------------------------------- #
def start_timer():
    global reps
    reps += 1
    work_sec = WORK_MIN * 60
    short_break_sec = SHORT_BREAK_MIN * 60
    long_break_sec = LONG_BREAK_MIN * 60

    if reps % 8 == 0:
        count_down(long_break_sec)
        timer_label.config(text="Break", fg=RED)
    elif reps % 2 == 0:
        count_down(short_break_sec)
        timer_label.config(text="Break", fg=PINK)
    else:
        count_down(work_sec)
        timer_label.config(text="Work", fg=GREEN)

# ---------------------------- COUNTDOWN MECHANISM ------------------------------- #
def raise_above_all(win):
    win.attributes('-topmost', True)
    win.update()
    win.attributes('-topmost', False)

def count_down(count):
    count_min = math.floor(count / 60)
    count_sec = count % 60
    if count_sec < 10:
        count_sec = f"0{count_sec}"
    canvas.itemconfig(timer_text, text=f"{count_min}:{count_sec}")
    if count > 0:
        global timer
        timer = window.after(1000, count_down, count - 1)
    else:
        start_timer()
        check_labels.config(text="✔"*math.floor(reps/2))
        raise_above_all(window)

# ---------------------------- UI SETUP ------------------------------- #
# Window
window = Tk()
window.title("Pomodoro")
window.config(padx=100, pady=50, bg=YELLOW)

# Labels
timer_label = Label(text="Timer", font=(FONT_NAME, 70, "bold"), bg=YELLOW, fg=GREEN)
timer_label.grid(column=1, row=0)

check_labels = Label(font=(FONT_NAME, 35, "bold"), bg=YELLOW, fg=GREEN)
check_labels.grid(column=1, row=3)

# Canvas
canvas = Canvas(width=200, height=224, bg=YELLOW, highlightthickness=0)  # highlightthickness -> 边框厚度
tomato_image = PhotoImage(file="tomato.png")
canvas.create_image(100, 112, image=tomato_image)
timer_text = canvas.create_text(100, 130, text="00:00", fill="white", font=(FONT_NAME, 35, "bold"))
canvas.grid(column=1, row=1)

# Buttons
start_button = Button(text="Start", height=2, width=2, bd=0, highlightthickness=0, command=start_timer)
start_button.grid(column=0, row=2)

reset_button = Button(text="Reset", height=2, width=2, bd=0, highlightthickness=0, command=reset_timer)
reset_button.grid(column=2, row=2)

window.mainloop()


#!/usr/bin/env python
#-*- coding:utf-8 -*-

from core.output import *
from core.report import Report
import oks

class TransactionsReport(Report):
    OPTIONS = [("status", oks.REPORT_OPTION_BOOL,
                "Incluir transações concluídas", False)]

    def __init__(self, db):
        Report.__init__(self, db)

    def make(self, type_):
        Report.make(self)
        if self.status == True:
            status = (oks.TYPE_STATUS_INCOMPLETE, oks.TYPE_STATUS_COMPLETE)
        else:
            status = oks.TYPE_STATUS_INCOMPLETE

        transactions = self.db.cached_load(oks.TRANSACTION,
                                           IO = type_,
                                           status = status)
        total = 0.0
        operations = {}
        for transaction in transactions:
            pid = transaction.pid
            if pid not in operations:
                operations[pid] = self.db.cached_load(oks.OPERATION,
                                                      row_id = pid)
            operation = operations[transaction.pid]

            self.data.append((transaction.ddate,
                              operation.company,
                              transaction.value,
                              operation.oid,
                              transaction.way))
            total += transaction.value

        self.data.sort(key = lambda row: row[0]) # Sort by date

        if total:
            self.bits.append(CurrencyData("total", "Total", total, unit="R$"))

    def make_output(self):
        Report.make_output(self)
        return TableSection("transactions_report", self.NAME,
                            (DateData("ddate", "Vencimento"),
                             StringData("company", "Empresa"),
                             CurrencyData("value", "Valor", unit="R$"),
                             IntegerData("oid", "ID"),
                             StringData("way", "Forma")),
                            self.data, self.bits)

class PayablesReport(TransactionsReport):
    NAME = "Contas a pagar"
    DESCRIPTION = "Listagem das contas a pagar."
    def __init__(self, db):
        TransactionsReport.__init__(self, db)

    def make(self):
        TransactionsReport.make(self, oks.TYPE_PAYABLE_TRANSACTION)

class ReceivablesReport(TransactionsReport):
    NAME = "Contas a receber"
    DESCRIPTION = "Listagem das contas a receber."
    def __init__(self, db):
        TransactionsReport.__init__(self, db)

    def make(self):
        TransactionsReport.make(self, oks.TYPE_RECEIVABLE_TRANSACTION)


import tensorflow as tf
from Parameters_rnn import parameters as pm
from data_processing_rnn import batch_iter, sequence

class RnnModel(object):

    def __init__(self):
        self.input_x = tf.placeholder(tf.int32, shape=[None, pm.seq_length], name='input_x')
        self.input_y = tf.placeholder(tf.float32, shape=[None, pm.num_classes], name='input_y')
        self.seq_length = tf.placeholder(tf.int32, shape=[None], name='sequen_length')
        self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
        self.global_step = tf.Variable(0, trainable=False, name='global_step')
        self.rnn()

    def rnn(self):

        with tf.device('/cpu:0'), tf.name_scope('embedding'):
            embedding = tf.get_variable('embedding', shape=[pm.vocab_size, pm.embedding_dim],
                                        initializer=tf.constant_initializer(pm.pre_trianing))
            self.embedding_input = tf.nn.embedding_lookup(embedding, self.input_x)

        with tf.name_scope('cell'):
            cell = tf.nn.rnn_cell.LSTMCell(pm.hidden_dim)
            cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=self.keep_prob)

            cells = [cell for _ in range(pm.num_layers)]
            Cell = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)

        with tf.name_scope('rnn'):
            #hidden一层 输入是[batch_size, seq_length, hidden_dim]
            #hidden二层 输入是[batch_size, seq_length, 2*hidden_dim]
            #2*hidden_dim = embendding_dim + hidden_dim
            output, _ = tf.nn.dynamic_rnn(cell=Cell, inputs=self.embedding_input, sequence_length=self.seq_length, dtype=tf.float32)
            output = tf.reduce_sum(output, axis=1)
            #output:[batch_size, seq_length, hidden_dim]

        with tf.name_scope('dropout'):
            self.out_drop = tf.nn.dropout(output, keep_prob=self.keep_prob)

        with tf.name_scope('output'):
            w = tf.Variable(tf.truncated_normal([pm.hidden_dim, pm.num_classes], stddev=0.1), name='w')
            b = tf.Variable(tf.constant(0.1, shape=[pm.num_classes]), name='b')
            self.logits = tf.matmul(self.out_drop, w) + b
            self.predict = tf.argmax(tf.nn.softmax(self.logits), 1, name='predict')

        with tf.name_scope('loss'):
            losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y)
            self.loss = tf.reduce_mean(losses)

        with tf.name_scope('optimizer'):
            optimizer = tf.train.AdamOptimizer(pm.learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.loss))#计算变量梯度，得到梯度值,变量
            gradients, _ = tf.clip_by_global_norm(gradients, pm.clip)
            #对g进行l2正则化计算，比较其与clip的值，如果l2后的值更大，让梯度*(clip/l2_g),得到新梯度
            self.optimizer = optimizer.apply_gradients(zip(gradients, variables), global_step=self.global_step)
            #global_step 自动+1

        with tf.name_scope('accuracy'):
            correct_prediction = tf.equal(self.predict, tf.argmax(self.input_y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')

    def feed_data(self, x_batch, y_batch, seq_len, keep_prob):

        feed_dict = {self.input_x: x_batch,
                     self.input_y: y_batch,
                     self.seq_length: seq_len,
                     self.keep_prob: keep_prob}

        return feed_dict

    def evaluate(self, sess, x, y):

        batch_test = batch_iter(x, y, pm.batch_size)
        for x_batch, y_batch in batch_test:
            seq_len = sequence(x_batch)
            feet_dict = self.feed_data(x_batch, y_batch, seq_len, 1.0)
            loss, accuracy = sess.run([self.loss, self.accuracy], feed_dict=feet_dict)

        return loss, accuracy


if __name__ == "__main__":

    liczby = [line.strip() for line in open('liczby.txt', 'r')]

    takie_same = [n for n in liczby if n[0] == n[-1]]

    # same = [x for x in liczby if x[0] == x[-1]]
    # print(same)
    # juz lambdy jest łatwiej pojąć

    print('Zad 4.1', len(takie_same), takie_same[0])

    czynniki = {liczba: [] for liczba in liczby}

    for liczba in czynniki.keys():
        n = int(liczba)
        i = 2
        while n > 1 and i != int(liczba):

            if n % i == 0:
                n //= i
                czynniki[liczba].append(i)
                i = 2
                continue
            i += 1

    most_factors_quantity = 0
    most_factors_number = 0

    for rozklad in czynniki.keys():
        if len(czynniki[rozklad]) > most_factors_quantity:
            most_factors_quantity = len(czynniki[rozklad])
            most_factors_number = rozklad

    diff_factors_quantity = 0
    diff_factors_number = 0

    for rozklad in czynniki.keys():
        uniques = set(czynniki[rozklad])

        if len(uniques) > diff_factors_quantity:
            diff_factors_quantity = len(uniques)
            diff_factors_number = rozklad

    print(
        'Zad 4.2',
        most_factors_number,
        most_factors_quantity,
        diff_factors_number,
        diff_factors_quantity
    )

    liczby = [int(liczba) for liczba in liczby]

    maksimum = max(liczby)
    minimum = min(liczby)

    trojki = set()
    piatki = set()

    for liczba in liczby:

        krotnosci = [n for n in liczby if n % liczba == 0]

        if len(krotnosci) < 2:
            continue

        krotnosci.sort()

        for i1 in krotnosci:
            for i2 in krotnosci:
                for i3 in krotnosci:

                    if i3 % i2 == 0 and i2 % i1 == 0 and i1 != i2 != i3 :
                        trojki.add(tuple(map( lambda x: str(x), (i1, i2, i3))))

                    for i4 in krotnosci:
                        for i5 in krotnosci:

                            if i5 % i4 == 0 and i4 % i3 == 0 and i3 % i2 == 0 and i2 % i1 == 0 and i1 != i2 != i3 != i4 != i5 :
                                piatki.add(tuple(map( lambda x: str(x), (i1, i2, i3, i4, i5))))

    print('Trojki')
    for t in trojki:
        print(" ".join(list(t)))

    print(len(trojki))

    print('Piatki')
    for p in piatki:
        print(" ".join(list(p)))

    print(len(piatki))

        #     for pivot in range(1, len(krotnosci) - step*2 ):

        #         if (krotnosci[pivot+2*step]*liczba % krotnosci[pivot+step]*liczba == 0
        #         and krotnosci[pivot+step]*liczba % krotnosci[pivot]*liczba == 0):

        #             trojki.add((
        #                 krotnosci[pivot]*liczba,
        #                 krotnosci[pivot+step]*liczba,
        #                 krotnosci[pivot+2*step]*liczba,
        #             ))

    # print(trojki)


from pprint import pprint

LED_MAPPINGS = [
    "abcefg", "cf", "acdeg", "acdfg", "bcdf", "abdfg", "abdefg", "acf",
    "abcdefg", "abcdfg"
]

UNIQUE_LED_SEGMENTS_LENGTH = [
    len(LED_MAPPINGS[1]),
    len(LED_MAPPINGS[4]),
    len(LED_MAPPINGS[7]),
    len(LED_MAPPINGS[8]),
]

if __name__ == "__main__":
    with open("input.txt", "r") as f:
        inputs = f.read().split("\n")

    unique_segments = list(map(lambda a: a.split("|")[1].strip(), inputs))
    unique_segments = [ u.split() for u in unique_segments]

    total_unique = 0
    for u in unique_segments:
        for segment in u:
            if len(segment) in UNIQUE_LED_SEGMENTS_LENGTH:
                total_unique += 1

    print(f"Result for Part 1: {total_unique}")


# -*- coding: utf-8 -*-
"""
Created on Tue Jun  6 13:59:07 2023

@author: Jose Fernando Galindo Suarez
@email: jgalindos@sena.edu.co
"""

import pandas as pd

ruta="c:/Borrar/"
data=pd.read_csv(ruta+'COVID19-JULIO2020.csv',low_memory=False)
print(data.shape)

data['Estado']=data['Estado'].str.replace('leve','Leve',regex=False)
DM_ESTADO=pd.DataFrame(data['Estado'])
DM_ESTADO.drop_duplicates(inplace=True)
DM_ESTADO['IDESTADO']=range(1, len(DM_ESTADO) + 1)
DM_ESTADO.set_index('IDESTADO', inplace=True)
DM_ESTADO=DM_ESTADO.fillna('NA')
DM_ESTADO.to_csv(ruta+'DM_ESTADO.csv')

data['Tipo']=data['Tipo'].str.replace('relacionado','Relacionado',regex=False)
data['Tipo']=data['Tipo'].str.replace('RELACIONADO','Relacionado',regex=False)
data['Tipo']=data['Tipo'].str.replace('En Estudio','En estudio',regex=False)
DM_TIPO=pd.DataFrame(data['Tipo'])
DM_TIPO.drop_duplicates(inplace=True)
DM_TIPO['IDTIPO']= range(1, len(DM_TIPO) + 1)
DM_TIPO.set_index('IDTIPO', inplace=True)
DM_TIPO.to_csv(ruta+'DM_TIPO.csv')

DM_FECHA=pd.DataFrame(data['Fecha'])
DM_FECHA.drop_duplicates(inplace=True)
DM_FECHA['IDFECHA']=range(1,len(DM_FECHA)+1)
DM_FECHA.set_index('IDFECHA',inplace=True)
DM_FECHA.to_csv(ruta+'DM_FECHA.csv')

DM_DEPARTAMENTO=pd.DataFrame(data['Departamento o Distrito '])
DM_DEPARTAMENTO.drop_duplicates(inplace=True)

DM_DEPARTAMENTO.columns=DM_DEPARTAMENTO.columns.str.replace('Departamento o Distrito ','NOMBRE')
DM_DEPARTAMENTO['IDDPTO']=data['Código DIVIPOLA']//1000
DM_DEPARTAMENTO.loc[DM_DEPARTAMENTO['NOMBRE']=='Barranquilla D.E.','IDDPTO']=8001
DM_DEPARTAMENTO.loc[DM_DEPARTAMENTO['NOMBRE']=='Cartagena D.T. y C.','IDDPTO']=13001
DM_DEPARTAMENTO.loc[DM_DEPARTAMENTO['NOMBRE']=='Santa Marta D.T. y C.','IDDPTO']=47001
DM_DEPARTAMENTO.loc[DM_DEPARTAMENTO['NOMBRE']=='Buenaventura D.E.','IDDPTO']=76109
DM_DEPARTAMENTO.set_index('IDDPTO',inplace=True)
DM_DEPARTAMENTO.to_csv(ruta+'DM_DEPARTAMENTO.csv')

DM_CIUDAD=pd.DataFrame(data['Ciudad de ubicación'])
DM_CIUDAD.drop_duplicates(inplace=True)
DM_CIUDAD['IDCIUDAD']=data['Código DIVIPOLA']
DM_CIUDAD.set_index('IDCIUDAD',inplace=True)
DM_CIUDAD.to_csv(ruta+'DM_CIUDAD.csv')

DM_ATENCION=pd.DataFrame(data['atención'])
DM_ATENCION=DM_ATENCION.fillna('NA')
DM_ATENCION.drop_duplicates(inplace=True)
DM_ATENCION['IDATENCION']=range(1,len(DM_ATENCION)+1)
DM_ATENCION.set_index('IDATENCION',inplace=True)
DM_ATENCION.to_csv(ruta+'DM_ATENCION.csv')

DM_PAIS=pd.DataFrame(data['País de procedencia'])
DM_PAIS=DM_PAIS.fillna('COLOMBIA')
data['País de procedencia'].fillna('COLOMBIA',inplace=True)
DM_PAIS.columns=DM_PAIS.columns.str.replace('País de procedencia','NOMBRE')
DM_PAIS['NOMBRE']=DM_PAIS.NOMBRE.str.replace('ESTADOS UNIDOS DE AMÉRICA','ESTADOS UNIDOS',regex=False)
DM_PAIS['NOMBRE']=DM_PAIS.NOMBRE.str.replace('ESTADOS UNIDOS DE AMERICA','ESTADOS UNIDOS',regex=False)
DM_PAIS['NOMBRE']=DM_PAIS.NOMBRE.str.replace('ARABIA SAUDÍ','ARABIA SAUDITA',regex=False)
DM_PAIS['NOMBRE']=DM_PAIS.NOMBRE.str.replace('MÉXICO','MEXICO',regex=False)
DM_PAIS['NOMBRE']=DM_PAIS.NOMBRE.str.replace('PANAMÁ','PANAMA',regex=False)
DM_PAIS['NOMBRE']=DM_PAIS.NOMBRE.str.replace('PERÚ','PERU',regex=False)
DM_PAIS.drop_duplicates(inplace=True)
DM_PAIS['IDPAIS']=range(1,len(DM_PAIS)+1)
DM_PAIS.set_index('IDPAIS',inplace=True)
DM_PAIS=DM_PAIS.sort_values('NOMBRE')
DM_PAIS.to_csv(ruta+'DM_PAIS.csv')

data['Sexo']=data.Sexo.str.replace('m','M',regex=False)
data['Sexo']=data.Sexo.str.replace('f','F',regex=False)
DM_SEXO=pd.DataFrame(data['Sexo'])
DM_SEXO.columns=DM_SEXO.columns.str.replace('SEXO','NOMBRE')
DM_SEXO.drop_duplicates(inplace=True)
DM_SEXO['IDSEXO']=range(1,len(DM_SEXO)+1)
DM_SEXO.set_index('IDSEXO',inplace=True)
DM_SEXO.to_csv(ruta+'DM_SEXO.csv')

data.columns=data.columns.str.replace('Tipo','IDTIPO')
data.columns=data.columns.str.replace('Estado','IDESTADO')
data.columns=data.columns.str.replace('Código DIVIPOLA','IDCIUDAD')
data.columns=data.columns.str.replace('atención','IDATENCION')
data.columns=data.columns.str.replace('País de procedencia','IDPAIS')
data.columns=data.columns.str.replace('Sexo','IDSEXO')
data.columns=data.columns.str.replace('Departamento o Distrito ','IDDPTO')

data['IDDPTO']=data['IDCIUDAD']//1000

data.drop('Ciudad de ubicación',axis=1,inplace=True)

data['IDTIPO']=data.IDTIPO.str.replace('Importado','1',regex=False)
data['IDTIPO']=data.IDTIPO.str.replace('Relacionado','2',regex=False)
data['IDESTADO']=data.IDESTADO.str.replace('Leve','1',regex=False)
data['IDESTADO']=data.IDESTADO.str.replace('Asintomático','2',regex=False)
data['IDATENCION']=data.IDATENCION.str.replace('Recuperado','1',regex=False)
data['IDATENCION']=data.IDATENCION.str.replace('Fallecido','2',regex=False)
data['IDATENCION']=data.IDATENCION.str.replace('NA','3',regex=False)
data['IDATENCION']=data.IDATENCION.str.replace('Casa','4',regex=False)
data['IDATENCION']=data.IDATENCION.str.replace('UCI','5',regex=False)
data['IDATENCION']=data.IDATENCION.str.replace('Hospital','6',regex=False)
data['IDESTADO']=data.IDESTADO.str.replace('Fallecido','3',regex=False)
data['IDTIPO']=data.IDTIPO.str.replace('En estudio','3',regex=False)

pais=pd.read_csv(ruta+'DM_PAIS.csv')

z1=pais['IDPAIS']
z2=pais['NOMBRE']

for idpais,nombre in zip(z1,z2):
    data['IDPAIS']=data.IDPAIS.str.replace(nombre,str(idpais),regex=False)

mfecha=pd.read_csv(ruta+'DM_FECHA.csv')
z1=mfecha['IDFECHA']
z2=mfecha['Fecha']

for idfecha,fecha in zip(z1,z2):
    data['Fecha']=data.Fecha.str.replace(fecha,str(idfecha),regex=False)

data.columns=data.columns.str.replace('Fecha','IDFECHA')
#data.drop('Fecha',axis=1,inplace=True)
rangos=[0,5,10,18,50,100,110]
nombrer=['A','B','C','D','E','F']
data['GEDAD']=pd.cut(data['Edad'],rangos,labels=nombrer)

data['ID']=range(1,len(data)+1)
data.set_index("ID",inplace=True)
data.to_csv(ruta+'TH_COVID19.csv',index=False)


import boto3
import os

# enviroment
region = os.environ["AWS_DEFAULT_REGION"]
# s3 = boto3.resource('s3')
s3 = boto3.client('s3')

result_bucket = os.environ["s3Result"]
html_bucket = os.environ["s3Html"]
def get_all_s3_objects(s3, **base_kwargs):
    continuation_token = None
    while True:
        list_kwargs = dict(MaxKeys=1000, **base_kwargs)
        if continuation_token:
            list_kwargs['ContinuationToken'] = continuation_token
        response = s3.list_objects_v2(**list_kwargs)
        yield from response.get('Contents', [])
        if not response.get('IsTruncated'):  # At the end of the list?
            break
        continuation_token = response.get('NextContinuationToken')

def handle_remove(record):
    print('Handling REMOVE event')
    #1. parse oldimage and price
    old_image = record['dynamodb']['OldImage']
    submitid = old_image['PK']['S']
    #2. remove output s3 data
    #2.1 remove output s3 data
    for object_file in get_all_s3_objects(s3,Bucket=result_bucket, Prefix=os.path.join("output",submitid+'/'),Delimiter='/'):
        print('Deleting', object_file['Key'])
        s3.delete_object(Bucket=result_bucket, Key=object_file['Key'])

def lambda_handler(event,context):
    """

    """
    print('------------1-------------')
    print(event)
    try:
        #1. Iterate over each record
        for record in event["Records"]:
            if record['eventName']  == "REMOVE":
                handle_remove(record)

        print('------------2-------------')
    except Exception as e:
        print(e)
        print('------------3-------------')


import cx_Freeze

executables = [cx_Freeze.Executable("SnakeGame.py")]

cx_Freeze.setup(
    name="Snake game",
    options={"build_exe": {"packages":["pygame"],
                           "include_files":["gamelogo.jpg","gamebg.jpg","gameover.png","back.mp3","gameover.mp3"]}},
    executables = executables

    )


import requests
import re
from bs4 import BeautifulSoup
from imdb_db import update_global_table
import json
import ast
import logging

def check_item(movie_or_tv, year, genre, movie_or_tv_filter, genre_filter, year_beg_filter, year_end_filter):
    """Returns True if the movie passes the filter and False otherwise"""
    if (not year) or (not movie_or_tv) or (not genre):
        return False
    if movie_or_tv != movie_or_tv_filter:
        return False
    if genre_filter != 'All' and genre_filter not in genre:
        return False
    if year < year_beg_filter or year > year_end_filter:
        return False
    return True

def get_cast(soup):
    """This function prints the casting of a movie/tv series including the actors and the directors, writers and
    creators. """
    production_team = []
    actors = []
    for child in soup.find('ul',
                           class_="ipc-metadata-list ipc-metadata-list--dividers-all title-pc-list "
                                  "ipc-metadata-list--baseAlt",
                           role="presentation").children:
        cat = child.contents[0].string
        if cat == 'Stars':
            content = child.div.ul.li
            while content:
                actor_id = int(''.join(c for c in content.a['href'] if c.isdigit()))
                actor = (content.a.string, actor_id)
                actors.append(actor)
                content = content.next_sibling
        else:
            content = child.div.ul.li
            while content:
                prod_id = int(''.join(c for c in content.a['href'] if c.isdigit()))
                prod = (content.a.string, prod_id, cat)
                production_team.append(prod)
                content = content.next_sibling

    return list(set(production_team)), list(set(actors))

def get_movie_data(movie, args, headers, api_key):
    """This function allows to retrieve the data from the web"""
    title = movie.find(class_="lister-item-header").a.string
    movie_url = movie.find(class_="lister-item-header").a['href']
    omdb_id = movie_url.split('/')[2]
    movie_id = int(''.join(c for c in movie_url if c.isdigit()))
    genre = movie.find(class_="genre").string.strip().split(',')
    release_year = None
    if movie.find(class_='text-muted'):
        if movie.find(class_='text-muted').string :
            year = movie.find(class_='text-muted').string[1:-1]
            tmp = re.findall('[0-9]{4}', year)
            if tmp:
                release_year = int(tmp[0])
    if movie.strong:
        rating = float(movie.strong.string)
    else:
        rating = None
    resume = movie.find_all('p')[1].string
    if resume:
        resume = resume.strip()

    movie_page = requests.get('https://www.imdb.com' + movie_url, headers=headers)

    if movie_page.status_code != 200:
        logging.error("URL doesn't exist ! Failed")

    movie_soup = BeautifulSoup(movie_page.content, "html.parser")

    if movie_soup.find(role="presentation", class_="ipc-inline-list__item", string="TV Series"):
        movie_or_tv = "TV"
    elif movie_soup.find(role="presentation", class_="ipc-inline-list__item", string="Video Game"):
        movie_or_tv = "Video Game"
    elif movie_soup.find(role="presentation", class_="ipc-inline-list__item", string="Video"):
        movie_or_tv = "Video"
    elif movie_soup.find(role="presentation", class_="ipc-inline-list__item", string="Podcast Series"):
        movie_or_tv = "Podcast Series"
    elif movie_soup.find(role="presentation", class_="ipc-inline-list__item", string="Music Video"):
        movie_or_tv = "Music Video"
    else:
        movie_or_tv = "Movie"

    if check_item(movie_or_tv, release_year, genre, args.movie_or_tv, args.genre, args.year_beg, args.year_end):
        prod, actors = get_cast(movie_soup)
        rotten_rating, meta_rating = get_ratings(omdb_id, api_key)

        return movie_id, title, movie_or_tv, release_year, genre, rating, resume, prod, actors, rotten_rating, meta_rating
    else:
        return None

def get_rotten_rating(ratings):
    """This function allows to retrieve the rating of Rotten Tomatoes from API"""
    for element in ratings:
        if element['Source'] == "Rotten Tomatoes":
            return element['Value']
    return None

def get_meta_rating(ratings):
    """This function allows to retrieve the rating of Metacritic from API"""
    for element in ratings:
        if element['Source'] == "Metacritic":
            return element['Value']
    return None

def get_ratings(movie_id, api_key):
    """This function allows to retrieve the rating from the API"""
    omdb_url = 'http://www.omdbapi.com/?apikey=' + api_key + '&i=' + movie_id
    page = requests.get(omdb_url)
    if page.status_code != 200:
        logging.error("URL doesn't exist ! Failed")
    movie_json = page.json()

    rotten_rating = None
    meta_rating = None

    if 'Ratings' in movie_json:
        if movie_json['Ratings'] != []:
            rotten_rating = get_rotten_rating(movie_json['Ratings'])
            if rotten_rating:
                rotten_rating = float(rotten_rating.split('%')[0])/10
            meta_rating = get_meta_rating(movie_json['Ratings'])
            if meta_rating:
                meta_rating = float(meta_rating.split('/')[0])/10

    return rotten_rating, meta_rating

def data_retrieving(args, my_url):
    """This function gets an URL from one of the categories of IMDB's website and prints each movie/tv show with
    relevant datas. """
    with open('conf_IMDB.json') as json_config:
        config = json.load(json_config)
        headers = ast.literal_eval(config['HEADERS'])
        api_key = config['API_KEY']

    page = requests.get(my_url, headers=headers)

    if page.status_code != 200:
        logging.error("URL doesn't exist ! Failed")

    while page:
        soup = BeautifulSoup(page.content, "html.parser")
        next_url = 'https://www.imdb.com' + soup.find(class_="desc").find_all('a')[-1]['href']
        movies_list = soup.find(class_="lister-list")

        movie = movies_list.find(class_="lister-item")
        while movie:
            try:
                data = get_movie_data(movie, args, headers, api_key)
                if data:
                    update_global_table(*data)
                    logging.info(f'Retrieving movie :{data[1]}')
                movie = movie.next_sibling.next_sibling
            except AttributeError:
                movie = movie.next_sibling.next_sibling

        page = requests.get(next_url, headers=headers)


import pathlib
from setuptools import setup

HERE = pathlib.Path(__file__).parent

README = (HERE / 'README.md').read_text()

setup(
    name='suite8080',
    version='0.5.0',
    description='Suite of Intel 8080 Assembly tools',
    long_description=README,
    long_description_content_type='text/markdown',
    url='https://github.com/pamoroso/suite8080',
    author='Paolo Amoroso',
    author_email='info@paoloamoroso.com',
    license='MIT',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.6',
        'Operating System :: OS Independent',
        'Environment :: Console',
        'Topic :: Software Development',
        'Topic :: Software Development :: Assemblers',
        'Topic :: Software Development :: Disassemblers',
        'Intended Audience :: Developers'
    ],
    python_requires='>=3.6',
    packages=['suite8080'],
    entry_points={
        'console_scripts': [
            'asm80=suite8080.asm80:main',
            'dis80=suite8080.dis80:main'
        ]
    }
)


#!/usr/bin/env python3

import json
import time

from time import mktime

f=open("../../data/metaculus.json")
jsondata=json.load(f)

for page in jsondata:
	for question in page["results"]:
		if question["possibilities"]["type"]=="binary" and (question["resolution"]==1 or question["resolution"]==0):
			try:
				restime=time.strptime(question["resolve_time"],"%Y-%m-%dT%H:%M:%S.%fZ")
			except:
				restime=time.strptime(question["resolve_time"],"%Y-%m-%dT%H:%M:%SZ")
			try:
				createtime=time.strptime(question["created_time"],"%Y-%m-%dT%H:%M:%S.%fZ")
			except:
				createtime=time.strptime(question["created_time"],"%Y-%m-%dT%H:%M:%SZ")
			for pred in question["prediction_timeseries"]:
				timediff=mktime(restime)-pred["t"]
				qtimediff=mktime(restime)-mktime(createtime)
				print("{},{},{},{},{}".format(question["id"], qtimediff, question["resolution"], pred["community_prediction"], timediff))


""" Initial schema

Revision ID: 629958e9b907
Revises:
Create Date: 2017-01-01 17:20:08.514311

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = '629958e9b907'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('iiif_image',
    sa.Column('surrogate_id', sa.Integer(), nullable=False),
    sa.Column('id', sa.String(), nullable=True),
    sa.Column('info', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
    sa.PrimaryKeyConstraint('surrogate_id'),
    sa.UniqueConstraint('id')
    )
    op.create_table('manifest',
    sa.Column('surrogate_id', sa.Integer(), nullable=False),
    sa.Column('id', sa.String(), nullable=True),
    sa.Column('origin', sa.String(), nullable=True),
    sa.Column('manifest', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
    sa.Column('label', sa.String(), nullable=True),
    sa.PrimaryKeyConstraint('surrogate_id'),
    sa.UniqueConstraint('id'),
    sa.UniqueConstraint('origin')
    )
    op.create_table('identifier',
    sa.Column('surrogate_id', sa.Integer(), nullable=False),
    sa.Column('id', sa.String(), nullable=True),
    sa.Column('type', sa.String(), nullable=True),
    sa.Column('manifest_id', sa.String(), nullable=False),
    sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], ),
    sa.PrimaryKeyConstraint('surrogate_id'),
    sa.UniqueConstraint('id')
    )
    op.create_table('image',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('url', sa.String(), nullable=True),
    sa.Column('width', sa.Integer(), nullable=True),
    sa.Column('height', sa.Integer(), nullable=True),
    sa.Column('format', sa.Text(), nullable=True),
    sa.Column('iiif_id', sa.String(length=22), nullable=True),
    sa.ForeignKeyConstraint(['iiif_id'], ['iiif_image.id'], ),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('url')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('image')
    op.drop_table('identifier')
    op.drop_table('manifest')
    op.drop_table('iiif_image')
    # ### end Alembic commands ###