import numpy as np
import pandas as pd
from scipy.sparse import lil_matrix, vstack
import copy
import sys
import networkx as nx

# Class Graph
# Create a graph element, which is the base element for the usage of this package
# Graphs can be: undirect, direct, with scalar or vector euclidean attribute, with nodes or edges attributes of same or different dimension
# A Graph object is made of:
# - Adj: dictionary showing the structure (every node is a key, the values are the adjency nodes to the key)
# - x: dictionary showing the attributes
#   x[i,i]: nodes attributes
#   x[i,j]: edge attributes
# - y: scalar or category attribute of the Graph (seful for classification or regression)

class Graph:

    # Initializing the graph
    # Input:
    # x: a dictionary with nodes attributes in [i,i] key and edge attributes in [j,i] position
    # Facultative Input:
    # adj: adjency list with [i] node as [k,l,m] the nodes is linked to as values (if None it is created in definethegraph())
    # y: numeric or categorical variable associated to the
    def __init__(self, x, adj, s):
        if (s is None):
            self.s = None
        else:
            self.s = s  # Regressor or class of the network
        # Check on the x: the null nodes should not be in the middle.
        # e.g. [0,1,2] can not have (0,0) (2,2) different from zero and (1,1) zero.
        self.x = copy.deepcopy(x)  # nodes and edges attributes
        self.attr = None  # matrix of feature vectors
        if (adj is None):
            self.adj = None
        else:
            self.adj = adj;  # adjacency list
        self.n_nodes = 0;  # number of nodes
        self.n_edges = 0;  # number of edges
        self.definethegraph()

    # Building the adj from x
    def definethegraph(self):
        # Function to define the adjency List

        # if it is empty, there is nothing to fill
        if (self.x == None):
            self.adj = None
            self.n_edges = 0
            self.n_nodes = 0
            self.node_attr = 0
            self.edge_attr = 0
        else:
            # number of nodes, edges and dimension of the structure
            # number of edges: (initialized and updated after)
            self.n_edges = 0
            # number of nodes:
            self.n_nodes = max([int(s[0]) if (s[0] >= s[1]) else s[1] for s in self.x.keys()]) + 1
            # length of node attributes
            _na = [len(v) for k, v in self.x.items() if type(v) is not int and k[0] == k[1]]
            if (not _na):
                self.node_attr = 1
            else:

                self.node_attr = max(_na)
                if (self.node_attr == 0):
                    self.node_attr = 1

            # length of edge attributes
            _ea = [len(v) for k, v in self.x.items() if type(v) is not int and k[0] != k[1]]
            if (not _ea):
                self.edge_attr = 0
            else:
                self.edge_attr = max(_ea)
                if (self.edge_attr == 0):
                    self.edge_attr = 1
            # building the adjency list:
            if (self.adj == None):
                self.adj = {}
                for i in range(self.n_nodes):
                    # All the nodes have an attribute by default
                    if (not (i, i) in self.x):
                        self.x.update({(i, i): [0] * self.node_attr})
                    # list of adjency nodes to node i
                    s = []
                    # self.x is a dictionary structure
                    for k, v in self.x.items():
                        # adjency nodes:
                        if (k[1] != k[0] and k[0] == i):
                            s = s + [k[1]]
                            # if(k[1] == i): s=s+[k[0]]
                    s = list(set(s))
                    if (len(s) > 0):
                        self.adj[i] = s
                        self.n_edges = self.n_edges + len(self.adj[i])
                    # else:
                    #    self.adj[i]=[]
                    # creating the self.x matrix (i.e. the attribute dictionary)
                    for j in range(self.n_nodes):
                        if ((i, j) in self.x):
                            if (type(self.x[i, j]) is not int):
                                _l = len(self.x[i, j])
                            else:
                                _l = 1
                            if (i == j and _l < self.node_attr and _l > 1): self.x[i, j] = self.x[i, j] + [0] * (
                                        self.node_attr - _l)
                            if (i == j and _l < self.node_attr and _l == 1 and isinstance(self.x[i, j], list)):
                                self.x[i, j] = self.x[i, j] + [0] * (self.node_attr - _l)
                            if (i == j and _l < self.node_attr and _l == 1 and not isinstance(self.x[i, j], list)):
                                self.x[i, j] = [self.x[i, j]] + [0] * (self.node_attr - _l)
                            if (i != j and _l < self.edge_attr and _l > 1): self.x[i, j] = self.x[i, j] + [0] * (
                                        self.edge_attr - _l)
                            if (i != j and _l < self.edge_attr and _l == 1 and isinstance(self.x[i, j], list)): self.x[
                                i, j] = self.x[i, j] + [0] * (self.edge_attr - _l)
                            if (i != j and _l < self.edge_attr and _l == 1 and not isinstance(self.x[i, j], list)):
                                self.x[i, j] = [self.x[i, j]] + [0] * (self.edge_attr - _l)
                        # else:
                        #    if(i==j): self.x[i,j]=[0]*self.node_attr
                        #    else: self.x[i,j]=[0]*self.edge_attr

    # If we are intetrested in the attributes, uncomment what is after the 'and'
    def __eq__(self, other):
        # """Override the default Equals behavior"""
        return self.x == other.x  # and self.attr == other.attr

    def __ne__(self, other):
        # """Override the default Unequal behavior"""
        return self.x != other.x  # or self.attr != other.attr

    # Create a deep copy of the object
    def cp(self):
        d = copy.deepcopy(self)
        return d

    # Number of nodes
    def nodes(self):
        return self.n_nodes

    # Number of edges
    def edges(self):
        return self.n_edges

    # Degree of node (unweighted)
    def degree(self, i):
        if (i < 0 or self.n_nodes <= i or not i in self.adj):
            return 0;
        # print i
        else:
            return len(self.adj[i])

    # Degree of node (weighted)
    def weighted_degree(self, i):
        if (i < 0 or self.n_nodes <= i or not i in self.adj):
            return 0;
        return sum(self.adj[i])

    # Adjency dictionary with attributes
    def matrix(self):
        return self.x

    # Adjency List
    def adjList(self):
        return self.adj

    # The following three functions deal with the nodel class label if there is one:
    def Features(self):
        if (self.s is None):
            print("Missing attribute s.")
        else:
            return s

    # Class Label assignment
    def setFeatures(self, feature):
        self.s = feature

    def OutputFeature(self):
        if (self.s is None):
            print("Missing features.")
        else:
            return self.s

    # Check if it has a label
    def HasFeatures(self):
        if (self.s is None):
            return False
        else:
            return True

    # Attributes node dimension
    def dimNodes(self):
        for i in range(self.n_nodes):
            if ((i, i) in self.x and self.x[i, i] != 0):
                return len(self.x[i, i])
        return 0

    # Attributes edge dimension
    def dimEdges(self):
        for i in range(self.n_nodes):
            for j in range(self.n_nodes):
                if ((i, j) in self.x and self.x[i, j] != 0):
                    return len(self.x[i, j])
        return 0

    # Return keys of nodes
    def nodes_list(self):
        n_list = []
        for k, v in self.x.items():
            if (k[0] == k[1]):
                n_list.append(k)
        return n_list

    # Return keys of edges
    def edges_list(self):
        e_list = []
        for k, v in self.x.items():
            if (k[0] != k[1]):
                e_list.append(k)
        return e_list

    # Get the adjency sparse matrix of 0 and 1
    def get_pure_matrix(self):
        _x = lil_matrix((self.n_nodes, self.n_nodes))
        for i in self.x.keys():
            _x[i] = 1
        return _x

    # Check if the graph is empty:
    def isZero(self):
        for i in range(self.n_nodes):
            if (self.x[i, i] != 0):
                return False;
            degree = self.degree(i)
            for j in range(degree):
                j0 = self.adj[i][j]
                if (self.x[i][j0] != 0):
                    return False
                return True

    # Scale: multuply all the weights (nodes and edges) with 'a'
    def scale(self, a):
        x_new = {}
        for i in range(self.n_nodes):
            x_new[i, i] = list(np.multiply(a, self.x[i, i]))
            degree = self.degree(i)
            for j in range(degree):
                j0 = self.adj[i][j]
                x_new[i, j0] = list(np.multiply(a, self.x[i, j0]))
        return Graph(x=x_new, adj=self.adj, s=self.s)

    # Permuting indexes of the nodes
    def permutelist(self):
        f = np.random.permutation(self.nodes())
        return f

    # Permute function
    # This function is a key function when dealing with more than one network.
    # it is permuting the node of the network and it is called in all the alignment process
    # Input:
    # - f: list of index of nodes (e.g. of a permutation of 3 nodes network: f=[1,0,2])
    def permute(self, f):
        _x = {}
        _adj = {}
        for i in range(self.n_nodes):
            fi = f[i]
            _x[fi, fi] = self.x[i, i]
            _adj[fi] = []
            for j in range(self.degree(i)):
                j0 = self.adj[i][j]
                fj = f[j0]
                _x[f[i], f[j0]] = self.x[i, j0]
                _adj[fi].append(fj)
        del (self.x, self.adj)
        self.x = copy.deepcopy(_x)
        self.adj = copy.deepcopy(_adj)

    # Scale the attributes of node and edges within a network
    def feature_scale(self):
        # initialize both minimum and maximum for attributes
        x_M_nodes = [None] * self.node_attr
        x_m_nodes = [sys.maxint] * self.node_attr
        x_M_edges = [None] * self.edge_attr
        x_m_edges = [sys.maxint] * self.edge_attr
        _x = self.x
        for i in range(self.n_nodes):

            x_M_nodes = np.maximum(x_M_nodes, self.x[i, i])
            x_m_nodes = np.minimum(x_m_nodes, self.x[i, i])

            for j in range(self.degree(i)):
                j0 = self.adj[i][j]
                x_M_edges = np.maximum(x_M_edges, self.x[i, j0])
                x_m_edges = np.minimum(x_m_edges, self.x[i, j0])
        range_nodes = np.array(np.subtract(x_M_nodes, x_m_nodes), dtype=float)
        range_edges = np.array(np.subtract(x_M_edges, x_m_edges), dtype=float)
        for i in range(self.n_nodes):
            a = np.array(np.subtract(_x[i, i], x_m_nodes), dtype=float)
            _x[i, i] = np.divide(a, range_nodes, out=np.zeros_like(a), where=range_nodes != 0).tolist()
            for j in range(self.degree(i)):
                j0 = self.adj[i][j]
                if (all(range_edges == 0)):
                    _x[i, j0] = np.divide(_x[i, j0], _x[i, j0]).tolist()
                else:
                    a = np.array(np.subtract(_x[i, j0], x_m_edges), dtype=float)
                    _x[i, j0] = np.divide(a, range_edges, out=np.zeros_like(a), where=range_edges != 0).tolist()
        self.x = _x

    # Increasing graph size adding new empty nodes to the network
    # input:
    # - size: scalar value representing the new desired size
    def grow(self, size):
        if (size <= self.n_nodes):
            return self
        else:
            if (self.attr is None):
                _x = {}
                _adj = {}
                for i in range(size):
                    for j in range(size):

                        if (i <= (self.n_nodes - 1) and j <= (self.n_nodes - 1)):
                            if ((i, j) in self.x):
                                # _adj[i]=self.adj[i]
                                _x[i, j] = self.x[i, j]

                        else:
                            if (i == j):
                                _x[i, j] = [0] * self.node_attr
                                # _adj[i]=[]
                            # else:
                            #    _x[i,j]=[0]*self.edge_attr

                self.x = _x
                # self.adj=_adj
                self.n_nodes = size
                # self.definethegraph()
            else:
                print("Hi Darling, attributes needed: use function grow_with_attributes instead!")

    # Increasing graph size adding new nodes to the network with the given attributes
    # input:
    # - size: scalar value representing the new desired size
    # - new_attr: attribute to be added
    def grow_with_attributes(self, size, new_attr):
        if (size <= self.n_nodes):
            return self
        else:
            _x = {}
            _adj = {}
            for i in range(size):
                if (i <= (self.n_nodes - 1)):
                    _adj[i] = self.adj[i]
                    for j in range(self.n_nodes):
                        _x[i, j] = self.x[i, j]
                else:
                    _adj[i] = []
            self.x = _x
            self.adj = _adj
            self.n_nodes = size
            self.attr = self.attr.append(new_attr)

    # Decrising the graph size
    # input:
    # - size: scalar value representing the new desired size
    def shrink(self, size):
        if (size < 1 or self.n_nodes <= size):
            print("Invalid size")
        else:
            if (size < self.n_nodes):
                _x = {}
                for i in range(size):
                    for j in range(size):
                        _x[i, j] = self.x[i, j]
                self.x = _x
                self.adj = Graph(self.x, y=None, adj=None)
                if (self.attr is not None):
                    self.attr = self.attr.loc[range(size), :]

    # Extract the network corresponding to the j-th edge layer
    # input:
    # - j: the desired layer
    # - node_too: if True extract also the layer from the node
    def extract_layer(self, j, node_too):
        # j is the j-th edges' attribute to built the network on
        _x = copy.deepcopy(self.x)
        for k, v in _x.items():
            if (node_too == True):
                _x[k] = [v[j]]
            else:
                if (k[0] != k[1]):
                    _x[k] = [v[j]]
        return _x

    # Delete the attribute selected:
    # input:
    # - j: the attribute to delete
    # - attr_type: either node or edge according to which
    def del_attribute(self, j, attr_type):
        if (attr_type == 'node'):
            if (self.dimNodes() < j):
                return "Error: the attribute index is too high"
            else:
                for k, v in self.x.items():
                    if (k[0] == k[1]):
                        del v[j]

    # From Graph to vector structure (1 if there is a link, 0 otherwise)
    # The vector is building unrolling the adj matrix by row
    # The dimension of the vector is #edges*n_attr_edges+#nodes*n_attr_nodes= N*(N-1)*#e_attr+N*n_attr
    def to_vector_with_attributes(self):
        n_a = self.node_attr
        e_a = self.edge_attr
        col_i = [str(item) for sublist in [[k] * n_a if k[0] == k[1] else [k] * e_a for k in self.x.keys()] for item in
                 sublist]
        col_i2 = list(map(lambda x: x[1] + str(col_i[:x[0]].count(x[1]) + 1) if col_i.count(x[1]) > 1 else x[1],
                          enumerate(col_i)))
        df_0 = pd.DataFrame([np.array([item for sublist in [v for v in self.x.values()] for item in sublist])],
                            columns=col_i2)
        return df_0

    # INTERMEDIATE STEP: # ATTRIBUTES == 1
    # From Graph to vector structure, for all the elements in the set iset of indices (i,j) given
    # If the graph has no (i,j) element, its value is set to 0
    # The vector is building unrolling the adj matrix by row
    # The dimension of the vector is len(iset)*n_attr_edges or len(iset)*n_attr_nodes
    def to_vector_with_select_attributes(self, iset):
        n_a = self.node_attr
        e_a = self.edge_attr
        assert n_a * e_a == 1, "This method assumes that the node or edge attribute is a scalar"
        # working on the column names:
        col_i = [str(item) for item in iset]
        # and then creating the dataframe
        df_0 = pd.DataFrame([np.array([self.x[item][0] if item in self.x.keys() else 0.0 for item in iset])],
                            columns=col_i)
        return df_0

    # GENERALIZATION to #attrib > 1 !!!
    # From Graph to vector structure, for all the elements in the set iset of indices (i,j) given
    # If the graph has no (i,j) element, its value is set to 0
    # The vector is building unrolling the adj matrix by row
    # The dimension of the vector is respectively #edges*n_attr_edges and #nodes*n_attr_nodes
    def to_vector_with_select_nodes(self, iset):
        n_a = self.node_attr
        # working on the column names:
        col_i = [str(item) for item in iset]
        # and then creating the dataFrame (actually no need to create an intermediate np.array)
        df_0 = pd.DataFrame([self.x[item] if item in self.x.keys() else [0.0] * n_a for item in iset], index=col_i)
        return df_0

    def to_vector_with_select_edges(self, iset):
        e_a = self.edge_attr
        # working on the column names:
        col_i = [str(item) for item in iset]
        # and then creating the dataFrame (actually no need to create an intermediate np.array)
        df_0 = pd.DataFrame([self.x[item] if item in self.x.keys() else [0.0] * e_a for item in iset], index=col_i)
        return df_0

    # To Networkx Object: this function convert the graph to a networkx object
    # input:
    # - layer: the layer to extract
    # - node_too: True if the layer should be extracted from the nodes, False otherwise
    def to_networkX(self, layer, node_too, directed):
        if (directed == True):
            G = nx.DiGraph()
        else:
            G = nx.Graph()
        if (layer != None):
            _x = self.extract_layer(layer, node_too)
            G.add_nodes_from(range(self.n_nodes))
            G.add_edges_from([e for e in list(_x.keys()) if e[0] != e[1]])
            if (node_too == True):
                nx.set_node_attributes(G, dict((k[0], {'weight': float(v[0])}) for k, v in _x.items() if k[0] == k[1]))
            nx.set_edge_attributes(G, dict((k, float(v[0])) for k, v in _x.items()), 'weight')
        else:
            print("Caution: No Layer Specified")
            G.add_nodes_from(range(self.n_nodes))
            G.add_edges_from([e for e in list(_x.keys()) if e[0] != e[1]])
            if (node_too == True):
                nx.set_node_attributes(G, dict((k[0], {'weight': float(v[0])}) for k, v in _x.items() if k[0] == k[1]))
            nx.set_edge_attributes(G, self.x, 'weight')
        return G

    # Drop a node or a set of nodes
    # input:
    # - id: the index of the node
    def drop_nodes(self, id):
        adj_new = copy.deepcopy(self.adj)
        x_new = copy.deepcopy(self.x)
        for i in range(self.n_nodes):
            if (i in id):
                if ((i, i) in x_new):
                    del x_new[i, i]
                # all the edges exiting the node i
                if (i in adj_new):

                    current_id = [(i, j) for j in adj_new[i]]
                    print('Delete i:')
                    print(current_id)
                    for key in current_id: del x_new[key]
                    del adj_new[i], current_id
            else:
                # all the edges entering in node i
                current_id = [(i, j) for j in adj_new[i] if j in id]
                adj_new[i] = [n for n in adj_new[i] if n not in id]
                for key in current_id: del x_new[key]
        return Graph(x=x_new, adj=adj_new, s=self.s)


# -*- coding: utf-8 -*-
"""
최종 프로젝트
1. 코스피주식 업종에 따라 군집분석 및 업종 선택
2. 업종 주식가격 데이터 크롤링
3. RNN
4. 시각화
5. 업종별 주식 선택
6. 포트폴리오 최적화

"""
import pandas as pd
import urllib.request as req # 원격 서버 url 자료 요청
import numpy as np

# 1. 코스피주식 업종에 따라 군집분석 및 업종 선택

path='C:\\ITWILL\\sihun'
Dataset= pd.read_excel(path+'\\Sectors.xlsx')
Dataset = Dataset.iloc[:,0:4]
Dataset2 = Dataset.loc[:,['종목코드','업종명']]

# sector 이름 출력
Sector_name =Dataset.iloc[:,-1]
Sector=Sector_name.drop_duplicates()

'''
0             서비스업
1             기타금융
3              유통업
5             섬유의복
10           운수창고업
11            음식료품
18              증권
19              보험
20            전기전자
25             건설업
29              화학
30            철강금속
47              기계
50             의약품
57           비금속광물
69            운수장비
71             통신업
72           기타제조업
115          전기가스업
152           종이목재
201             은행
216             광업
284           의료정밀
309    농업, 임업 및 어업
Name: 업종명, dtype: object
'''
Secotor=list(Sector)

data=[]
for a in Sector:
    a=Dataset2[Dataset.iloc[:,-1] == a]
    data.append(a)

#DataFrame 생성 <- 첨삭 요구
service_industry=data[0]
Other_finance=data[1]
Distribution=data[2]
Textile_garments=data[3]
Transportation_warehousing_business=data[4]
F_B=data[5]
securities=data[6]
insurance=data[7]
electrical =data[8]
construction=data[9]
chemistry=data[10]
steel_metal=data[11]
machine=data[12]
medicines=data[13]
Non_metallic_minerals=data[14]
transport_equipment=data[15]
telecommunications=data[16]
Other_manufacturing=data[17]
Electricity_Gas=data[18]
paper_wood=data[19]
bank=data[20]
Mining=data[21]
Medical_Precision=data[22]
Agriculture_Forestry_Fisheries=data[23]

# 저평가된 주식을 찾는다는 가정하에 주식의 P/E와  ROE를 기준으로 찾는다.
# P/E는 12~15 이하인 주식 .
# ROE가 20% 이상인 주식 ...
#주당순이익이 꾸준히 증가 ...
#http://data.krx.co.kr/contents/MDC/MDI/mdiLoader/index.cmd?menuId=MDC020

df=pd.read_excel('C:\\ITWILL\\sihun\\df_P_E.xlsx')
df=df.iloc[:,0:7]
type(df)

#종목코드기준으로 P/E, 종가추출
service=service_industry.iloc[:,] # 서비스 업 주식 코드
type(service)

company=[]
for a in list(df.code):
    company.append(a)
len(company) #874

P_E=[]
cnt=0
cnt2=-1
for company1 in company:
    cnt2=+1
    try:
        if service[cnt] == company1:
            P_E.extend(df.iloc[cnt2])
            cnt=+1
        else:
            print('error')
            print(cnt)
    except Exception as e:
        print('error')

P_E=pd.DataFrame(P_E)
P_E.to_excel(excel_writer='C:\\ITWILL\\sihun\\service')

# Sector 별로 저평가된 주식을 찾고 주식을 선택한다.

#####################################################################
## 선택한 주식의 가격정보를 가지고 온다.
##################################################################
# 20페이지 일별 주식 가격 가지고 오기'
import pandas as pd
import urllib.request as req
import requests
#1. 한국 거래소 종목 가져오기
stock_code=pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download',header=0)[0]

#한글 컬럼명을 영어로 변경
stock_code = stock_code.rename(columns={'회사명': 'company', '종목코드': 'code'})

#종목코드가 6자리이기 때문에 6자리르 맞춰주기 위해 설절해줌
stock_code.code=stock_code.code.map('{:06d}'.format)

#크롤링
company='LG화학' # [수정 필요] list형으로 변경해서 모든 선택된 주식의 정보를 가지고 온다.
code=stock_code[stock_code.company==company].code.values[0] # 한국 증권 거래소에서 가지고온 데이터

Date=[]
price=[]
for page in range(1,21):
    url=f'https://finance.naver.com/item/sise_day.nhn?code={code}&page={page}'
    table= pd.read_html(requests.get(url, headers={'User-agent': 'Mozilla/5.0'}).text)
    temp=table[0]
    temp=temp.dropna() # temp 형식 DataFrame
    date=temp.iloc[:,0]
    price_d=temp.iloc[:,1]
    price.extend(price_d) # append or extend
    Date.extend(date)

##################################################################################################
### RNN 으로 각주식의 수익 예측 후 수익성 좋은 주식을 로만 다시 주식 선택 <- tensorflow 사용
##################################################################################################

# data 가지고 오기
data = price
len(data) # 100
print(data)

# 2. RNN 구제에 맞게 reshape
#x_data
x = np.array([data[i+j] for i in range(len(data)-10) for j in range(10)])
# 90 * 10 =900
'''
0~9
1~10
2~11
3~12
  :
89~98
'''
len(x) #900

# train/val split for x
x_train = x[:700].reshape(70,10,1)
x_test = x[700:].reshape(-1,10,1)

x_train.shape #(70,10,1) - (batch_size, time steps, features)
x_test.shape # (20, 10, 1) - (batch_size, time steps, features)

#y_data
y = np.array([data[i+10] for i in range(len(data)-10)]) # 0 ~ 89
'''
10
11
 :
99
'''
len(y) # 90

# train/val split for y (70 vs 20)
y_train = y[:70].reshape(70,1)
y_test = y[70:].reshape(20,1)
y_train.shape # (70, 1)
y_test.shape #  (20, 1)

# 3. model 생성
input_shape = (10, 1)

model = Sequential()

#RNN layer 추가
model.add(SimpleRNN(units=8, input_shape=input_shape, activation='tanh'))

#hidden layer 추가가능 (복잡한 계산의 경우)

# DNN layer 추가
model.add(Dense(units=1)) # 출력 : 회귀모델 - MSE, MAE
#최종 출력은 예측된 숫자나 class 분류 결과이기 때문에 DNN을 이용합니다.

# model 학습환경
model.compile(optimizer='sgd', loss='mse', metrics=['mae']) # sgd, adam 비교해서 선택

# model training
model.fit(x_train,y_train, epochs=400, verbose=1)

# model prediction
y_pred = model.predict(x_test)
print(y_pred)

# real value vs predict value
plt.plot(y_test, 'y--', label='real value')
plt.plot(y_pred, 'r--',label='predict value')
plt.legend(loc='best')
plt.show()

#########################################################################################
### 포트폴리오 최적화(최소한의 리스크로 최대의 수익을 창출할수있는 포트폴리오 생성) <- use R
#########################################################################################


import sys
from GUI.main import GUI
from database.database_manager import DatabaseManager
from face_detection import FaceDetector
from face_recognition import FaceRecognition
from voice_authentication import VoiceAuthentication

def create_system_admin(database, _email):
    if not database.user_exists_by_email(_email):
        print("User not found, please create a user before executing this action")
        return

    if database.give_admin_privileges_by_email(_email):
        print("Admin privileges were given to", _email)
        return

    print("Something went wrong, action aborted.")

def remove_system_admin(database, _email):
    if not database.user_exists_by_email(_email):
        print("User not found, please create a user before executing this action")
        return

    if database.remove_admin_privileges_by_email(_email):
        print("Admin privileges were removed from", _email)
        return

    print("Something went wrong, action aborted.")

def main():
    database = DatabaseManager()
    if len(sys.argv) == 3:
        _email = sys.argv[2]
        if sys.argv[1] == "createadmin":
            create_system_admin(database, _email)
        elif sys.argv[1] == "removeadmin":
            remove_system_admin(database, _email)
        else:
            print("Invalid command")

    else:
        face_detector = FaceDetector(0.3, 2)
        face_recognition = FaceRecognition(database, confidence=65.0)
        voice_auth = VoiceAuthentication(database, confidence=90)
        GUI(face_detector, face_recognition, voice_auth)
    database.close()

if __name__ == "__main__":
    main()


from uiucapi.query import *
from requests_xml import XMLSession
from URLBuilder import *

session = XMLSession()

class Request(object):

    def __init__(self, in_year, in_semester):
        self.subjects = list()
        self.course_list = list()
        self.year = in_year
        self.semester = in_semester
        self.subject_code = ""
        self.course_num = ""
        self.u = URLBuilder(self.year, self.semester)

    def set_subject_code(self, in_subject):
        """
        method to set subject code
        :param in_subject: input subject
        """
        self.subject_code = in_subject
        self.u.set_subject_code(self.subject_code)

    def set_course_num(self, in_course_num):
        """
        method to set particular course
        :param in_course_num: input course number
        """
        self.course_num = in_course_num
        self.u.set_course_num(self.course_num)

    def build_course_str(self):
        course_str = self.subject_code + ' ' + self.course_num + ' '
        if self.semester == 'fall':
            course_str += 'FA'
        elif self.semester == 'spring':
            course_str += 'SP'
        course_str += self.year[2:]
        return course_str

    def get_subjects(self):
        """
        method to build a list of all subjects
        """
        for i in range(1, 200):
            s = '//subject[%s]/@id' % (str(i))
            r = session.get(self.u.get_url())
            item = r.xml.xpath(s, first=True)
            if item is None:
                break
            self.subjects.append(item)

    def get_courses(self):
        """
        method to set all courses
        """
        r = session.get(self.u.get_url())
        for i in range(1, 200):
            xp = '//course[%s]/@id' % (str(i))
            l = r.xml.xpath(xp, first=True)
            if l is None:
                break
            self.course_list.append(l)

    def course_description(self):
        """
        method to get course description
        :return: description of the course
        """
        r = session.get(self.u.get_url())
        desc = r.xml.xpath("//description/text()", first=True)
        return desc

    def get_prereq(self):
        """
        get all prerequisites for a given course
        :return: prerequisites of the given course
        """
        s = self.course_description()
        temp = "Prerequisite:"
        index = s.find(temp)
        s = s[index: len(s)]
        s = s[len(temp):]
        parts = s.split(";")
        parts = [part.strip() for part in parts]
        pre_reqs = [list() for _ in parts]
        for j in range(len(parts)):
            s = parts[j]
            i = len(s) - 1
            while i >= 0:
                if s[i].isdigit() or s[i] == " " or s[i].isupper():
                    end = i + 1
                    while i >= 0 and (s[i].isdigit() or s[i] == " " or s[i].isupper()):
                        i -= 1
                    i += 1
                    pre_reqs[j].append(s[i:end])
                i -= 1
        for i in range(len(pre_reqs)):
            temp = []
            for option in pre_reqs[i]:
                option = option.strip()
                if len(option) > 1:
                    temp.append(option)
            pre_reqs[i] = temp
        return pre_reqs

    def get_open_sections(self):
        """
        get a list of all open sections for the course
        :return: a list of all open sections for the course
        """
        course_str = self.build_course_str()
        course = get_course(course_str)
        open_sections = []
        lec_r = False
        labs_r = False
        disc_r = False

        if ('Lecture' or 'Lecture-Discussion') in self.get_types():
            lec_r = True
        if ('Discussion' or 'Lecture-Discussion' or 'Discussion/Recitation') in self.get_types():
            disc_r = True
        if ('Laboratory' or 'Laboratory-Discussion') in self.get_types():
            labs_r = True

        for section in course.sections:

            if "open" in section.registration_status.lower():
                open_sections.append(tuple((section.section_number, section.crn)))
        lec_open = False
        disc_open = False
        labs_open = False
        for sec in open_sections:
            x, y = sec
            if 'lecture' in self.get_section_type(y).lower() or not lec_r:
                lec_open = True
            if 'discussion' in self.get_section_type(y).lower() or not disc_r:
                disc_open = True
            if 'lab' in self.get_section_type(y).lower() or not labs_r:
                labs_open = True
        if lec_open and disc_open and labs_open:
            return open_sections
        else:
            raise Exception("Course not available")

    def get_credit_hours(self):
        """
        method to get number of credit hours the course is
        :return: the number of credit hours
        """
        r = session.get(self.u.get_url())
        credit_hours = r.xml.xpath("//creditHours/text()", first=True)
        return int(credit_hours)

    def get_no_lectures(self):
        """
        method to get number of lecture sections for the course
        :return: the number of open lecture sections
        """
        sections = self.get_open_sections()
        count_lec = 0
        for name, crn in sections:
            if 'lecture' in self.req.get_section_type(crn).lower():
                count_lec += 1
        return count_lec

    def get_no_disc(self):
        """
        method to get number of discussion sections for the course
        :return: the number of open discussion sections
        """
        sections = self.get_open_sections()
        count_disc = 0
        for name, crn in sections:
            if 'discussion' in self.req.get_section_type(crn).lower():
                count_disc += 1
        return count_disc

    def get_no_lab(self):
        """
        method to get number of lab sections for the course
        :return: the number of open lab sections
        """
        sections = self.get_open_sections()
        count_disc = 0
        for name, crn in sections:
            if 'lab' in self.req.get_section_type(crn).lower():
                count_disc += 1
        return count_disc

    def get_time(self, section_str):
        """
        method to get the starting and ending times
        :param section_str: the CRN of the section
        :return: the starting and ending times of the section
        """
        r = session.get(self.u.url + str(section_str) + ".xml")
        start_time = r.xml.xpath("//meetings/meeting/start/text()", first=True)
        end_time = r.xml.xpath("//meetings/meeting/end/text()", first=True)
        s = ""
        e = ""
        if start_time[6] == 'P':
            s += str(12 + int(start_time[1]))
            e += str(12 + int(end_time[1]))
        else:
            s += start_time[0]
            e += end_time[0]
            s += start_time[1]
            e += end_time[1]
        s += start_time[2:5]
        e += end_time[2:5]
        classes_on_day = self.get_days(section_str).strip()
        return tuple((s.strip(), e.strip(), classes_on_day))

    def get_location(self, section_str):
        """
        method to get the location
        :param section_str: the CRN of the section
        :return: the room number and building of the section
        """
        r = session.get(self.u.url + str(section_str) + ".xml")
        building = r.xml.xpath("//meetings/meeting/buildingName/text()", first=True)
        room = r.xml.xpath("//meetings/meeting/roomNumber/text()", first=True)
        return room + ' ' + building

    def get_instructor(self, section_str):
        """
        method to get the name of the instructor
        :param section_str: the CRN of the section
        :return: the name of the instructor
        """
        r = session.get(self.u.url + str(section_str) + ".xml")
        instructor = r.xml.xpath("//meetings/meeting/instructor/text()", first=True)
        return instructor

    def get_days(self, section_str):
        """
        method to get days of the week
        :param section_str: the CRN of the section
        :return: the days of the week on which there is class
        """
        r = session.get(self.u.url + str(section_str) + ".xml")
        days = r.xml.xpath("//meetings/meeting/daysOfTheWeek/text()", first=True)
        return days

    def get_section_type(self, section_str):
        """
        method to get type of the section
        :param section_str: the CRN of the section
        :return: the type of the section
        """
        r = session.get(self.u.url + str(section_str) + ".xml")
        type = r.xml.xpath("//meetings/meeting/type/text()", first=True)
        return type

    def get_types(self):
        """
        method to return all required classes
        :return:
        """
        types = []
        r = session.get(self.u.get_url())
        for i in range(1, 200):
            crn = '//section[%s]/@id' % (str(i))
            item = r.xml.xpath(crn, first=True)
            if item is None:
                break
            types.append(self.get_section_type(item))
        return types


import collections
from openpype_modules.ftrack.lib import BaseEvent

class TaskToVersionStatus(BaseEvent):
    """Changes status of task's latest AssetVersions on its status change."""

    settings_key = "status_task_to_version"

    # Attribute for caching session user id
    _cached_user_id = None

    def is_event_invalid(self, session, event):
        """Skip task status changes for session user changes.

        It is expected that there may be another event handler that set
        version status to task in that case skip all events caused by same
        user as session has to avoid infinite loop of status changes.
        """
        # Cache user id of currently running session
        if self._cached_user_id is None:
            session_user_entity = session.query(
                "User where username is \"{}\"".format(session.api_user)
            ).first()
            if not session_user_entity:
                self.log.warning(
                    "Couldn't query Ftrack user with username \"{}\"".format(
                        session.api_user
                    )
                )
                return False
            self._cached_user_id = session_user_entity["id"]

        # Skip processing if current session user was the user who created
        # the event
        user_info = event["source"].get("user") or {}
        user_id = user_info.get("id")

        # Mark as invalid if user is unknown
        if user_id is None:
            return True
        return user_id == self._cached_user_id

    def filter_event_entities(self, event):
        """Filter if event contain relevant data.

        Event cares only about changes of `statusid` on `entity_type` "Task".
        """

        entities_info = event["data"].get("entities")
        if not entities_info:
            return

        filtered_entity_info = collections.defaultdict(list)
        for entity_info in entities_info:
            # Care only about tasks
            if entity_info.get("entity_type") != "Task":
                continue

            # Care only about changes of status
            changes = entity_info.get("changes") or {}
            statusid_changes = changes.get("statusid") or {}
            if (
                statusid_changes.get("new") is None
                or statusid_changes.get("old") is None
            ):
                continue

            # Get project id from entity info
            project_id = None
            for parent_item in reversed(entity_info["parents"]):
                if parent_item["entityType"] == "show":
                    project_id = parent_item["entityId"]
                    break

            if project_id:
                filtered_entity_info[project_id].append(entity_info)

        return filtered_entity_info

    def _get_ent_path(self, entity):
        return "/".join(
            [ent["name"] for ent in entity["link"]]
        )

    def launch(self, session, event):
        '''Propagates status from version to task when changed'''
        if self.is_event_invalid(session, event):
            return

        filtered_entity_infos = self.filter_event_entities(event)
        if not filtered_entity_infos:
            return

        for project_id, entities_info in filtered_entity_infos.items():
            self.process_by_project(session, event, project_id, entities_info)

    def process_by_project(self, session, event, project_id, entities_info):
        if not entities_info:
            return

        project_name = self.get_project_name_from_event(
            session, event, project_id
        )
        # Load settings
        project_settings = self.get_project_settings_from_event(
            event, project_name
        )

        event_settings = (
            project_settings["ftrack"]["events"][self.settings_key]
        )
        _status_mapping = event_settings["mapping"]
        if not event_settings["enabled"]:
            self.log.debug("Project \"{}\" has disabled {}.".format(
                project_name, self.__class__.__name__
            ))
            return

        if not _status_mapping:
            self.log.debug((
                "Project \"{}\" does not have set status mapping for {}."
            ).format(project_name, self.__class__.__name__))
            return

        status_mapping = {
            key.lower(): value
            for key, value in _status_mapping.items()
        }

        asset_types_filter = event_settings["asset_types_filter"]

        task_ids = [
            entity_info["entityId"]
            for entity_info in entities_info
        ]

        last_asset_versions_by_task_id = (
            self.find_last_asset_versions_for_task_ids(
                session, task_ids, asset_types_filter
            )
        )

        # Query Task entities for last asset versions
        joined_filtered_ids = self.join_query_keys(
            last_asset_versions_by_task_id.keys()
        )
        if not joined_filtered_ids:
            return

        task_entities = session.query(
            "select status_id, link from Task where id in ({})".format(
                joined_filtered_ids
            )
        ).all()
        if not task_entities:
            return

        status_ids = set()
        for task_entity in task_entities:
            status_ids.add(task_entity["status_id"])

        task_status_entities = session.query(
            "select id, name from Status where id in ({})".format(
                self.join_query_keys(status_ids)
            )
        ).all()
        task_status_name_by_id = {
            status_entity["id"]: status_entity["name"]
            for status_entity in task_status_entities
        }

        # Final process of changing statuses
        project_entity = session.get("Project", project_id)
        av_statuses_by_low_name, av_statuses_by_id = (
            self.get_asset_version_statuses(project_entity)
        )

        asset_ids = set()
        for asset_versions in last_asset_versions_by_task_id.values():
            for asset_version in asset_versions:
                asset_ids.add(asset_version["asset_id"])

        asset_entities = session.query(
            "select name from Asset where id in ({})".format(
                self.join_query_keys(asset_ids)
            )
        ).all()
        asset_names_by_id = {
            asset_entity["id"]: asset_entity["name"]
            for asset_entity in asset_entities
        }
        for task_entity in task_entities:
            task_id = task_entity["id"]
            status_id = task_entity["status_id"]
            task_path = self._get_ent_path(task_entity)

            task_status_name = task_status_name_by_id[status_id]
            task_status_name_low = task_status_name.lower()

            new_asset_version_status = None
            mapped_status_names = status_mapping.get(task_status_name_low)
            if mapped_status_names:
                for status_name in mapped_status_names:
                    _status = av_statuses_by_low_name.get(status_name.lower())
                    if _status:
                        new_asset_version_status = _status
                        break

            if not new_asset_version_status:
                new_asset_version_status = av_statuses_by_low_name.get(
                    task_status_name_low
                )
            # Skip if tasks status is not available to AssetVersion
            if not new_asset_version_status:
                self.log.debug((
                    "AssetVersion does not have matching status to \"{}\""
                ).format(task_status_name))
                continue

            last_asset_versions = last_asset_versions_by_task_id[task_id]
            for asset_version in last_asset_versions:
                version = asset_version["version"]
                self.log.debug((
                    "Trying to change status of last AssetVersion {}"
                    " for task \"{}\""
                ).format(version, task_path))

                asset_id = asset_version["asset_id"]
                asset_type_name = asset_names_by_id[asset_id]
                av_ent_path = task_path + " Asset {} AssetVersion {}".format(
                    asset_type_name,
                    version
                )

                # Skip if current AssetVersion's status is same
                status_id = asset_version["status_id"]
                current_status_name = av_statuses_by_id[status_id]["name"]
                if current_status_name.lower() == task_status_name_low:
                    self.log.debug((
                        "AssetVersion already has set status \"{}\". \"{}\""
                    ).format(current_status_name, av_ent_path))
                    continue

                new_status_id = new_asset_version_status["id"]
                new_status_name = new_asset_version_status["name"]
                # Skip if status is already same
                if asset_version["status_id"] == new_status_id:
                    continue

                # Change the status
                try:
                    asset_version["status_id"] = new_status_id
                    session.commit()
                    self.log.info("[ {} ] Status updated to [ {} ]".format(
                        av_ent_path, new_status_name
                    ))
                except Exception:
                    session.rollback()
                    self.log.warning(
                        "[ {} ]Status couldn't be set to \"{}\"".format(
                            av_ent_path, new_status_name
                        ),
                        exc_info=True
                    )

    def get_asset_version_statuses(self, project_entity):
        """Status entities for AssetVersion from project's schema.

        Load statuses from project's schema and store them by id and name.

        Args:
            project_entity (ftrack_api.Entity): Entity of ftrack's project.

        Returns:
            tuple: 2 items are returned first are statuses by name
                second are statuses by id.
        """
        project_schema = project_entity["project_schema"]
        # Get all available statuses for Task
        statuses = project_schema.get_statuses("AssetVersion")
        # map lowered status name with it's object
        av_statuses_by_low_name = {}
        av_statuses_by_id = {}
        for status in statuses:
            av_statuses_by_low_name[status["name"].lower()] = status
            av_statuses_by_id[status["id"]] = status

        return av_statuses_by_low_name, av_statuses_by_id

    def find_last_asset_versions_for_task_ids(
        self, session, task_ids, asset_types_filter
    ):
        """Find latest AssetVersion entities for task.

        Find first latest AssetVersion for task and all AssetVersions with
        same version for the task.

        Args:
            asset_versions (list): AssetVersion entities sorted by "version".
            task_ids (list): Task ids.
            asset_types_filter (list): Asset types short names that will be
                used to filter AssetVersions. Filtering is skipped if entered
                value is empty list.
        """

        # Allow event only on specific asset type names
        asset_query_part = ""
        if asset_types_filter:
            # Query all AssetTypes
            asset_types = session.query(
                "select id, short from AssetType"
            ).all()
            # Store AssetTypes by id
            asset_type_short_by_id = {
                asset_type["id"]: asset_type["short"]
                for asset_type in asset_types
            }

            # Lower asset types from settings
            # WARNING: not sure if is good idea to lower names as Ftrack may
            #   contain asset type with name "Scene" and "scene"!
            asset_types_filter_low = set(
                asset_types_name.lower()
                for asset_types_name in asset_types_filter
            )
            asset_type_ids = []
            for type_id, short in asset_type_short_by_id.items():
                # TODO log if asset type name is not found
                if short.lower() in asset_types_filter_low:
                    asset_type_ids.append(type_id)

            # TODO log that none of asset type names were found in ftrack
            if asset_type_ids:
                asset_query_part = " and asset.type_id in ({})".format(
                    self.join_query_keys(asset_type_ids)
                )

        # Query tasks' AssetVersions
        asset_versions = session.query((
            "select status_id, version, task_id, asset_id"
            " from AssetVersion where task_id in ({}){}"
            " order by version descending"
        ).format(self.join_query_keys(task_ids), asset_query_part)).all()

        last_asset_versions_by_task_id = collections.defaultdict(list)
        last_version_by_task_id = {}
        not_finished_task_ids = set(task_ids)
        for asset_version in asset_versions:
            task_id = asset_version["task_id"]
            # Check if task id is still in `not_finished_task_ids`
            if task_id not in not_finished_task_ids:
                continue

            version = asset_version["version"]

            # Find last version in `last_version_by_task_id`
            last_version = last_version_by_task_id.get(task_id)
            if last_version is None:
                # If task id does not have version set yet then it's first
                # AssetVersion for this task
                last_version_by_task_id[task_id] = version

            elif last_version > version:
                # Skip processing if version is lower than last version
                # and pop task id from `not_finished_task_ids`
                not_finished_task_ids.remove(task_id)
                continue

            # Add AssetVersion entity to output dictionary
            last_asset_versions_by_task_id[task_id].append(asset_version)

        return last_asset_versions_by_task_id

def register(session):
    TaskToVersionStatus(session).register()


import requests
import sys

from bs4 import BeautifulSoup
import pandas as pd

import config

def get_lyrics(song_title, artist_name):
    print(song_title)
    song_title = song_title.replace('?', '')
    artist_name = artist_name.replace('?', '')
    # https://github.com/willamesoares/lyrics-crawler

    base_url = 'https://api.genius.com'
    headers = {'Authorization': 'Bearer ' + config.genius_api_token}
    search_url = base_url + '/search'
    data = {'q': song_title + ' ' + artist_name}
    response = requests.get(search_url, data=data, headers=headers)
    json = response.json()
    remote_song_info = None

    for hit in json['response']['hits']:
        if artist_name.lower() in hit['result']['primary_artist']['name'].lower():
            remote_song_info = hit
            break
    if remote_song_info is None:
        # retry with only first word
        if len(song_title.split()) > 1:
            return get_lyrics(song_title.split()[0], artist_name)
        else:
            return None
    else:
        song_url = remote_song_info['result']['url']
        return scrap_song_url(song_url)

def scrap_song_url(url):
    # https://github.com/willamesoares/lyrics-crawler
    page = requests.get(url)
    html = BeautifulSoup(page.text, 'html.parser')
    [h.extract() for h in html('script')]
    lyrics = html.find('div', class_='lyrics').get_text()

    return lyrics

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("No filename provided")
        exit()
    fname = sys.argv[1].rsplit(".", 1)[0]
    songs = pd.read_csv(fname + ".csv")
    songs["lyrics"] = songs.apply(lambda row: get_lyrics(row["name"], row["artists"]), axis=1)
    songs.to_csv(fname + "_lyrics.csv")


# ! /usr/bin/env python
# - * - coding:utf-8 - * -
# __author__ : KingWolf
# createtime : 2019/1/19 4:06

from selenium import webdriver
import time
import os

"""

    处理prompt对话框

"""
#案例1

options = webdriver.ChromeOptions()
options.add_argument('user-data-dir=F:\profile')
chromedriver_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),'chromedriver.exe')
driver = webdriver.Chrome(chrome_options=options,executable_path=chromedriver_path)

driver.get(r'F:\GitExtensions_python\project_spider\exercise_learn\selenium\selenium_study_aotumation\alert_confirm_prompt\alert_confirm_prompt_test1.html')
driver.implicitly_wait(10)
# 获取prompt对话框的按钮，点击按钮，弹出prompt对话框
driver.find_element_by_id('prompt').click()
time.sleep(1)

# 获取prompt对话框
prompt = driver.switch_to.alert

# 获取对话框内容
text = prompt.text
print(text)

# 在弹框内输入信息
prompt.send_keys('狼胸最牛逼！')
# 点击“确认”按钮，提交输入的内容
prompt.accept()

time.sleep(2)
driver.quit()


import json
import os
import re

import requests
from lib.cuckoo.common.abstracts import CUCKOO_ROOT
from lib.cuckoo.core.plugins import import_package, list_plugins

import modules.signatures

def mapMitre(oldID):
    try:
        response = requests.get(f"https://attack.mitre.org/techniques/{oldID}")
    except Exception as e:
        print(e)
        return oldID

    if response.status_code == 200:
        match1 = re.search(rb'url=([^"]+)"', response.content)
        if match1:
            url = match1.group(1).decode("utf-8")
            pattern = r"\/techniques\/(\w+)(?:\/(\d+))?"
            matches = re.search(pattern, url)
            if matches:
                ttp_id = matches.group(1)
                sub_id = matches.group(2) if matches.group(2) else ""
                # Create the final output with TTP ID and sub ID
                final_output = f"{ttp_id}.{sub_id}" if sub_id else ttp_id
                return final_output  # Output: i.e. T1218.004 if sub ID exists, otherwise T1218

    return oldID

if __name__ == "__main__":
    ttpDict = {}
    ttps_map_file = os.path.join(CUCKOO_ROOT, "data", "mitre", "TTPs.json")
    if os.path.exists(ttps_map_file):
        try:
            ttpDict = json.loads(open(ttps_map_file, "r").read())
        except Exception as e:
            print("Can't load TTPs.json file", e)

    import_package(modules.signatures)
    for sig in list_plugins(group="signatures"):
        if not sig.ttps:
            continue
        for ttp in sig.ttps:
            if "." in ttp or "U" in ttp or "S" in ttp:
                continue
            print(f"Fetching update for {sig.name} and TTP: {ttp}")
            ttpDict[ttp.upper()] = mapMitre(ttp)

    # Save the configuration to a file
    open(ttps_map_file, "w").write(json.dumps(ttpDict, indent=4))
    print("Done")
    # print(mapMitre("T1215"))


"""
17. Letter Combinations of a Phone Number

Given a string containing digits from 2-9 inclusive, return all possible letter combinations
that the number could represent. A mapping of digit to letters (just like on the telephone buttons)
is given below. Note that 1 does not map to any letters.

Example:
Input: "23"
Output: ["ad", "ae", "af", "bd", "be", "bf", "cd", "ce", "cf"].

Time and space: 3^N * 4^M
"""

class Solution:
    def letterCombinations(self, digits):
        """
        :type digits: str
        :rtype: List[str]
        """
        self.d = {'0': '','1': '','2': 'abc','3': 'def','4': 'ghi','5': 'jkl',
        '6': 'mno','7': 'pqrs','8': 'tuv','9': 'wxyz'}

        combinations = list()

        def helper(strs, path):
            print(strs, path)
            if not strs:
                combinations.append(path)
                return
            first, rest = strs[0], strs[1:]
            letters = self.d[first]
            for letter in letters:
                helper(rest, path+letter)

        helper(digits, "")
        return [] if digits == "" else combinations

class Solution:
    def letterCombinations(self, digits: str) -> List[str]:
        self.data = {"1":"", "2":"abc", "3":"def", "4":"ghi", "5":"jkl", "6":"mno", "7":"pqrs","8":"tuv","9":"wxyz","0":" "}
        if not digits:
            return []
        result = [""]
        for d in digits:
            nextLevel = []
            letters = self.data[d]
            for let in letters:
                for strs in result:
                    nextLevel.append(strs+let)
            result = nextLevel
        return result


class Solution:
    def countOperations(self, num1: int, num2: int) -> int:
        count=0
        if num1==0 or num2==0:
            return 0
        while(True):
            if num1>=num2:
                num1-=num2
            elif num2>=num1:
                num2-=num1
            count+=1
            if num1==0 or num2==0:
                break
        return count


#create student information form using TKinter
#name,address,email,schoolname,stding in,year,dob,gender,schooltype- Marathi medium,english,convent,sem
import tkinter as tk
from tkinter.constants import X
win=tk.Tk()
win.title("studinfo")

win.geometry('600x600+100+200')
name=tk.Label(win,text="FullName:").place(x=20,y=30)

address=tk.Label(win,text="Address :").place(x=20,y=60)
Email=tk.Label(win,text="Address :").place(x=20,y=90)
Schoolname=tk.Label(win,text="Address :").place(x=20,y=120)
Stding_in_year=tk.Label(win,text="Address :").place(x=20,y=150)
dob=tk.Label(win,text="DOB").place(x=20,y=180)

tp=tk.Label(win,text="_________________________________________________________").place(x=10,y=260)
e1=tk.Entry(win).place(x=80,y=30)
e2=tk.Entry(win).place(x=80,y=60)
e3=tk.Entry(win).place(x=80,y=90)
e4=tk.Entry(win).place(x=80,y=120)
e5=tk.Entry(win).place(x=80,y=150)
e6=tk.Entry(win).place(x=80,y=180)
Gender=tk.Label(win,text="Select Gender").place(x=20,y=280)
radio1=tk.Radiobutton(win,text="male",value=1).place(x=20,y=300)
radio2=tk.Radiobutton(win,text="Femail",value=0).place(x=20,y=320)

tp2=tk.Label(win,text="_________________________________________________________").place(x=10,y=350)
schooltype=tk.Label(win,text="Schooltype").place(x=20,y=370)
radio3=tk.Radiobutton(win,text="Marathi",value=1).place(x=20,y=390)
radio4=tk.Radiobutton(win,text="English",value=0).place(x=20,y=420)

tp3=tk.Label(win,text="_________________________________________________________").place(x=10,y=440)
School_Caregory=tk.Label(win,text="School_Caregory").place(x=20,y=460)
radio5=tk.Radiobutton(win,text="Convent",value=0).place(x=20,y=480)
radio6=tk.Radiobutton(win,text="Semi",value=1).place(x=20,y=500)

tp4=tk.Label(win,text="_________________________________________________________").place(x=10,y=520)
submit=tk.Button(win,text="Submit",activebackground="red",activeforeground="blue").place(x=10,y=550)

win.mainloop()


import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import os
import time
from tqdm import *
import json

if __name__ == '__main__':

    # Setup Firefox to work with selenium for FPbase
    options = Options()
    options.headless = True
    driver = webdriver.Firefox(options=options)
    driver.get('https://www.fpbase.org/protein/avgfp')
    time.sleep(5)
    source = driver.page_source
    driver.quit()

    # Parse the HTML
    soup = BeautifulSoup(source, 'html.parser')

    # Get all the fluorescent proteins in the tree
    fps = {}

    for node in soup.find_all('g', {'class': 'node'}):
        link = node.find('a')['xlink:href']
        safe_name = os.path.basename(os.path.normpath(link))
        text = node.find('text')
        if text:
            name = text.contents[0]
        else:
            name = 'avGFP'
        fps[safe_name] = {'link': link,
                          'name': name}

    # Get the data for each individual protein
    for protein in tqdm(fps):
        resp = requests.get('https://www.fpbase.org/protein/{}'.format(protein))
        soup = BeautifulSoup(resp.content, 'html.parser')

        seq_div = soup.find('div', {'class': 'sequence'})
        sequence = seq_div.find('div', {'class': 'aminosequence'}).contents[0]

        fps[protein]['sequence'] = sequence

        deriv_div = seq_div.find('p', {'class': 'text-muted'})

        if deriv_div:
            parent = os.path.basename(os.path.normpath(deriv_div.find('a')['href']))
            mutations = deriv_div.find('span', {'class': 'mut-rel-root'}).contents[0].split('/')

            fps[protein]['parent'] = parent
            fps[protein]['mutations'] = mutations

    # Write out the sequence datasets
    with open('../examples/gfp/gfp.fa', 'w') as handle:
        for fp in fps:
            handle.write('>{}\n'.format(fp))
            handle.write('{}\n'.format(fps[fp]['sequence']))

    with open('../examples/gfp/gfp_exclude_long.fa', 'w') as handle:
        for fp in fps:
            seq = fps[fp]['sequence']
            if len(seq) in [238, 239]:
                handle.write('>{}\n'.format(fp))
                handle.write('{}\n'.format(seq))

    with open('../examples/gfp/gfp_239.fa', 'w') as handle:
        for fp in fps:
            seq = fps[fp]['sequence']
            if len(seq) == 239:
                handle.write('>{}\n'.format(fp))
                handle.write('{}\n'.format(seq))

    # Write all of the data out to a JSON file
    with open('../aux_data/gfp.json', 'w') as handle:
        json.dump(fps, handle)


n = int(input())
a = list(map(int,input().split()))
x = 0
for i in range(n):
  if a[x] < a[i]:
    x  = i
answer1 = 0
for i in range(x):
  answer1 += a[i]
answer2 = 0
for i in range(x+1,n):
  answer2 += a[i]
print(answer1)
print(answer2)


import sys
MOD = 1000000007

def pow(x, n):
    if n == 1:
        return x
    if n % 2 == 0:
        return (pow(x, n//2)**2) % MOD
    else:
        return (pow(x, (n-1)//2)**2 * x) % MOD

M = int(input())
fac = [1]
s = 1
for i in range(1, 4000001):
    s = (s*i) % MOD
    fac.append(s)

for i in range(M):
    N, K = map(int, sys.stdin.readline().strip().split())
    a = fac[N]
    b = fac[K]
    c = fac[N-K]
    temp = pow((b*c) % MOD, MOD-2)
    total = a*temp % MOD
    sys.stdout.write(str(total)+'\n')

백준/Gold/13977. 이항 계수와 쿼리/이항 계수와 쿼리.py

이항 계수와 쿼리.py


import os
import time
import requests as req
from termcolor import colored
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec

def download_file(name: str, url: str) -> str:
    options = Options()
    options.add_argument('--headless')
    options.add_argument('--log-level=3')
    browser = Chrome(chrome_options=options)
    print('\nPlease wait, It make take while....')
    browser.get(url)
    print('\nGetting download link....')
    try:
        f = WebDriverWait(browser, 20).until(ec.presence_of_element_located((By.LINK_TEXT, 'Download ( PDF )')))
    except:
        f = browser.find_element_by_link_text('Go to PDF')
    link = str(f.get_attribute('href'))
    print('\nStarting Download.....\n')
    finished = False
    try:
            with open('book.pdf', 'wb') as pdf:
                res = req.get(link, stream=True)
                length = res.headers.get('content-length')
                if length is None:
                    pdf.write(res.content)
                else:
                    downloaded = 0
                    length = int(length)/(1024*1024)
                    for content in res.iter_content(chunk_size=4096):
                        pdf.write(content)
                        downloaded += len(content)/(1024*1024)
                        per = round((downloaded/length)*100)
                        print(f"<|{('█'*round(per/10)).ljust(20, ' ')}|> {per}%  ( {round(downloaded, 2)}MB | {round(length, 2)}MB )", end='\r')
                    finished = True
                    print(colored(f'\n\nDownload Successful...\n\nFile saved at {os.getcwd()}', 'green'))
                    print('\nDon\'t close until program ends....')
    except Exception as err:
            print(colored(err, 'Error happend during downloading ', 'red'))
            os.remove(f'{name}.pdf')
            finished = False

    time.sleep(2)
    browser.close()
    os.rename('book.pdf', f'{name}.pdf')
    return finished


from pyspark.sql import SparkSession
from pyspark import SparkConf
from contextlib import contextmanager

@contextmanager
def SparkIO(conf: SparkConf = SparkConf()):
    app_name = conf.get("spark.app.name")
    master = conf.get("spark.master")
    print(f'Create SparkSession app {app_name} with {master} mode')
    spark = SparkSession.builder.config(conf=conf).getOrCreate()
    try:
        yield spark
    except Exception:
        raise Exception
    finally:
        print(f'Stop SparkSession app {app_name}')
        spark.stop()


import bts
from sympy.ntheory import primefactors, factorint
import gmpy2
from gmpy2 import mpz
# add, divexact, f_mod, mpz_urandomb + random_state, mul, powmod, isqrt

# вычисляет порядок элемента g в группе F*_p
def order(g, p):
    # полагаем, что порядок максимальный
    t = p - 1
    while 1:
        # разложение на простые множители числа t (каноническое разложение)
        factors = primefactors(t)
        ord = t
        for i in range(len(factors)):
            # t_i делит p-1
            # t_i = t // factors[i]
            t_i = gmpy2.divexact(mpz(t), mpz(factors[i]))
            # g_i = pow(g, t_i, p)
            g_i = gmpy2.powmod(mpz(g), mpz(t_i), mpz(p))
            # если g^t_i сравнимо с 1, то порядок g можно уменьшить до t_i
            if g_i == 1:
                t = t_i
                break
        # в случае, если g^t_i не сравнимо с 1 для всех t_i делителей p-1, то порядок g равен t
        if ord == t:
            return ord

# решение сравнения g^x=h(mod p) методом BSGS, ord(g)=N
def shanks(p, g, h, N):
    print("Shanks")
    # выбор n
    # n = 1 + int(pow(N, 1/2))
    n = 1 + gmpy2.isqrt(mpz(N))
    print("N =", N, "\nn =", n)
    # l = []
    # x=a+b*n
    a = 0
    b = 0
    # формирование списка (бинарного дерева поиска) BS, g^i, 0<=i<n
    # для хранения списка используется бинарное дерево поиска
    g_i = 1
    l = bts.BTS(0, g_i)
    for i in range(n):
        # l.append(pow(g, i, p))
        # l.append(g_i)
        l.insert(i, g_i)
        # g_i = (g_i * g) % p
        g_i = gmpy2.f_mod(mpz(gmpy2.mul(mpz(g_i), mpz(g))), mpz(p))
    # поиск мультипликативного обратного для g и возведение его в степень n
    # g_inv = pow(g, -n, p)
    g_inv = gmpy2.powmod(mpz(g), mpz(-n), mpz(p))
    # формирование списка GS, h*g^-nj, 0<=j<n
    # g_0 = h % p
    g_j = gmpy2.f_mod(mpz(h), mpz(p))
    for j in range(n):
        # g_j = (h * pow(g_inv, j, p)) % p
        # проверка на пересечение со списком BS
        # if g_j in l:
        ind = l.search(g_j)
        if ind != -1:
            b = j
            # a = l.index(g_j)
            a = ind
            break
        # пересечение списков не найдено, логарифм не существует
        if ind == -1 and j == n - 1:
            return -1
        # g_j = (g_j * g_inv) % p
        g_j = gmpy2.f_mod(mpz(gmpy2.mul(mpz(g_j), mpz(g_inv))), mpz(p))
    print("x = i + j * n =", a, "+", b, "*", n, "=", a + b * n)
    return a + b * n

# алгоритм сведения к собственным подгруппам, решающий сравнение g^x=h(mod p), ord(g)=N=p'^e с помощью алгоритма Шенкса
def group(p, g, h, N):
    print("Group")
    x = []
    # определяем p' и e, в данном случае ord=p'
    factors = factorint(N)
    ord, e = sorted(factors.items())[0]
    print("N = p ^ e =", ord, "^", e)
    if e == 1:
        return shanks(p, g, h, N)
    # представляем x=x_0+x_1*p'+x_2*p'^2+...+x_(e-1)*p'^(e-1)
    # g_i = pow(g, N // ord, p)
    g_i = gmpy2.powmod(mpz(g), mpz(gmpy2.divexact(mpz(N), mpz(ord))), mpz(p))
    # проходим циклом по всем x_k, 0<=k<e, решая сравнение g_(k+1)^x_k=h_(k+1)(mod p) с помощью алгоритма Шенкса
    # g_(k+1)=g_1=g^(p'^(e-1)) для любых k, поэтому вычисляется до начала цикла
    for i in range(1, e + 1):
        # вычислям значение показателя p'^(e-i)
        exp = pow(ord, e - i)
        print("Reduction to subgroups")
        print("p ^ (e - " + str(i) + ") =", ord, "^", e - i)
        # h_i = pow(h, exp, p)
        h_i = gmpy2.powmod(mpz(h), mpz(exp), mpz(p))
        # формируем значение h_i, которое будет зависеть от произведений g^(-x_j*p'^(e-i+j)), 0<=j<k
        # например, h_1=h^(p^(e-1)), h_2=h^(p^(e-2))*g^(-x_0*p^(e-2)), h_3=h^(p^(e-3))*g^(-x_0*p^(e-3))*g^(-x_1*p^(e-2)) и т.д.
        for j in range(len(x)):
            exp = pow(ord, e - i + j)
            # h_i *= pow(g, -x[j] * exp, p)
            h_i = gmpy2.mul(mpz(h_i), mpz(gmpy2.powmod(mpz(g), mpz(-x[j] * exp), mpz(p))))
        # h_i = h_i % p
        h_i = gmpy2.f_mod(mpz(h_i), mpz(p))
        print("g_" + str(i), "=", g_i)
        print("h_" + str(i), "=", h_i)
        print("Solving g_" + str(i), "^ x_" + str(i - 1), "= h_" + str(i), "(mod p)")
        print(g_i, "^ x_" + str(i - 1), "=", h_i, "( mod", p, ")")
        # решаем простое сравнение g_(k+1)^x_k=h_(k+1)(mod p) с помощью алгоритма Шенкса, ord(g_(k+1))=p' будет простым числом
        # откуда находим x_k
        x_k = shanks(p, g_i, h_i, ord)
        # если решения нет, то возвращаем -1
        if x_k == -1:
            return -1
        print("x_" + str(i - 1), "=", x_k, "( mod", ord, ")")
        x.append(x_k)
    res = 0
    # восстанавливаем x=x_0+x_1*p'+x_2*p'^2+...+x_(e-1)*p'^(e-1)
    for j in range(len(x)):
        res += x[j] * pow(ord, j)
    return res

# решение сравнения g^x=h(mod p), где ord(g)=N=p_1^e_1 * p_2^e_2 * ... * p_m^e_m
# сведением исходной задачи к набору задач DLP с ord(g_i)=p^e
def pohlig_hellman(p, g, h, N):
    # факторизуем порядок ord(g)=N
    factors = factorint(N)
    print("ord(g) = N =", N, "=", factors)
    y = []
    # редукция
    # проходим по всем сомножителям p_i^e_i числа N, находим N_i=N/p_i^e_i, g_i=g^N_i, h_i=h^N_i
    # решаем сравнение g_i^y_i=h_i(mod p_i^e_i) с помощью алгоритма сведения к собственным подгруппам
    i = 1
    for p_i, e_i in sorted(factors.items()):
        print("\nPohlig-Hellman\ni =", i)
        ord = pow(p_i, e_i)
        print("p_" + str(i), "=", p_i)
        print("e_" + str(i), "=", e_i)
        # N_i = N // ord
        N_i = gmpy2.divexact(mpz(N), mpz(ord))
        print("N_" + str(i), "=", N_i)
        # g_i = pow(g, N_i, p)
        g_i = gmpy2.powmod(mpz(g), mpz(N_i), mpz(p))
        print("g_" + str(i), "=", g_i)
        # h_i = pow(h, N_i, p)
        h_i = gmpy2.powmod(mpz(h), mpz(N_i), mpz(p))
        print("h_" + str(i), "=", h_i)
        print("Solving g_" + str(i), "^ y_" + str(i), "= h_" + str(i), "(mod p)")
        print(g_i, "^ y_" + str(i), "=", h_i, "( mod", p, ")")
        y_i = group(p, g_i, h_i, ord)
        # если решения нет, то возвращаем -1
        if y_i == -1:
            print("No solution")
            return -1
        y.append(y_i)
        print("y_" + str(i), "=", y_i, "( mod", ord, ")")
        i += 1
    # восстановление x
    # решаем систему сравнений с помощью КТО
    print("\nSolving comparison system")
    x = 0
    for i in range(len(y)):
        p_i, e_i = sorted(factors.items())[i]
        ord = pow(p_i, e_i)
        print("y_" + str(i + 1), "=", y[i], "( mod", ord, ")")
        # N_i = N // ord
        N_i = gmpy2.divexact(mpz(N), mpz(ord))
        # x += y[i] * N_i * pow(N_i, -1, ord)
        N_i_inv = gmpy2.invert(mpz(N_i), mpz(ord))
        x = gmpy2.add(mpz(x), mpz(gmpy2.mul(mpz(y[i]), mpz(gmpy2.mul(mpz(N_i), mpz(N_i_inv))))))
    return x % N


class AppException(Exception):
    """Base class for application exceptions"""

    def __init__(self, message='', payload=None):
        if payload is None:
            payload = {}
            super().__init__(message)
        self.message = message
        self.payload = payload

class DomainException(AppException):
    """Base exception for domain-related exceptions"""


fname = input('enter a file name: ')
try:
    with open(fname) as fhand:
        fline = fhand.readlines()
except:
    print(f'{fname} is not a file name!')
    quit()

counts_mail = {}

for line in fline:
    line = line.split()
    if len(line) != 0 and line[0] == 'From':
        if line[1] not in counts_mail:
            counts_mail[line[1]] = 1

        else:
            counts_mail[line[1]] += 1

lst = list()
for key, val in list(counts_mail.items()):
    lst.append((val,key))

lst.sort(reverse = True)

for key, val in lst[:1]:
    print(key, val)


# Tipo de dato, extensión en python:

#1) Cadena de texto | String, str

Nombre = str("Elkin")

print(type (Nombre))

#2) Números decimales, float

Estatura = float(1.84)

#print(type (Estatura))

#print (f"El nombre de la persona es {Nombre}, y su estatura es de {Estatura} metros")

#3) Números enteros, int

Edad = 30

#print(Edad)

#4) True / False (verdadero, falso)

Es_Mayor_de_edad = True

#print(Es_Mayor_de_edad)

# Crear un mini programa de salida con un input

Ingresesunombre = str(input("Ingrese por favor su nombre: "))

#print(Ingresesunombre)

print(f"Hola usuario: {Ingresesunombre}, su edad es de {Edad + 10}")

print("Hello Wordl")


import copy
import torch
import spacy
import numpy as np
import pandas as pd
from src.utils import read_file, split_ids

nlp = spacy.load("en_core_sci_md")

class ClassificationDataset(torch.utils.data.Dataset):
    """Holder object for the following items:
    @param input_ids... see HF documention.
    @param attention_mask... see HF documentation.
    @param questions are the questions to generate.
    @param question_attention_mask are the attention_masks for the questions
    to generate.
    @param ids is a list of strings that correspond to the ids of the questions
    @param triggers is a separate list of triggers."""

    def __init__(self, input_ids, attention_mask, questions, question_attn_mask, ids, triggers=None):
        self.input_ids = input_ids
        self.attention_mask = attention_mask
        self.questions = questions
        self.question_attn_mask = question_attn_mask
        self.ids = ids
        self.triggers = triggers

    def __getitem__(self, idx):
        item = {
            "input_ids": self.input_ids[idx],
            "attention_mask": self.attention_mask[idx],
            "labels": self.questions[idx],
            "decoder_attention_mask": self.question_attn_mask[idx],
            "id": self.ids[idx],
        }

        return item

    def __len__(self):
        return len(self.input_ids)

class BARTDataLoader:
    """Data loader class for BART model.
    @param data_loc is the file location of the dataframe.
    @param relations_text_loc is the folder location of the n2c2 discharge summaries.
    @param tokenizer is the Huggingface tokenizer.
    @param max_len is the maximum length of a sequence from the passage.
    @param max_q_len is the maximum question length.
    @param chunk_mode will determine how we split up discharge summaries.
    @param split_questions if true, will take a sequence of questions for the
    same trigger and split it up into multiple labels.
    @param trigger_aware if true, will append the trigger to the beginning of
    the inputs to the model.
    @param stride is the typical stride when examining LMs.
    @param val_split is the validation split percent
    @param test_split is the test split percent.
    @param seed is the seed to split the data with."""

    def __init__(
        self,
        data_loc: str,
        relations_text_loc: str,
        tokenizer,
        debug_mode: bool = False,
        max_len: int = 512,
        max_q_len: int = 184,
        max_trigger_len: int = 184,
        chunk_mode: str = "sentence-level",
        split_questions: bool = False,
        stride: int = 512,
        val_split: int = 0.10,
        test_split: int = 0.20,
        seed: int = 0,
    ):

        self.data_loc = data_loc
        self.relations_text_loc = relations_text_loc
        self.tokenizer = tokenizer
        self.debug_mode = debug_mode
        self.max_len = max_len
        self.max_q_len = max_q_len
        self.chunk_mode = chunk_mode
        self.split_questions = split_questions
        self.max_trigger_len = max_trigger_len
        self.stride = stride
        self.val_split = val_split
        self.test_split = test_split
        self.seed = seed

    def _split_data_into_sentences_more_context(self, all_tokenized_outputs):
        """Split the data into sentences and tokenize + pad it.
        @param all_tokenized_outputs is a list of dictionaries with the following input:
            @param input_ids is the input ids of the discharge summary.
            @param trigger_location is a list of 1s and 0s into @param input_ids.
            @param trigger_ids is a list of ids of the text for the trigger_ids.
            @param id is the discharge summary id.
            @param question is ids of the question asked (padded already).
            @param question_attn_mask is the attention mask of the @param question.
            @param sentence_offsets is the offset of the start of each sentence.

        @return a ClassificationDataset instance."""
        xs, questions, question_attn_mask, attn_mask, ids, triggers = [], [], [], [], [], []
        prompt = self.tokenizer.encode(
            " After reading the above EMR, what question do you have about ", add_special_tokens=False
        )
        end_prompt = self.tokenizer.encode("? Question: ", add_special_tokens=False)
        for idx, row in enumerate(all_tokenized_outputs):
            tg_st = row["trigger_location"].index(1)
            tg_end = tg_st + sum(row["trigger_location"])
            sent_onset = np.where(np.asarray(row["sentence_offsets"]) > tg_st)[0][0] - 1
            sent_end = np.where(np.asarray(row["sentence_offsets"]) > tg_end)[0][0]

            tk_onset = row["sentence_offsets"][sent_onset]
            tk_end = row["sentence_offsets"][sent_end]
            selected_sent = row["input_ids"][tk_onset:tk_end][: self.max_len]
            pad = (self.max_len + self.max_trigger_len + len(prompt) + len(end_prompt)) - (
                len(selected_sent) + len(row["trigger_ids"])
            )
            input_ = [self.tokenizer.bos_token_id] + selected_sent
            input_ += prompt + row["trigger_ids"][: self.max_trigger_len] + end_prompt + [self.tokenizer.eos_token_id]
            pad_input = input_ + [self.tokenizer.pad_token_id] * pad

            # Add input_ids, input attn mask, trigger locations, question_ids,
            # question attn map, and the article ID.
            xs.append(pad_input)
            attn_mask.append([1] * len(input_) + [0] * pad)
            questions.append(row["question"][: self.max_q_len])
            question_attn_mask.append(row["question_attn_mask"])
            ids.append(row["id"])
            triggers.append(row["trigger_str"])

        return ClassificationDataset(xs, attn_mask, questions, question_attn_mask, ids, triggers=triggers)

    def _split_data_into_strides(self, all_tokenized_outputs):
        """Split the data into @param stride sized chunks and tokenize + pad it.
        @param all_tokenized_outputs is a list of dictionaries with the following input:
            @param input_ids is the input ids of the discharge summary.
            @param trigger_location is a list of 1s and 0s into @param input_ids.
            @param trigger_ids is a list of ids of the text for the trigger_ids.
            @param id is the discharge summary id.
            @param question is ids of the question asked (padded already).
            @param question_attn_mask is the attention mask of the @param question.
            @param sentence_offsets is the offset of the start of each sentence.

        @return a ClassificationDataset instance."""
        # Now split up data to max_len + stride.
        xs, questions, question_attn_mask, attn_mask, ids, triggers = [], [], [], [], [], []
        prompt = self.tokenizer.encode(
            " After reading the above EMR, what question do you have about ", add_special_tokens=False
        )
        end_prompt = self.tokenizer.encode("? Question: ", add_special_tokens=False)
        for idx, row in enumerate(all_tokenized_outputs):
            for i in range(0, len(row["input_ids"]), self.stride):
                pad = (self.max_len + self.max_trigger_len + len(prompt) + len(end_prompt)) - (
                    len(row["input_ids"][i : i + self.max_len - 2]) + len(row["trigger_ids"])
                )
                input_ = [self.tokenizer.bos_token_id] + row["input_ids"][i : i + self.max_len - 2]
                input_ = input_ + prompt + row["trigger_ids"] + end_prompt + [self.tokenizer.eos_token_id]
                pad_input = input_ + [self.tokenizer.pad_token_id] * pad

                # If all the labels are 0, there are no questions here! Discard.
                trig = row["trigger_location"][i : i + len(input_)]
                if sum(trig) == 0:
                    continue

                # Add input_ids, input attn mask, trigger locations, question_ids,
                # question attn map, and the article ID.
                xs.append(pad_input)
                attn_mask.append([1] * len(input_) + [0] * pad)
                questions.append(row["question"])
                question_attn_mask.append(row["question_attn_mask"])
                ids.append(row["id"])
                triggers.append(row["trigger_str"])

        return ClassificationDataset(xs, attn_mask, questions, question_attn_mask, ids, triggers=triggers)

    def _split_data_into_no_inputs(self, all_tokenized_outputs):
        """Split the data into @param stride sized chunks and tokenize + pad it.
        @param all_tokenized_outputs is a list of dictionaries with the following input:
            @param input_ids is the input ids of the discharge summary.
            @param trigger_location is a list of 1s and 0s into @param input_ids.
            @param trigger_ids is a list of ids of the text for the trigger_ids.
            @param id is the discharge summary id.
            @param question is ids of the question asked (padded already).
            @param question_attn_mask is the attention mask of the @param question.
            @param sentence_offsets is the offset of the start of each sentence.

        @return a ClassificationDataset instance."""
        to_pad_length = max([len(x["trigger_ids"]) for x in all_tokenized_outputs])
        xs, questions, question_attn_mask, attn_mask, ids, triggers = [], [], [], [], [], []
        for idx, row in enumerate(all_tokenized_outputs):
            pad_tokens = to_pad_length - len(row["trigger_ids"])
            xs.append(row["trigger_ids"] + [self.tokenizer.pad_token_id] * pad_tokens)
            attn_mask.append([1] * len(row["trigger_ids"]) + [0] * pad_tokens)
            questions.append(row["question"])
            question_attn_mask.append(row["question_attn_mask"])
            ids.append(row["id"])
            triggers.append(row["trigger_str"])

        return ClassificationDataset(xs, attn_mask, questions, question_attn_mask, ids, triggers)

    def _tokenize_df(self, df, is_training):
        """Turn a dataframe into tokenized + padded inputs + labels.
        @param df is the dataframe to process.
        @param is_training is if the dataframe output is for training or evaluating."""
        all_tokenized_outputs = []
        for index, row in df.iterrows():
            text, remapper = read_file(row.id, self.relations_text_loc)
            tokenized_output = self.tokenizer(text, return_offsets_mapping=True, add_special_tokens=False)

            # Find the offset
            labels = [0] * len(tokenized_output.input_ids)
            new_st, new_end = remapper[row.start_index], remapper[row.end_index]
            tok_st = np.where(np.asarray(tokenized_output.offset_mapping) > new_st)[0][0]
            tok_end = np.where(np.asarray(tokenized_output.offset_mapping) > new_end)[0][0]
            for i in range(tok_st, tok_end + 1):
                labels[i] = 1

            # Add to our map
            tk_q = self.tokenizer(row.question, max_length=self.max_q_len, padding="max_length")
            tk_tr = self.tokenizer(
                row.reasoning, max_length=self.max_trigger_len, truncation=True, add_special_tokens=False
            )
            sentence_ch_offsets = [x.end_char for x in nlp(text).sents]
            sentence_tk_offsets = [
                np.where(np.asarray(tokenized_output.offset_mapping) > x)[0][0] for x in sentence_ch_offsets[:-1]
            ]
            sentence_tk_offsets.append(len(tokenized_output.input_ids) - 1)

            outputs_ = {
                "input_ids": tokenized_output.input_ids,
                "trigger_location": labels,
                "trigger_ids": tk_tr["input_ids"],
                "id": row.id,
                "question": tk_q["input_ids"],
                "question_attn_mask": tk_q["attention_mask"],
                "sentence_offsets": sentence_tk_offsets,
                "trigger_str": row.reasoning,
            }

            if self.split_questions and is_training:
                tk_qs = [
                    self.tokenizer(q + "?", max_length=self.max_q_len, padding="max_length")
                    for q in row.question.split("?")[:-1]
                ]
                for q in tk_qs:
                    c_outputs_ = copy.deepcopy(outputs_)
                    c_outputs_["question"] = q["input_ids"]
                    c_outputs_["question_attn_mask"] = q["attention_mask"]
                    all_tokenized_outputs.append(c_outputs_)
            else:
                all_tokenized_outputs.append(outputs_)

        if self.chunk_mode == "sentence-level":
            return self._split_data_into_sentences_more_context(all_tokenized_outputs)
        elif self.chunk_mode == "split-level":
            return self._split_data_into_strides(all_tokenized_outputs)
        else:
            return self._split_data_into_no_inputs(all_tokenized_outputs)

    def tokenize_generated_df(self, df):
        """Turn a dataframe into tokenized + padded inputs + labels. Use this function
        for when trying to get predictions on dataset with predicted triggers.
        @param df is the dataframe to process. This DF must have a column named
        `text`."""
        all_tokenized_outputs = []
        for index, row in df.iterrows():
            text = row["text"]

            tokenized_output = self.tokenizer(text, return_offsets_mapping=True, add_special_tokens=False)

            # Find the offset
            labels = [0] * len(tokenized_output.input_ids)
            new_st, new_end = row.st_ch, row.end_ch
            try:
                tok_st = np.where(np.asarray(tokenized_output.offset_mapping) > new_st)[0][0]
                tok_end = np.where(np.asarray(tokenized_output.offset_mapping) > new_end)[0][0]
            except:
                tok_end = len(tokenized_output.offset_mapping) - 1

            for i in range(tok_st, tok_end + 1):
                labels[i] = 1

            # Add to our map
            tk_tr = self.tokenizer(
                row.trigger, max_length=self.max_trigger_len, truncation=True, add_special_tokens=False
            )
            sentence_ch_offsets = [x.end_char for x in nlp(text).sents]
            sentence_tk_offsets = [
                np.where(np.asarray(tokenized_output.offset_mapping) > x)[0][0] for x in sentence_ch_offsets[:-1]
            ]
            sentence_tk_offsets.append(len(tokenized_output.input_ids) - 1)

            outputs_ = {
                "input_ids": tokenized_output.input_ids,
                "trigger_location": labels,
                "trigger_ids": tk_tr["input_ids"],
                "id": row.id,
                "question": [self.tokenizer.bos_token_id],
                "question_attn_mask": [1],
                "sentence_offsets": sentence_tk_offsets,
                "trigger_str": row.trigger,
            }

            all_tokenized_outputs.append(outputs_)

        if self.chunk_mode == "sentence-level":
            return self._split_data_into_sentences_more_context(all_tokenized_outputs)
        elif self.chunk_mode == "split-level":
            return self._split_data_into_strides(all_tokenized_outputs)
        else:
            return self._split_data_into_no_inputs(all_tokenized_outputs)

    def load_data(self) -> (ClassificationDataset, ClassificationDataset, ClassificationDataset):
        """Load the data and split it."""
        df = pd.read_csv(self.data_loc)
        ordered_ids = []
        seen = set()
        for id_ in list(df.id.values):
            if not (id_ in seen):
                seen.add(id_)
                ordered_ids.append(id_)

        tr_ids, val_ids, test_ids = split_ids(
            ordered_ids, seed=self.seed, val_split=self.val_split, test_split=self.test_split
        )

        # Create each df
        tr_df = df[df.id.isin(tr_ids)]
        vl_df = df[df.id.isin(val_ids)]
        te_df = df[df.id.isin(test_ids)]

        # If we are in debug mode, use a smaller percent of the data.
        if self.debug_mode:
            tr_df = tr_df[: int(len(tr_df) * 0.1)]
            vl_df = vl_df[: int(len(vl_df) * 0.1)]
            te_df = te_df[: int(len(te_df) * 0.1)]

        # Now tokenize each df
        tr_set = self._tokenize_df(tr_df, is_training=True)
        vl_set = self._tokenize_df(vl_df, is_training=False)
        te_set = self._tokenize_df(te_df, is_training=False)
        return tr_set, vl_set, te_set