"""
RecImpute - A Recommendation System of Imputation Techniques for Missing Values in Time Series,
eXascale Infolab, University of Fribourg, Switzerland
***
ModelsTrainer.py
@author: @chacungu
"""

import itertools
import logging
from multiprocessing import Pool, Manager
import numpy as np
import operator
import pandas as pd
import random as rdm
from scipy.stats import ttest_rel
from sklearn import pipeline
from sklearn.base import clone
from sklearn.ensemble import VotingClassifier
from sklearn.model_selection import StratifiedKFold, train_test_split as sklearn_train_test_split
from sklearn.pipeline import Pipeline
from tqdm import tqdm # .notebook

from Training.ClfPipeline import ClfPipeline
from Training.RecommendationModel import RecommendationModel
from Training.TrainResults import TrainResults
from Utils.Utils import Utils

def _parallel_training(args):
    """
    Trains and evaluates a pipeline. This method is used by ModelRace to train pipelines in parallel.

    Keyword arguments:
    pipe -- the ClfPipeline to train
    train_index_cv -- list of indices of the training samples in Xp_train
    Xp_train -- numpy array of train entries
    yp_train -- numpy array of train entries' labels
    X_val -- numpy array of validation entries
    y_val -- numpy array of validation entries' labels
    data_cols -- training set's features name
    labeler_properties -- dict specifying the labels' properties
    labels_set -- list of unique labels that may appear in y_train and y_val

    Return:
    If an error was thrown: the pipe's id
    Otherwise:
    1. the pipe's id
    2. the measured F1-score
    3. the measured recall@3
    4. the training time
    """
    pipe, train_index_cv, rmvd_pipes, Xp_train, yp_train, X_val, y_val, data_cols, labeler_properties, labels_set = args
    if pipe.rm.id not in rmvd_pipes:
        # get the stratified fold data
        Xp_train_cv = Xp_train.iloc[train_index_cv]
        yp_train_cv = yp_train.iloc[train_index_cv]
        assert Xp_train_cv.index.identical(yp_train_cv.index)
        Xp_train_cv = Xp_train_cv.to_numpy().astype('float32')
        yp_train_cv = yp_train_cv.to_numpy().astype('str').flatten()

        # training & evaluation
        try:
            with Utils.catchtime('Training pipe %s' % pipe.rm.pipe, verbose=False) as t:
                metrics_, _ = pipe.rm.train_and_eval(Xp_train_cv, yp_train_cv, X_val, y_val,
                                                    data_cols, labeler_properties, labels_set,
                                                    plot_cm=False, save_if_best=False)
            runtime = t.end - t.start
            return pipe.rm.id, metrics_['F1-Score'], metrics_['Recall@3'], runtime
        except:
            logging.exception('Error for pipe %s\n\n\n' % pipe) # TODO tmp print
            return pipe.rm.id

class ModelsTrainer:
    """
    Class which handles classification / regression models and provides methods for its training and evaluation.
    """

    CONF = Utils.read_conf_file('modelstrainer')

    # constructor

    def __init__(self, training_set):
        """
        Initializes a ModelsTrainer object.

        Keyword arguments:
        training_set -- TrainingSet instance
        """
        self.training_set = training_set
        self.models = None # list of RecommendationModels

    # public methods

    def select(self, pipelines, all_pipelines_txt, S, selection_len, score_margin,
               training_set_params=None, n_splits=3, test_method=ttest_rel, p_value=.01, alpha=.5, beta=.5, gamma=.5,
               allow_early_eliminations=True, early_break=False):
        """
        Selects (and partially trains if pipelines' training can be paused and resumed) the most-promising pipelines.

        Keyword arguments:
        pipelines -- list of ClfPipeline instances to select from
        all_pipelines_txt -- list of tuples containing the description of each pipeline (steps and params)
        S -- list of percentages for the partial training data sets
        selection_len -- approximate number of pipelines that should remain after the selection process is done.
        score_margin -- Scores acceptance margin. During selection, if a pipeline performs below MAX_SCORE - MARGIN it gets
                        eliminated before even finishing cross-validation. Scores vary between 0 and 1.
        training_set_params -- dict specifying the data's properties (e.g. should it be balanced, reduced, etc.) (default: None)
        n_splits -- number of k-fold stratified splits (default: 3)
        test_method -- Significance test. Takes as input two lists of equal length and returns a tuples which 2nd value is the
                       p-value. (default: scipy.stats.ttest_rel)
        p_value -- p_value used with the paired t-test to decide if a difference is significant or not (default: 0.01).
        alpha -- alpha parameter in the score function. Weight of F1-Score. Must be between 0 and 1 (default: .5).
        beta -- beta parameter in the score function. Weight of Recall@3. Must be between 0 and 1 (default: .5).
        gamma -- gamma parameter in the score function. Weight of training time. Must be between 0 and 1 (default: .5).
        allow_early_eliminations -- True if early eliminations are allowed, False if every model should finish their cv-partial-training
                                     even if early results show evidences of bad performance (default True).
        early_break -- True if the process can stop before all the iterations are done IF the target number of pipes has been reached,
                       False if all the iterations should be executed before returning. (default False)

        Return:
        List of selected ClfPipeline.
        """
        try:
            training_set_params = self.training_set.get_default_properties() if training_set_params is None else training_set_params

            all_data, _, labels_set, X_train, y_train, X_val, y_val = next(self.training_set.yield_splitted_train_val(training_set_params, 1))
            X_train, y_train = pd.DataFrame(X_train), pd.DataFrame(y_train)
            assert X_train.index.identical(y_train.index)
            print('\n0/3 - data loaded') # TODO tmp print

            init_nb_pipes = len(pipelines)
            pruning_factor = (init_nb_pipes / selection_len)**(1/len(S))

            get_max_nb_p_at_i = lambda iter_id: round(init_nb_pipes // (pruning_factor**iter_id))

            manager = Manager()

            train_index = []

            with Utils.catchtime('ModelRace runtime', verbose=True): # TODO tmp print
                for i in tqdm(range(len(S)), leave=False):
                    with Utils.catchtime('Selection iteration %i' % (i+1), verbose=True): # TODO tmp print

                        logging.info('Iteration %i started with following pipelines:')
                        for pipe in pipelines:
                            logging.info(pipe)

                        n = (X_train.shape[0] * (S[i] - (S[i-1] if i > 0 else 0))) // 100 # number of data to add for partial training

                        # generate new pipelines from the remaining set of candidates
                        if i > 0:
                            max_nb_pipes_at_iter_i = get_max_nb_p_at_i(i) # init_nb_pipes // (pruning_factor**i)
                            nb_pipes_to_generate = max(max_nb_pipes_at_iter_i - len(pipelines), len(pipelines)//10)
                            new_pipes = ClfPipeline.generate_from_set(pipelines, all_pipelines_txt, nb_pipes_to_generate)
                            pipelines.extend(new_pipes)
                            print('\nTried to generate %i new pipelines.' % nb_pipes_to_generate) # TODO tmp print
                            print('Generated %i new pipelines from the remaining candidates (max nb pipes at iter %i is %i).' % (len(new_pipes), i, max_nb_pipes_at_iter_i))

                        # prepare the partial training set
                        X_train_unused = X_train.loc[~X_train.index.isin(train_index)]
                        y_train_unused = y_train.loc[~y_train.index.isin(train_index)]
                        assert X_train_unused.index.identical(y_train_unused.index)

                        try:
                            train_index_new = sklearn_train_test_split(X_train_unused, stratify=y_train_unused, train_size=n)[0].index.tolist()
                        except:
                            train_index_new = sklearn_train_test_split(X_train_unused, train_size=n)[0].index.tolist()
                        assert all(id not in train_index for id in train_index_new)

                        train_index.extend(train_index_new)
                        print('\n1/3 - begining of new partial training: %i%% -> %i' % (S[i], len(train_index))) # TODO tmp print

                        Xp_train = X_train.loc[train_index]
                        yp_train = y_train.loc[train_index]
                        assert Xp_train.index.identical(yp_train.index)

                        # train, eval and perform statistic tests
                        # create params list
                        rmvd_pipes = manager.dict()
                        param_list = []
                        max_nb_scores = max(len(pipe.scores) for pipe in pipelines) + (n_splits if i > 0 else 10)
                        for pipe in pipelines:
                            n_splits_ = max_nb_scores - len(pipe.scores)
                            for train_index_cv, _ in StratifiedKFold(n_splits=n_splits_).split(Xp_train, yp_train):
                                param_list.append(
                                    (pipe, train_index_cv, rmvd_pipes, # dynamic params
                                    Xp_train, yp_train, X_val, y_val, all_data.columns, self.training_set.get_labeler_properties(), labels_set) # constant params
                                )
                        rdm.shuffle(param_list) # shuffle the param list such that the premature elimination is more efficient

                        # run _parallel_training
                        metrics = {}
                        max_score = -np.inf
                        with Pool() as pool:
                            for res in tqdm(pool.imap_unordered(_parallel_training, param_list), total=len(param_list), leave=False):
                                if res is not None:
                                    try:
                                        p_id, f1, r3, t = res
                                    except:
                                        # eliminate this pipe since there most-likely was an exception thrown during its training
                                        #  maybe due to a problematic parameters' combination?
                                        p_id = res
                                        rmvd_pipes[p_id] = True
                                        print('%i has been eliminated probably due to an exception being thrown' % p_id)
                                        continue

                                    # save the evaluation results
                                    if p_id not in metrics:
                                        metrics[p_id] = []
                                    metrics[p_id].append((f1, r3, t))

                                    # early eliminations
                                    if allow_early_eliminations and len(metrics[p_id]) >= 5:
                                        mean_score = np.mean([f1 for (f1,_,_) in metrics[p_id]]) # avg of f1-scores
                                        if mean_score > max_score:
                                            max_score = mean_score
                                        elif mean_score < max_score - score_margin: # eliminate prematurely if the pipe performs really poorly
                                            rmvd_pipes[p_id] = True # eliminate this pipe prematurely (do not finish its cross-validation training)
                                            print('%i has been eliminated early: avg_score=%.2f and max_score=%.2f' % (p_id, mean_score, max_score))

                        print('\n%i pipelines\' training have been stopped prematurely due to poor performances.' % len(rmvd_pipes)) # TODO tmp print

                        print('\n3/3 - statistic tests') # TODO tmp print
                        # normalize runtime between 0 and 1 & compute the score of each pipeline on each cv-split
                        metrics = {p_id: val for p_id, val in metrics.items() if p_id not in rmvd_pipes}
                        max_runtime = max([i[2] for cv_scores in metrics.values() for i in cv_scores])
                        scores_ = {id: [ModelsTrainer._compute_selection_score(f1,r3,t/max_runtime, alpha, beta, gamma)
                                        for f1,r3,t in cv_scores]
                                    for id, cv_scores in metrics.items()}

                        # add the newly measured scores to the global scores list & remove the pipes that were stopped prematurely
                        for i in range(len(pipelines) - 1, -1, -1):
                            pipe = pipelines[i]
                            if pipe.id in scores_:
                                pipe.scores.extend(scores_[pipe.id])
                            else:
                                del pipelines[i]

                        # statistical tests - remove pipes that are significantly worse than any other pipe
                        less_aggressive_pruning = (len(pipelines) <= get_max_nb_p_at_i(i)) if i > 0 else True
                        worse_pipes = self._apply_test(
                            pipelines,
                            test_method,
                            K=pruning_factor,
                            p_value=p_value,
                            less_aggressive_pruning=less_aggressive_pruning
                        )

                        if len(pipelines) < 20: # TODO tmp print
                            print([(p.id, np.mean(p.scores)) for p in pipelines]) # TODO tmp print
                            print([
                                (test_method(a.scores, b.scores)[1], a.rm.id,b.rm.id)  # we keep the worse pipelines of the 2
                                    for a,b in itertools.combinations(pipelines, r=2) # for all pairs of pipes
                            ]) # TODO tmp print

                        # remove the worse pipes
                        pipelines = [p for p in pipelines if p not in worse_pipes]

                        print('\nThere remains %i pipelines. %i have been eliminated by t-test.' % (len(pipelines), len(worse_pipes)))

                        if early_break and len(pipelines) <= selection_len:
                            break

                # if we have more pipes remaining that what we wanted
                if len(pipelines) > selection_len:
                    print('Too many pipelines remaining. Last attempt to eliminate "worse" candidates.')
                    pipelines = sorted(pipelines, key=lambda p: np.mean(p.scores), reverse=True)
                    for i in reversed(range(len(pipelines))): # rank pipes based on their avg scores
                        p_i = pipelines[i]

                        # pairwise ttest if p is worse than any other: prune p
                        for j, p_j in enumerate(pipelines):
                            if i is not j and test_method(p_i.scores, p_j.scores)[1] < p_value and np.mean(p_i.scores) < np.mean(p_j.scores):
                                print('%i was eliminated due to significantly worse performances than another candidate.' % p_i.id)
                                del pipelines[i]
                                break

                        if len(pipelines) <= selection_len:
                            break

        except Exception as e:
            logging.exception('Got exception while selecting pipelines.')
            if len(pipelines) >= init_nb_pipes:
                raise e
        return pipelines

    def train(self, models, train_on_all_data=False):
        """
        Trains and evaluates models given to this initialization of this trainer. Uses T-Daub for best-algorithms selection
        and cross-validation.

        Keyword arguments:
        models -- list of RecommendationModel instances that should be trained and evaluated
        train_on_all_data -- True if the models should be trained on ALL data once the cross-val is done, False
                             otherwise (default: False)

        Return:
        A TrainResults' instance containing the training results
        """

        vc_estimators = []
        for m in models:
            est = m.get_trained_pipeline(use_pipeline_prod=False)
            vc_estimators.append( (str(m.id), est if est is not None else m.pipe) )
        vc = VotingClassifier(vc_estimators, voting='soft')

        models.insert(
            0,
            RecommendationModel(-1, 'classifier', Pipeline([('voting ensemble', vc)]))
        )

        self.models = models

        train_results = self._train(models, train_on_all_data=train_on_all_data)
        return train_results

    # private methods

    def _apply_test(self, pipelines, test_method, K=4, p_value=.01, less_aggressive_pruning=True):
        """
        Applies a significance test (paired t-test) to the pipelines to identify and return those that perform worse than others.

        Keyword arguments:
        pipelines -- list of ClfPipeline to apply t-test to.
        test_method -- Significance test. Takes as input two lists of equal length and returns a tuples which 2nd value is the p-value.
        K -- Pruning factor. Example: if K=4, the expected number of pipes to "survive" is 1/4 of the original set. (default: 4)
        p_value -- p_value used with the paired t-test to decide if a difference is significant or not (default: 0.01).
        less_aggressive_pruning -- uses a less aggressive pruning strategy if True, eliminates any pipe that is significantly worse than any other
                                   pipe otherwise. (default: True)

        Return:
        List of _TmpPipeline that are performing worse than others.
        """
        eliminated_pipes = []
        pipes_combinations = list(itertools.combinations(pipelines, r=2))
        worse_counters = {p: 0 for p in pipelines}

        T = len(pipelines) // K if less_aggressive_pruning else 0

        i = len(pipes_combinations)-1
        while i > 0:
            a,b = pipes_combinations[i]

            # if the paired t-test shows a statistical difference between the two
            try: significance_diff = test_method(a.scores, b.scores)[1] < p_value
            except: significance_diff = True
            if significance_diff:
                worse_pipe = a if np.mean(a.scores) < np.mean(b.scores) else b
                worse_counters[worse_pipe] += 1

                if worse_counters[worse_pipe] > T:
                    # eliminate the worst one
                    eliminated_pipes.append(worse_pipe)
                    # remove all pairs of pipes continaining the eliminated pipe to avoid unnecessary comparisons
                    for j in reversed(range(len(pipes_combinations))):
                        x,y = pipes_combinations[j]
                        if worse_pipe in (x,y):
                            del pipes_combinations[j]
                            i -= 1 if j < i else 0
            i -= 1
        return eliminated_pipes

    def _train(self, models_to_train, train_on_all_data=False, training_set_params=None, save_results=True, save_if_best=True):
        """
        Trains and evaluates a list of models over cross-validation (and after gridsearch if it is necessary).

        Keyword arguments:
        models_to_train -- list of RecommendationModel instances that should be trained and evaluated
        train_on_all_data -- True if the models should be trained on ALL data once the cross-val is done, False
                             otherwise (default: False)
        training_set_params -- dict specifying the data's properties (e.g. should it be balanced, reduced, etc.) (default: None)
        save_results -- True if the results should be saved to disk, False otherwise (default: True)
        save_if_best -- True if the model should be saved if it is the best performing one, false otherwise (default: False)

        Return:
        A TrainResults' instance containing the training results
        """
        training_set_params = self.training_set.get_default_properties() if training_set_params is None else training_set_params

        train_results = TrainResults(models_to_train, self.training_set.get_labeler_properties()['type'])
        try:
            for split_id, yielded in enumerate(self.training_set.yield_splitted_train_val(training_set_params,
                                                                                ModelsTrainer.CONF['NB_CV_SPLITS'])):
                print('\nCross-validation split n°%i' % (split_id+1))
                all_data, all_labels, labels_set, X_train, y_train, X_val, y_val = yielded

                print('X_train shape:', X_train.shape, ', X_val shape:', X_val.shape) # TODO tmp print
                print(np.asarray(np.unique(y_train, return_counts=True)).T) # TODO tmp print
                print(np.asarray(np.unique(y_val, return_counts=True)).T) # TODO tmp print

                for model in models_to_train:

                    # training
                    print('Training %s.' % model)
                    with Utils.catchtime('Training model %s @ split %i' % (model, split_id)):
                        try:
                            scores, cm = model.train_and_eval(X_train, y_train, X_val, y_val,
                                                            all_data.columns, self.training_set.get_labeler_properties(), labels_set,
                                                            plot_cm=True, save_if_best=save_if_best)
                        except Exception as e:
                            print('Encountered exception while training %s. \n %s' % (model, e))
                            scores, cm = None, None

                    # save results
                    # TrainResults contain a dict grouping the results of each trained model
                    train_results.add_model_cv_split_result(split_id, model, scores, cm)

            if train_on_all_data: # train the models on all data:
                print('\nTraining models on all data.')
                # Warning this model should not be used for any kind of evaluation but only for production use
                _, X_all, y_all = self.training_set.get_all_data(training_set_params)
                for model in models_to_train:
                    with Utils.catchtime('Training model %s on all data' % model):
                        try:
                            model.trained_pipeline_prod = clone(model.pipe).fit(X_all, y_all)
                        except Exception as e:
                            print('Encountered exception while training %s on all data. \n %s' % (model, e))

        finally:
            # save results to disk
            if save_results:
                train_results.save(self.training_set, save_train_set=ModelsTrainer.CONF['SAVE_TRAIN_SET'])

        return train_results

    # static methods

    def _compute_selection_score(f1, r3, t, alpha, beta, gamma):
        """
        Method that computes the score based on which a model will be evaluated during the selection process.

        Keyword arguments:
        f1 -- F1-score of a model measured on a test set
        r3 -- recall@3 of a model measured on a test set
        t -- model's training time
        alpha -- parameter value to define the impact of the F1-score in the score
        beta -- parameter value to define the impact of the Recall@3 in the score
        gamma -- parameter value to define the impact of the model's training time in the score

        Return:
        Returns the score of a model used during the selection process.
        """
        return ((alpha*f1) + (beta*r3) - (gamma*t) + gamma) / (alpha+beta+gamma)


import os
import json
from torch.utils.data import Dataset
import numpy as np
import torch
from datasets.argoverse_v1_dataset import process_argoverse2
import sys
# sys.path.append("/ssd/taomingjin/venv/HiVT2")
num={}
class ShangqiDataset(Dataset):
    def __init__(self, list_path):
        with open(list_path, "r") as file:
            self.dir_path = file.readlines()

    def __getitem__(self, index):
        dir_path = self.dir_path[index].strip("\n")
        ego_path = os.path.join(dir_path, 'ego.csv')
        obj_path = os.path.join(dir_path, 'obj.csv')
        # temporal_data = process_argoverse2(dir_path,[ego_path, obj_path])
        # torch.save(temporal_data,dir_path+".pt")
        bagInfo_path = os.path.join(dir_path, 'bagInfo.json')  # 先打开bagInfo文件，提取出开始时间，持续时间
        with open(bagInfo_path) as f:
            json_data=json.load(f)
            start=json_data['start']
            duration1=json_data['duration']
        with open('/ssd/taomingjin/venv/HiVT2/scene_tag.json') as json_file:
            scene_tag = json.load(json_file)
        scene_data=[]
        for file_name in os.listdir(dir_path):  # 遍历文件夹下的所有文件
            if file_name.endswith('scenario_records.json'):
                # 如果文件是scenario_JSON文件
                # count+=1
                file_path = os.path.join(dir_path, file_name)  # 获取文件的完整路径
                with open(file_path, 'r') as f:  # 打开文件
                    json_data = json.load(f)  # 解析JSON数据
                    # 处理JSON数据，例如打印数据
                    # print(json_data)
                    scene=json_data['scenario'][0]['scene_tag']
                    scene_start=max(json_data['scenario'][0]['t0']-start,0)
                    scene_end=json_data['scenario'][0]['tN']-start

                    scene_data.append([0,scene_tag[scene],scene_start,scene_end])
                    if scene_end<0:
                        return None,None
        scene_data=np.stack(scene_data,axis=0)
        return torch.load(dir_path+".pt"),scene_data

    def __len__(self):
        return len(self.dir_path)

def collate_fn(batch):
    batch= [data for data in batch if data[0] is not None]
    data,label=list(zip(*batch))
    for i, box in enumerate(label):
        box[:, 0] = i
    boxes=np.concatenate(label,axis=0)
    target_box=torch.from_numpy(boxes)
    return data,target_box

# if __name__ == '__main__':
#     shangqidata = ShangqiDataset("/ssd/share/shangqi_data/last_dir_path.txt")
#     model=torch.load("/ssd/share/shangqi_data/PL069/20221101/20221101-101739hedian.pt")
#     for i in range(len(shangqidata)):
#         data=shangqidata[i]
#     res={}


#Modo 1 de fazer utilizando Numpy e Array
import numpy as np
n = [int(input('Digite um valor para a posição 0: ')),
     int(input('Digite um valor para a posição 1: ')),
     int(input('Digite um valor para a posição 2: ')),
     int(input('Digite um valor para a posição 3: ')),
     int(input('Digite um valor para a posição 4: '))]
n = np.array(n)
print(f'Você digitou os valores: {n}, ')
max = max(n)
min = min(n)
resultmax = np.where(n == max)
resultmin = np.where(n == min)
print(f'O maior valor digitado foi {max} nas posições: {resultmax[0]}.')
print(f'O menor valor digitado foi {min} nas posições: {resultmin[0]}.')

print('\n\n')

#Modo 2 de fazer utilizando apenas lista
listanum = []
mai = men = 0
for c in range(0, 5):
     listanum.append(int(input(f'Digite um valor para a posição {c}: ')))
     if c == 0:
          mai = men = listanum[c]
     else:
          if listanum[c] > mai:
               mai = listanum[c]
          if listanum[c] < men:
               men = listanum[c]
print(f'Você digitou os valores {listanum}')
print(f'O maior valor digitado foi {mai} nas posições: ', end='')
for i, v in enumerate(listanum):
     if v == mai:
          print(f'{i}, ', end='')
print()
print(f'O menor valor digitado foi {men} nas posições ', end='')
for i, v in enumerate(listanum):
     if v == men:
          print(f'{i}, ', end='')
print()


# Definition of ListNode
class ListNode(object):

    def __init__(self, val, next=None):
        self.val = val
        self.next = next

class Solution:
    """
    @param: head: The head of linked list.
    @param: val: An integer.
    @return: The head of new linked list.
    """
    def insertNode(self, head, val):
        node = ListNode(val)
        dummy = ListNode(0)
        dummy.next = head
        head = dummy

        while head.next != None and head.next.val <= val:
            head = head.next
        node.next = head.next
        head.next = node

        return dummy.next


#Code by Arturo Beristain López

import numpy as np
import matplotlib.pyplot as plt
import cv2

capture = cv2.VideoCapture(0,cv2.CAP_DSHOW)

# Initialize plot.
fig, ax = plt.subplots()
plt.ion()
plt.show()

color = ('b', 'g', 'r')

while True:
    (grabbed, frame) = capture.read()

    if not grabbed:
        break

    cv2.imshow('RGB', frame)
    ax.cla()        #clear last plot
    for i, col in enumerate(color):
        histr = cv2.calcHist([frame], [i], None, [256], [0, 256])

        ax.plot(histr, color=col, alpha=0.5, lw = 2)
        plt.xlim([0, 256])
        plt.title('RGB Histogram')
        ax.set_xlabel('Intensity')
        ax.set_ylabel('Frequency')

    fig.canvas.draw()

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

capture.release()
cv2.destroyAllWindows()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Dec 15 10:15:41 2018

@author: arthurmaroquenefroissart
"""

#2. Using Github’s API, create a function that:
#• takes all repositories from your account
#• prints a short description of each repository, with its name, number
#of stars, main language, and url.

#%%
import requests

def github_api_repository(user):
    url="https://api.github.com/users/{}/repos".format(user)

    repositories = requests.get(url).json()

    for repository in repositories:

        print("The name of the Github's repository is the following :{} \n The main_language is:{} \n The number_of_stars is:{} \n Here is the url:{} \n".format(
    repository["name"],repository["language"],str(repository["stargazers_count"]),repository["url"]))


from __future__ import print_function

import os
import re
import six

from sage.repl.load import load_wrap

implicit_mul_level = False
numeric_literal_prefix = '_sage_const_'

def implicit_multiplication(level=None):
    """
    Turns implicit multiplication on or off, optionally setting a
    specific ``level``.  Returns the current ``level`` if no argument
    is given.

    INPUT:

    - ``level`` - an integer (default: None); see :func:`implicit_mul`
      for a list

    EXAMPLES::

      sage: implicit_multiplication(True)
      sage: implicit_multiplication()
      5
      sage: preparse('2x')
      'Integer(2)*x'
      sage: implicit_multiplication(False)
      sage: preparse('2x')
      '2x'
    """
    global implicit_mul_level
    if level is None:
        return implicit_mul_level
    elif level is True:
        implicit_mul_level = 5
    else:
        implicit_mul_level = level

def isalphadigit_(s):
    """
    Return ``True`` if ``s`` is a non-empty string of alphabetic characters
    or a non-empty string of digits or just a single ``_``

    EXAMPLES::

        sage: from sage.repl.preparse import isalphadigit_
        sage: isalphadigit_('abc')
        True
        sage: isalphadigit_('123')
        True
        sage: isalphadigit_('_')
        True
        sage: isalphadigit_('a123')
        False
    """
    return s.isalpha() or s.isdigit() or s == "_"

keywords = """
and       del       from      not       while
as        elif      global    or        with
assert    else      if        pass      yield
break     except    import    print
class     exec      in        raise
continue  finally   is        return
def       for       lambda    try
""".split()

in_single_quote = False
in_double_quote = False
in_triple_quote = False

def in_quote():
    return in_single_quote or in_double_quote or in_triple_quote

def strip_string_literals(code, state=None):
    r"""
    Returns a string with all literal quotes replaced with labels and
    a dictionary of labels for re-substitution.  This makes parsing
    easier.

    INPUT:

    - ``code`` - a string; the input

    - ``state`` - a 2-tuple (default: None); state with which to
      continue processing, e.g., across multiple calls to this
      function

    OUTPUT:

    - a 3-tuple of the processed code, the dictionary of labels, and
      any accumulated state

    EXAMPLES::

        sage: from sage.repl.preparse import strip_string_literals
        sage: s, literals, state = strip_string_literals(r'''['a', "b", 'c', "d\""]''')
        sage: s
        '[%(L1)s, %(L2)s, %(L3)s, %(L4)s]'
        sage: literals
        {'L1': "'a'", 'L2': '"b"', 'L3': "'c'", 'L4': '"d\\""'}
        sage: print(s % literals)
        ['a', "b", 'c', "d\""]
        sage: print(strip_string_literals(r'-"\\\""-"\\"-')[0])
        -%(L1)s-%(L2)s-

    Triple-quotes are handled as well::

        sage: s, literals, state = strip_string_literals("[a, '''b''', c, '']")
        sage: s
        '[a, %(L1)s, c, %(L2)s]'
        sage: print(s % literals)
        [a, '''b''', c, '']

    Comments are substitute too::

        sage: s, literals, state = strip_string_literals("code '#' # ccc 't'"); s
        'code %(L1)s #%(L2)s'
        sage: s % literals
        "code '#' # ccc 't'"

    A state is returned so one can break strings across multiple calls to
    this function::

        sage: s, literals, state = strip_string_literals('s = "some'); s
        's = %(L1)s'
        sage: s, literals, state = strip_string_literals('thing" * 5', state); s
        '%(L1)s * 5'

    TESTS:

    Even for raw strings, a backslash can escape a following quote::

        sage: s, literals, state = strip_string_literals(r"r'somethin\' funny'"); s
        'r%(L1)s'
        sage: dep_regex = r'^ *(?:(?:cimport +([\w\. ,]+))|(?:from +(\w+) +cimport)|(?:include *[\'"]([^\'"]+)[\'"])|(?:cdef *extern *from *[\'"]([^\'"]+)[\'"]))' # Ticket 5821
    """
    new_code = []
    literals = {}
    counter = 0
    start = q = 0
    if state is None:
        in_quote = False
        raw = False
    else:
        in_quote, raw = state
    while True:
        sig_q = code.find("'", q)
        dbl_q = code.find('"', q)
        hash_q = code.find('#', q)
        q = min(sig_q, dbl_q)
        if q == -1: q = max(sig_q, dbl_q)
        if not in_quote and hash_q != -1 and (q == -1 or hash_q < q):
            # it's a comment
            newline = code.find('\n', hash_q)
            if newline == -1: newline = len(code)
            counter += 1
            label = "L%s" % counter
            literals[label] = code[hash_q+1:newline]
            new_code.append(code[start:hash_q].replace('%','%%'))
            new_code.append("#%%(%s)s" % label)
            start = q = newline
        elif q == -1:
            if in_quote:
                counter += 1
                label = "L%s" % counter
                literals[label] = code[start:]
                new_code.append("%%(%s)s" % label)
            else:
                new_code.append(code[start:].replace('%','%%'))
            break
        elif in_quote:
            if code[q-1] == '\\':
                k = 2
                while code[q-k] == '\\':
                    k += 1
                if k % 2 == 0:
                    q += 1
            if code[q:q+len(in_quote)] == in_quote:
                counter += 1
                label = "L%s" % counter
                literals[label] = code[start:q+len(in_quote)]
                new_code.append("%%(%s)s" % label)
                q += len(in_quote)
                start = q
                in_quote = False
            else:
                q += 1
        else:
            raw = q>0 and code[q-1] in 'rR'
            if len(code) >= q+3 and (code[q+1] == code[q] == code[q+2]):
                in_quote = code[q]*3
            else:
                in_quote = code[q]
            new_code.append(code[start:q].replace('%', '%%'))
            start = q
            q += len(in_quote)

    return "".join(new_code), literals, (in_quote, raw)

def containing_block(code, idx, delimiters=['()','[]','{}'], require_delim=True):
    """
    Find the code block given by balanced delimiters that contains the position ``idx``.

    INPUT:

    - ``code`` - a string

    - ``idx`` - an integer; a starting position

    - ``delimiters`` - a list of strings (default: ['()', '[]',
      '{}']); the delimiters to balance. A delimiter must be a single
      character and no character can at the same time be opening and
      closing delimiter.

    - ``require_delim`` - a boolean (default: True); whether to raise
      a SyntaxError if delimiters are present. If the delimiters are
      unbalanced, an error will be raised in any case.

    OUTPUT:

    - a 2-tuple ``(a,b)`` of integers, such that ``code[a:b]`` is
      delimited by balanced delimiters, ``a<=idx<b``, and ``a``
      is maximal and ``b`` is minimal with that property. If that
      does not exist, a ``SyntaxError`` is raised.

    - If ``require_delim`` is false and ``a,b`` as above can not be
      found, then ``0, len(code)`` is returned.

    EXAMPLES::

        sage: from sage.repl.preparse import containing_block
        sage: s = "factor(next_prime(L[5]+1))"
        sage: s[22]
        '+'
        sage: start, end = containing_block(s, 22)
        sage: start, end
        (17, 25)
        sage: s[start:end]
        '(L[5]+1)'
        sage: s[20]
        '5'
        sage: start, end = containing_block(s, 20); s[start:end]
        '[5]'
        sage: start, end = containing_block(s, 20, delimiters=['()']); s[start:end]
        '(L[5]+1)'
        sage: start, end = containing_block(s, 10); s[start:end]
        '(next_prime(L[5]+1))'

    TESTS::

        sage: containing_block('((a{))',0)
        Traceback (most recent call last):
        ...
        SyntaxError: Unbalanced delimiters
        sage: containing_block('((a{))',1)
        Traceback (most recent call last):
        ...
        SyntaxError: Unbalanced delimiters
        sage: containing_block('((a{))',2)
        Traceback (most recent call last):
        ...
        SyntaxError: Unbalanced delimiters
        sage: containing_block('((a{))',3)
        Traceback (most recent call last):
        ...
        SyntaxError: Unbalanced delimiters
        sage: containing_block('((a{))',4)
        Traceback (most recent call last):
        ...
        SyntaxError: Unbalanced delimiters
        sage: containing_block('((a{))',5)
        Traceback (most recent call last):
        ...
        SyntaxError: Unbalanced delimiters
        sage: containing_block('(()()',1)
        (1, 3)
        sage: containing_block('(()()',3)
        (3, 5)
        sage: containing_block('(()()',4)
        (3, 5)
        sage: containing_block('(()()',0)
        Traceback (most recent call last):
        ...
        SyntaxError: Unbalanced delimiters
        sage: containing_block('(()()',0, require_delim=False)
        (0, 5)
        sage: containing_block('((})()',1, require_delim=False)
        (0, 6)
        sage: containing_block('abc',1, require_delim=False)
        (0, 3)

    """
    openings = "".join([d[0] for d in delimiters])
    closings = "".join([d[-1] for d in delimiters])
    levels = [0] * len(openings)
    p = 0
    start = idx
    while start >= 0:
        if code[start] in openings:
            p = openings.index(code[start])
            levels[p] -= 1
            if levels[p] == -1:
                break
        elif code[start] in closings and start < idx:
            p = closings.index(code[start])
            levels[p] += 1
        start -= 1
    if start == -1:
        if require_delim:
            raise SyntaxError("Unbalanced or missing delimiters")
        else:
            return 0, len(code)
    if levels.count(0) != len(levels)-1:
        if require_delim:
            raise SyntaxError("Unbalanced delimiters")
        else:
            return 0, len(code)
    p0 = p
    # We now have levels[p0]==-1. We go to the right hand side
    # till we find a closing delimiter of type p0 that makes
    # levels[p0]==0.
    end = idx
    while end < len(code):
        if code[end] in closings:
            p = closings.index(code[end])
            levels[p] += 1
            if p==p0 and levels[p] == 0:
                break
        elif code[end] in openings and end > idx:
            p = openings.index(code[end])
            levels[p] -= 1
        end += 1
    if levels.count(0) != len(levels):
        # This also occurs when end==len(code) without finding a closing delimiter
        if require_delim:
            raise SyntaxError("Unbalanced delimiters")
        else:
            return 0, len(code)
    return start, end+1

def parse_ellipsis(code, preparse_step=True):
    """
    Preparses [0,2,..,n] notation.

    INPUT:

    - ``code`` - a string

    - ``preparse_step`` - a boolean (default: True)

    OUTPUT:

    - a string

    EXAMPLES::

        sage: from sage.repl.preparse import parse_ellipsis
        sage: parse_ellipsis("[1,2,..,n]")
        '(ellipsis_range(1,2,Ellipsis,n))'
        sage: parse_ellipsis("for i in (f(x) .. L[10]):")
        'for i in (ellipsis_iter(f(x) ,Ellipsis, L[10])):'
        sage: [1.0..2.0]
        [1.00000000000000, 2.00000000000000]

    TESTS:

    Check that nested ellipsis is processed correctly (:trac:`17378`)::

        sage: preparse('[1,..,2,..,len([1..3])]')
        '(ellipsis_range(Integer(1),Ellipsis,Integer(2),Ellipsis,len((ellipsis_range(Integer(1),Ellipsis,Integer(3))))))'

    """
    ix = code.find('..')
    while ix != -1:
        if ix == 0:
            raise SyntaxError("Cannot start line with ellipsis.")
        elif code[ix-1]=='.':
            # '...' be valid Python in index slices
            code = code[:ix-1] + "Ellipsis" + code[ix+2:]
        elif len(code) >= ix+3 and code[ix+2]=='.':
            # '...' be valid Python in index slices
            code = code[:ix] + "Ellipsis" + code[ix+3:]
        else:
            start_list, end_list = containing_block(code, ix, ['()','[]'])

            #search the current containing block for other '..' occurrences that may
            #be contained in proper subblocks. Those need to be processed before
            #we can deal with the present level of ellipses.
            ix = code.find('..',ix+2,end_list)
            while ix != -1:
                if code[ix-1]!='.' and code[ix+2]!='.':
                    start_list,end_list = containing_block(code,ix,['()','[]'])
                ix = code.find('..',ix+2,end_list)

            arguments = code[start_list+1:end_list-1].replace('...', ',Ellipsis,').replace('..', ',Ellipsis,')
            arguments = re.sub(r',\s*,', ',', arguments)
            if preparse_step:
                arguments = arguments.replace(';', ', step=')
            range_or_iter = 'range' if code[start_list]=='[' else 'iter'
            code = "%s(ellipsis_%s(%s))%s" %  (code[:start_list],
                                               range_or_iter,
                                               arguments,
                                               code[end_list:])
        ix = code.find('..')
    return code

def extract_numeric_literals(code):
    """
    Pulls out numeric literals and assigns them to global variables.
    This eliminates the need to re-parse and create the literals,
    e.g., during every iteration of a loop.

    INPUT:

    - ``code`` - a string; a block of code

    OUTPUT:

    - a (string, string:string dictionary) 2-tuple; the block with
      literals replaced by variable names and a mapping from names to
      the new variables

    EXAMPLES::

        sage: from sage.repl.preparse import extract_numeric_literals
        sage: code, nums = extract_numeric_literals("1.2 + 5")
        sage: print(code)
        _sage_const_1p2  + _sage_const_5
        sage: print(nums)
        {'_sage_const_1p2': "RealNumber('1.2')", '_sage_const_5': 'Integer(5)'}

        sage: extract_numeric_literals("[1, 1.1, 1e1, -1e-1, 1.]")[0]
        '[_sage_const_1 , _sage_const_1p1 , _sage_const_1e1 , -_sage_const_1en1 , _sage_const_1p ]'

        sage: extract_numeric_literals("[1.sqrt(), 1.2.sqrt(), 1r, 1.2r, R.1, R0.1, (1..5)]")[0]
        '[_sage_const_1 .sqrt(), _sage_const_1p2 .sqrt(), 1 , 1.2 , R.1, R0.1, (_sage_const_1 .._sage_const_5 )]'
    """
    return preparse_numeric_literals(code, True)

all_num_regex = None

def preparse_numeric_literals(code, extract=False):
    """
    This preparses numerical literals into their Sage counterparts,
    e.g. Integer, RealNumber, and ComplexNumber.

    INPUT:

    - ``code`` - a string; a code block to preparse

    - ``extract`` - a boolean (default: False); whether to create
      names for the literals and return a dictionary of
      name-construction pairs

    OUTPUT:

    - a string or (string, string:string dictionary) 2-tuple; the
      preparsed block and, if ``extract`` is True, the
      name-construction mapping

    EXAMPLES::

        sage: from sage.repl.preparse import preparse_numeric_literals
        sage: preparse_numeric_literals("5")
        'Integer(5)'
        sage: preparse_numeric_literals("5j")
        "ComplexNumber(0, '5')"
        sage: preparse_numeric_literals("5jr")
        '5J'
        sage: preparse_numeric_literals("5l")
        '5l'
        sage: preparse_numeric_literals("5L")
        '5L'
        sage: preparse_numeric_literals("1.5")
        "RealNumber('1.5')"
        sage: preparse_numeric_literals("1.5j")
        "ComplexNumber(0, '1.5')"
        sage: preparse_numeric_literals(".5j")
        "ComplexNumber(0, '.5')"
        sage: preparse_numeric_literals("5e9j")
        "ComplexNumber(0, '5e9')"
        sage: preparse_numeric_literals("5.")
        "RealNumber('5.')"
        sage: preparse_numeric_literals("5.j")
        "ComplexNumber(0, '5.')"
        sage: preparse_numeric_literals("5.foo()")
        'Integer(5).foo()'
        sage: preparse_numeric_literals("5.5.foo()")
        "RealNumber('5.5').foo()"
        sage: preparse_numeric_literals("5.5j.foo()")
        "ComplexNumber(0, '5.5').foo()"
        sage: preparse_numeric_literals("5j.foo()")
        "ComplexNumber(0, '5').foo()"
        sage: preparse_numeric_literals("1.exp()")
        'Integer(1).exp()'
        sage: preparse_numeric_literals("1e+10")
        "RealNumber('1e+10')"
        sage: preparse_numeric_literals("0x0af")
        'Integer(0x0af)'
        sage: preparse_numeric_literals("0x10.sqrt()")
        'Integer(0x10).sqrt()'
        sage: preparse_numeric_literals('0o100')
        'Integer(0o100)'
        sage: preparse_numeric_literals('0b111001')
        'Integer(0b111001)'
        sage: preparse_numeric_literals('0xe')
        'Integer(0xe)'
        sage: preparse_numeric_literals('0xEAR')
        '0xEA'
        sage: preparse_numeric_literals('0x1012Fae')
        'Integer(0x1012Fae)'
    """
    literals = {}
    last = 0
    new_code = []

    global all_num_regex
    if all_num_regex is None:
        dec_num = r"\b\d+"
        hex_num = r"\b0x[0-9a-f]+"
        oct_num = r"\b0o[0-7]+"
        bin_num = r"\b0b[01]+"
        # This is slightly annoying as floating point numbers may start
        # with a decimal point, but if they do the \b will not match.
        float_num = r"((\b\d+([.]\d*)?)|([.]\d+))(e[-+]?\d+)?"
        all_num = r"((%s)|(%s)|(%s)|(%s)|(%s))(rj|rL|jr|Lr|j|L|r|)\b" % (hex_num, oct_num, bin_num, float_num, dec_num)
        all_num_regex = re.compile(all_num, re.I)

    for m in all_num_regex.finditer(code):
        start, end = m.start(), m.end()
        num = m.group(1)
        postfix = m.groups()[-1].upper()

        if 'R' in postfix:
            if not six.PY2:
                postfix = postfix.replace('L', '')
            num_name = num_make = num + postfix.replace('R', '')
        elif 'L' in postfix:
            if six.PY2:
                continue
            else:
                num_name = num_make = num + postfix.replace('L', '')
        else:

            # The Sage preparser does extra things with numbers, which we need to handle here.
            if '.' in num:
                if start > 0 and num[0] == '.':
                    if code[start-1] == '.':
                        # handle Ellipsis
                        start += 1
                        num = num[1:]
                    elif re.match(r'[a-zA-Z0-9_\])]', code[start-1]):
                        # handle R.0
                        continue
                elif end < len(code) and num[-1] == '.':
                    if re.match('[a-zA-Z_]', code[end]):
                        # handle 4.sqrt()
                        end -= 1
                        num = num[:-1]
            elif end < len(code) and code[end] == '.' and not postfix and re.match(r'\d+$', num):
                # \b does not match after the . for floating point
                # two dots in a row would be an ellipsis
                if end+1 == len(code) or code[end+1] != '.':
                    end += 1
                    num += '.'

            num_name = numeric_literal_prefix + num.replace('.', 'p').replace('-', 'n').replace('+', '')

            if 'J' in postfix:
                num_make = "ComplexNumber(0, '%s')" % num
                num_name += 'j'
            elif len(num) < 2 or num[1] in 'oObBxX':
                num_make = "Integer(%s)" % num
            elif '.' in num or 'e' in num or 'E' in num:
                num_make = "RealNumber('%s')" % num
            elif num[0] == "0":
                num_make = "Integer('%s')" % num
            else:
                num_make = "Integer(%s)" % num

            literals[num_name] = num_make

        new_code.append(code[last:start])
        if extract:
            new_code.append(num_name+' ')
        else:
            new_code.append(num_make)
        last = end

    new_code.append(code[last:])
    code = ''.join(new_code)
    if extract:
        return code, literals
    else:
        return code

def strip_prompts(line):
    r"""
    Removes leading sage: and >>> prompts so that pasting of examples
    from the documentation works.

    INPUT:

    - ``line`` - a string to process

    OUTPUT:

    - a string stripped of leading prompts

    EXAMPLES::

        sage: from sage.repl.preparse import strip_prompts
        sage: strip_prompts("sage: 2 + 2")
        '2 + 2'
        sage: strip_prompts(">>>   3 + 2")
        '3 + 2'
        sage: strip_prompts("  2 + 4")
        '  2 + 4'
    """
    for prompt in ['sage:', '>>>']:
        if line.startswith(prompt):
            line = line[len(prompt):].lstrip()
            break
    return line

def preparse_calculus(code):
    """
    Supports calculus-like function assignment, e.g., transforms::

       f(x,y,z) = sin(x^3 - 4*y) + y^x

    into::

       __tmp__=var("x,y,z")
       f = symbolic_expression(sin(x**3 - 4*y) + y**x).function(x,y,z)

    AUTHORS:

    - Bobby Moretti

      - Initial version - 02/2007

    - William Stein

      - Make variables become defined if they aren't already defined.

    - Robert Bradshaw

      - Rewrite using regular expressions (01/2009)

    EXAMPLES::

        sage: preparse("f(x) = x^3-x")
        '__tmp__=var("x"); f = symbolic_expression(x**Integer(3)-x).function(x)'
        sage: preparse("f(u,v) = u - v")
        '__tmp__=var("u,v"); f = symbolic_expression(u - v).function(u,v)'
        sage: preparse("f(x) =-5")
        '__tmp__=var("x"); f = symbolic_expression(-Integer(5)).function(x)'
        sage: preparse("f(x) -= 5")
        'f(x) -= Integer(5)'
        sage: preparse("f(x_1, x_2) = x_1^2 - x_2^2")
        '__tmp__=var("x_1,x_2"); f = symbolic_expression(x_1**Integer(2) - x_2**Integer(2)).function(x_1,x_2)'

    For simplicity, this function assumes all statements begin and end
    with a semicolon::

        sage: from sage.repl.preparse import preparse_calculus
        sage: preparse_calculus(";f(t,s)=t^2;")
        ';__tmp__=var("t,s"); f = symbolic_expression(t^2).function(t,s);'
        sage: preparse_calculus(";f( t , s ) = t^2;")
        ';__tmp__=var("t,s"); f = symbolic_expression(t^2).function(t,s);'

    TESTS:

    The arguments in the definition must be symbolic variables (:trac:`10747`)::

        sage: preparse_calculus(";f(_sage_const_)=x;")
        Traceback (most recent call last):
        ...
        ValueError: Argument names should be valid python identifiers.

    Although preparse_calculus returns something for f(1)=x, when
    preparsing a file an exception is raised because it is invalid python::

        sage: preparse_calculus(";f(1)=x;")
        ';__tmp__=var("1"); f = symbolic_expression(x).function(1);'

        sage: from sage.repl.preparse import preparse_file
        sage: preparse_file("f(1)=x")
        Traceback (most recent call last):
        ...
        ValueError: Argument names should be valid python identifiers.

        sage: from sage.repl.preparse import preparse_file
        sage: preparse_file("f(x,1)=2")
        Traceback (most recent call last):
        ...
        ValueError: Argument names should be valid python identifiers.
    """
    new_code = []
    last_end = 0
    #                                   f         (  vars  )   =      expr
    for m in re.finditer(r";(\s*)([a-zA-Z_]\w*) *\(([^()]+)\) *= *([^;#=][^;#]*)", code):
        ident, func, vars, expr = m.groups()
        stripped_vars = [v.strip() for v in vars.split(',')]
        # if the variable name starts with numeric_literal_prefix
        # the argument name for the symbolic expression is a numeric literal
        # such as f(2)=5
        if any(n.startswith(numeric_literal_prefix) for n in stripped_vars):
            raise ValueError("Argument names should be valid python identifiers.")
        vars = ','.join(stripped_vars)

        new_code.append(code[last_end:m.start()])
        new_code.append(';%s__tmp__=var("%s"); %s = symbolic_expression(%s).function(%s)' %
                        (ident, vars, func, expr, vars))
        last_end = m.end()

    if last_end == 0:
        return code
    else:
        new_code.append(code[m.end():])
        return ''.join(new_code)

def preparse_generators(code):
    r"""
    Parses generator syntax, converting::

        obj.<gen0,gen1,...,genN> = objConstructor(...)

    into::

        obj = objConstructor(..., names=("gen0", "gen1", ..., "genN"))
        (gen0, gen1, ..., genN,) = obj.gens()

    and::

        obj.<gen0,gen1,...,genN> = R[interior]

    into::

        obj = R[interior]; (gen0, gen1, ..., genN,) = obj.gens()

    INPUT:

    - ``code`` - a string

    OUTPUT:

    - a string

    LIMITATIONS:

       - The entire constructor *must* be on one line.

    AUTHORS:

    - 2006-04-14: Joe Wetherell (jlwether@alum.mit.edu)

      - Initial version.

    - 2006-04-17: William Stein

      - Improvements to allow multiple statements.

    - 2006-05-01: William

      - Fix bug that Joe found.

    - 2006-10-31: William

      - Fix so obj doesn't have to be mutated.

    - 2009-01-27: Robert Bradshaw

      - Rewrite using regular expressions

    TESTS::

        sage: from sage.repl.preparse import preparse, preparse_generators

    Vanilla::

        sage: preparse("R.<x> = ZZ['x']")
        "R = ZZ['x']; (x,) = R._first_ngens(1)"
        sage: preparse("R.<x,y> = ZZ['x,y']")
        "R = ZZ['x,y']; (x, y,) = R._first_ngens(2)"

    No square brackets::

        sage: preparse("R.<x> = PolynomialRing(ZZ, 'x')")
        "R = PolynomialRing(ZZ, 'x', names=('x',)); (x,) = R._first_ngens(1)"
        sage: preparse("R.<x,y> = PolynomialRing(ZZ, 'x,y')")
        "R = PolynomialRing(ZZ, 'x,y', names=('x', 'y',)); (x, y,) = R._first_ngens(2)"

    Names filled in::

        sage: preparse("R.<x> = ZZ[]")
        "R = ZZ['x']; (x,) = R._first_ngens(1)"
        sage: preparse("R.<x,y> = ZZ[]")
        "R = ZZ['x, y']; (x, y,) = R._first_ngens(2)"

    Names given not the same as generator names::

        sage: preparse("R.<x> = ZZ['y']")
        "R = ZZ['y']; (x,) = R._first_ngens(1)"
        sage: preparse("R.<x,y> = ZZ['u,v']")
        "R = ZZ['u,v']; (x, y,) = R._first_ngens(2)"

    Number fields::

        sage: preparse("K.<a> = QQ[2^(1/3)]")
        'K = QQ[Integer(2)**(Integer(1)/Integer(3))]; (a,) = K._first_ngens(1)'
        sage: preparse("K.<a, b> = QQ[2^(1/3), 2^(1/2)]")
        'K = QQ[Integer(2)**(Integer(1)/Integer(3)), Integer(2)**(Integer(1)/Integer(2))]; (a, b,) = K._first_ngens(2)'

    Just the .<> notation::

        sage: preparse("R.<x> = ZZx")
        'R = ZZx; (x,) = R._first_ngens(1)'
        sage: preparse("R.<x, y> = a+b")
        'R = a+b; (x, y,) = R._first_ngens(2)'
        sage: preparse("A.<x,y,z>=FreeAlgebra(ZZ,3)")
        "A = FreeAlgebra(ZZ,Integer(3), names=('x', 'y', 'z',)); (x, y, z,) = A._first_ngens(3)"

    Ensure we don't eat too much::

        sage: preparse("R.<x, y> = ZZ;2")
        'R = ZZ; (x, y,) = R._first_ngens(2);Integer(2)'
        sage: preparse("R.<x, y> = ZZ['x,y'];2")
        "R = ZZ['x,y']; (x, y,) = R._first_ngens(2);Integer(2)"
        sage: preparse("F.<b>, f, g = S.field_extension()")
        "F, f, g  = S.field_extension(names=('b',)); (b,) = F._first_ngens(1)"

    For simplicity, this function assumes all statements begin and end
    with a semicolon::

        sage: preparse_generators(";  R.<x>=ZZ[];")
        ";  R = ZZ['x']; (x,) = R._first_ngens(1);"

    See :trac:`16731` ::

        sage: preparse_generators('R.<x> = ')
        'R.<x> = '
    """
    new_code = []
    last_end = 0
    #                                  obj       .< gens >      ,  other   =   constructor
    for m in re.finditer(r";(\s*)([a-zA-Z_]\w*)\.<([^>]+)> *((?:,[\w, ]+)?)= *([^;#]+)", code):
        ident, obj, gens, other_objs, constructor = m.groups()
        gens = [v.strip() for v in gens.split(',')]
        constructor = constructor.rstrip()
        if len(constructor) == 0:
            pass   # SyntaxError will be raised by Python later
        elif constructor[-1] == ')':
            if '(' not in constructor:
                raise SyntaxError("Mismatched ')'")
            opening = constructor.rindex('(')
            # Only use comma if there are already arguments to the constructor
            comma = ', ' if constructor[opening+1:-1].strip() != '' else ''
            names = "('%s',)" % "', '".join(gens)
            constructor = constructor[:-1] + comma + "names=%s)" % names
        elif constructor[-1] == ']':
            # Could be nested.
            if '[' not in constructor:
                raise SyntaxError("Mismatched ']'")
            opening = constructor.rindex('[')
            closing = constructor.index(']', opening)
            if constructor[opening+1:closing].strip() == '':
                names = "'" + ', '.join(gens) + "'"
                constructor = constructor[:opening+1] + names + constructor[closing:]
        else:
            pass
        gens_tuple = "(%s,)" % ', '.join(gens)
        new_code.append(code[last_end:m.start()])
        new_code.append(";%s%s%s = %s; %s = %s._first_ngens(%s)" %
                        (ident, obj, other_objs, constructor, gens_tuple, obj, len(gens)))
        last_end = m.end()

    if last_end == 0:
        return code
    else:
        new_code.append(code[m.end():])
        return ''.join(new_code)

quote_state = None

def preparse(line, reset=True, do_time=False, ignore_prompts=False,
             numeric_literals=True):
    r"""
    Preparses a line of input.

    INPUT:

    - ``line`` - a string

    - ``reset`` - a boolean (default: True)

    - ``do_time`` - a boolean (default: False)

    - ``ignore_prompts`` - a boolean (default: False)

    - ``numeric_literals`` - a boolean (default: True)

    OUTPUT:

    - a string

    EXAMPLES::

        sage: preparse("ZZ.<x> = ZZ['x']")
        "ZZ = ZZ['x']; (x,) = ZZ._first_ngens(1)"
        sage: preparse("ZZ.<x> = ZZ['y']")
        "ZZ = ZZ['y']; (x,) = ZZ._first_ngens(1)"
        sage: preparse("ZZ.<x,y> = ZZ[]")
        "ZZ = ZZ['x, y']; (x, y,) = ZZ._first_ngens(2)"
        sage: preparse("ZZ.<x,y> = ZZ['u,v']")
        "ZZ = ZZ['u,v']; (x, y,) = ZZ._first_ngens(2)"
        sage: preparse("ZZ.<x> = QQ[2^(1/3)]")
        'ZZ = QQ[Integer(2)**(Integer(1)/Integer(3))]; (x,) = ZZ._first_ngens(1)'
        sage: QQ[2^(1/3)]
        Number Field in a with defining polynomial x^3 - 2

        sage: preparse("a^b")
        'a**b'
        sage: preparse("a^^b")
        'a^b'
        sage: 8^1
        8
        sage: 8^^1
        9
        sage: 9^^1
        8

        sage: preparse("A \\ B")
        'A  * BackslashOperator() * B'
        sage: preparse("A^2 \\ B + C")
        'A**Integer(2)  * BackslashOperator() * B + C'
        sage: preparse("a \\ b \\") # There is really only one backslash here, it's just being escaped.
        'a  * BackslashOperator() * b \\'

        sage: preparse("time R.<x> = ZZ[]", do_time=True)
        '__time__=misc.cputime(); __wall__=misc.walltime(); R = ZZ[\'x\']; print("Time: CPU %.2f s, Wall: %.2f s"%(misc.cputime(__time__), misc.walltime(__wall__))); (x,) = R._first_ngens(1)'
    """
    global quote_state
    if reset:
        quote_state = None

    L = line.lstrip()
    if len(L) > 0 and L[0] in ['#', '!']:
        return line

    if L.startswith('...'):
        i = line.find('...')
        return line[:i+3] + preparse(line[i+3:], reset=reset, do_time=do_time, ignore_prompts=ignore_prompts)

    if ignore_prompts:
        # Get rid of leading sage: and >>> so that pasting of examples from
        # the documentation works.
        line = strip_prompts(line)

    # This part handles lines with semi-colons all at once
    # Then can also handle multiple lines more efficiently, but
    # that optimization can be done later.
    L, literals, quote_state = strip_string_literals(line, quote_state)

    # Ellipsis Range
    # [1..n]
    try:
        L = parse_ellipsis(L, preparse_step=False)
    except SyntaxError:
        pass

    if implicit_mul_level:
        # Implicit Multiplication
        # 2x -> 2*x
        L = implicit_mul(L, level = implicit_mul_level)

    if numeric_literals:
        # Wrapping
        # 1 + 0.5 -> Integer(1) + RealNumber('0.5')
        L = preparse_numeric_literals(L)

    # Generators
    # R.0 -> R.gen(0)
    L = re.sub(r'([_a-zA-Z]\w*|[)\]])\.(\d+)', r'\1.gen(\2)', L)

    # Use ^ for exponentiation and ^^ for xor
    # (A side effect is that **** becomes xor as well.)
    L = L.replace('^', '**').replace('****', '^')

    # Make it easy to match statement ends
    L = ';%s;' % L.replace('\n', ';\n;')

    if do_time:
        # Separate time statement
        L = re.sub(r';(\s*)time +(\w)', r';time;\1\2', L)

    # Construction with generators
    # R.<...> = obj()
    # R.<...> = R[]
    L = preparse_generators(L)

    # Calculus functions
    # f(x,y) = x^3 - sin(y)
    L = preparse_calculus(L)

    # Backslash
    L = re.sub(r'''\\\s*([^\t ;#])''', r' * BackslashOperator() * \1', L)

    if do_time:
        # Time keyword
        L = re.sub(r';time;(\s*)(\S[^;]*)',
                   r';\1__time__=misc.cputime(); __wall__=misc.walltime(); \2; print(' +
                        '"Time: CPU %%.2f s, Wall: %%.2f s"%%(misc.cputime(__time__), misc.walltime(__wall__)))',
                   L)

    # Remove extra ;'s
    L = L.replace(';\n;', '\n')[1:-1]

    line = L % literals

    return line

######################################################
## Apply the preparser to an entire file
######################################################

def preparse_file(contents, globals=None, numeric_literals=True):
    """
    Preparses input, attending to numeric literals and load/attach
    file directives.

    .. note:: Temporarily, if @parallel is in the input, then
       numeric_literals is always set to False.

    INPUT:

    - ``contents`` - a string

    - ``globals`` - dict or None (default: None); if given, then
      arguments to load/attach are evaluated in the namespace of this
      dict.

    - ``numeric_literals`` - bool (default: True), whether to factor
      out wrapping of integers and floats, so they don't get created
      repeatedly inside loops

    OUTPUT:

    - a string

    TESTS::

        sage: from sage.repl.preparse import preparse_file
        sage: lots_of_numbers = "[%s]" % ", ".join(str(i) for i in range(3000))
        sage: _ = preparse_file(lots_of_numbers)
        sage: print(preparse_file("type(100r), type(100)"))
        _sage_const_100 = Integer(100)
        type(100 ), type(_sage_const_100 )
    """
    if not isinstance(contents, six.string_types):
        raise TypeError("contents must be a string")

    if globals is None:
        globals = {}

    # This is a hack, since when we use @parallel to parallelize code,
    # the numeric literals that are factored out do not get copied
    # to the subprocesses properly.  See trac #4545.
    if '@parallel' in contents:
        numeric_literals = False

    if numeric_literals:
        contents, literals, state = strip_string_literals(contents)
        contents, nums = extract_numeric_literals(contents)
        contents = contents % literals
        if nums:
            # Stick the assignments at the top, trying not to shift
            # the lines down.
            ix = contents.find('\n')
            if ix == -1: ix = len(contents)
            if not re.match(r"^ *(#.*)?$", contents[:ix]):
                contents = "\n"+contents
            assignments = ["%s = %s" % x for x in nums.items()]
            # the preparser recurses on semicolons, so we only attempt
            # to preserve line numbers if there are a few
            if len(assignments) < 500:
                contents = "; ".join(assignments) + contents
            else:
                contents = "\n".join(assignments) + "\n\n" + contents

    # The list F contains the preparsed lines so far.
    F = []
    # A is the input, as a list of lines.
    A = contents.splitlines()
    # We are currently parsing the i-th input line.
    i = 0
    while i < len(A):
        L = A[i]
        do_preparse = True
        for cmd in ['load', 'attach']:
            if L.lstrip().startswith(cmd+' '):
                j = L.find(cmd+' ')
                s = L[j+len(cmd)+1:].strip()
                if not s.startswith('('):
                    F.append(' '*j + load_wrap(s, cmd=='attach'))
                    do_preparse = False
                    continue
        if do_preparse:
            F.append(preparse(L, reset=(i==0), do_time=True, ignore_prompts=False,
                              numeric_literals=not numeric_literals))
        i += 1

    return '\n'.join(F)

def implicit_mul(code, level=5):
    r"""
    Inserts \*'s to make implicit multiplication explicit.

    INPUT:

    - ``code``  -- a string; the code with missing \*'s

    - ``level`` -- an integer (default: 5); how aggressive to be in
      placing \*'s

      -  0 - Do nothing
      -  1 - Numeric followed by alphanumeric
      -  2 - Closing parentheses followed by alphanumeric
      -  3 - Spaces between alphanumeric
      - 10 - Adjacent parentheses (may mangle call statements)

    OUTPUT:

    - a string

    EXAMPLES::

        sage: from sage.repl.preparse import implicit_mul
        sage: implicit_mul('(2x^2-4x+3)a0')
        '(2*x^2-4*x+3)*a0'
        sage: implicit_mul('a b c in L')
        'a*b*c in L'
        sage: implicit_mul('1r + 1e3 + 5exp(2)')
        '1r + 1e3 + 5*exp(2)'
        sage: implicit_mul('f(a)(b)', level=10)
        'f(a)*(b)'
    """
    def re_no_keyword(pattern, code):
        for _ in range(2): # do it twice in because matches don't overlap
            for m in reversed(list(re.finditer(pattern, code))):
                left, right = m.groups()
                if left not in keywords and right not in keywords:
                    code = "%s%s*%s%s" % (code[:m.start()],
                                          left,
                                          right,
                                          code[m.end():])
        return code

    code, literals, state = strip_string_literals(code)
    if level >= 1:
        no_mul_token = " '''_no_mult_token_''' "
        code = re.sub(r'\b0x', r'0%sx' % no_mul_token, code)  # hex digits
        code = re.sub(r'( *)time ', r'\1time %s' % no_mul_token, code)  # first word may be magic 'time'
        code = re.sub(r'\b(\d+(?:\.\d+)?(?:e\d+)?)([rR]\b)', r'\1%s\2' % no_mul_token, code)  # exclude such things as 10r
        code = re.sub(r'\b(\d+(?:\.\d+)?)e([-\d])', r'\1%se%s\2' % (no_mul_token, no_mul_token), code)  # exclude such things as 1e5
        code = re_no_keyword(r'\b((?:\d+(?:\.\d+)?)|(?:%s[0-9eEpn]*\b)) *([a-zA-Z_(]\w*)\b' % numeric_literal_prefix, code)
    if level >= 2:
        code = re.sub(r'(\%\(L\d+\))s', r'\1%ss%s' % (no_mul_token, no_mul_token), code) # literal strings
        code = re_no_keyword(r'(\)) *(\w+)', code)
    if level >= 3:
        code = re_no_keyword(r'(\w+) +(\w+)', code)
    if level >= 10:
        code = re.sub(r'\) *\(', ')*(', code)
    code = code.replace(no_mul_token, '')
    return code % literals

def _strip_quotes(s):
    """
    Strips one set of outer quotes.

    INPUT:

    - ``s`` - a string

    OUTPUT:

    - a string with any single and double quotes on either side of
      ``s`` removed

    EXAMPLES:

    Both types of quotes work::

        sage: import sage.repl.preparse
        sage: sage.repl.preparse._strip_quotes('"foo.sage"')
        'foo.sage'
        sage: sage.repl.preparse._strip_quotes("'foo.sage'")
        'foo.sage'

    The only thing that is stripped is at most one set of outer quotes::

        sage: sage.repl.preparse._strip_quotes('""foo".sage""')
        '"foo".sage"'
    """
    if len(s) == 0:
        return s
    if s[0] in ["'", '"']:
        s = s[1:]
    if s[-1] in ["'", '"']:
        s = s[:-1]
    return s

def handle_encoding_declaration(contents, out):
    r"""
    Find a PEP 263-style Python encoding declaration in the first or
    second line of ``contents``. If found, output it to ``out`` and return
    ``contents`` without the encoding line; otherwise output a default
    UTF-8 declaration and return ``contents``.

    EXAMPLES::

        sage: from sage.repl.preparse import handle_encoding_declaration
        sage: import sys
        sage: c1='# -*- coding: latin-1 -*-\nimport os, sys\n...'
        sage: c2='# -*- coding: iso-8859-15 -*-\nimport os, sys\n...'
        sage: c3='# -*- coding: ascii -*-\nimport os, sys\n...'
        sage: c4='import os, sys\n...'
        sage: handle_encoding_declaration(c1, sys.stdout)
        # -*- coding: latin-1 -*-
        'import os, sys\n...'
        sage: handle_encoding_declaration(c2, sys.stdout)
        # -*- coding: iso-8859-15 -*-
        'import os, sys\n...'
        sage: handle_encoding_declaration(c3, sys.stdout)
        # -*- coding: ascii -*-
        'import os, sys\n...'
        sage: handle_encoding_declaration(c4, sys.stdout)
        # -*- coding: utf-8 -*-
        'import os, sys\n...'

    TESTS:

    These are some of the tests listed in PEP 263::

        sage: contents = '#!/usr/bin/python\n# -*- coding: latin-1 -*-\nimport os, sys'
        sage: handle_encoding_declaration(contents, sys.stdout)
        # -*- coding: latin-1 -*-
        '#!/usr/bin/python\nimport os, sys'

        sage: contents = '# This Python file uses the following encoding: utf-8\nimport os, sys'
        sage: handle_encoding_declaration(contents, sys.stdout)
        # This Python file uses the following encoding: utf-8
        'import os, sys'

        sage: contents = '#!/usr/local/bin/python\n# coding: latin-1\nimport os, sys'
        sage: handle_encoding_declaration(contents, sys.stdout)
        # coding: latin-1
        '#!/usr/local/bin/python\nimport os, sys'

    Two hash marks are okay; this shows up in SageTeX-generated scripts::

        sage: contents = '## -*- coding: utf-8 -*-\nimport os, sys\nprint(x)'
        sage: handle_encoding_declaration(contents, sys.stdout)
        ## -*- coding: utf-8 -*-
        'import os, sys\nprint(x)'

    When the encoding declaration doesn't match the specification, we
    spit out a default UTF-8 encoding.

    Incorrect coding line::

        sage: contents = '#!/usr/local/bin/python\n# latin-1\nimport os, sys'
        sage: handle_encoding_declaration(contents, sys.stdout)
        # -*- coding: utf-8 -*-
        '#!/usr/local/bin/python\n# latin-1\nimport os, sys'

    Encoding declaration not on first or second line::

        sage: contents ='#!/usr/local/bin/python\n#\n# -*- coding: latin-1 -*-\nimport os, sys'
        sage: handle_encoding_declaration(contents, sys.stdout)
        # -*- coding: utf-8 -*-
        '#!/usr/local/bin/python\n#\n# -*- coding: latin-1 -*-\nimport os, sys'

    We don't check for legal encoding names; that's Python's job::

        sage: contents ='#!/usr/local/bin/python\n# -*- coding: utf-42 -*-\nimport os, sys'
        sage: handle_encoding_declaration(contents, sys.stdout)
        # -*- coding: utf-42 -*-
        '#!/usr/local/bin/python\nimport os, sys'

    .. NOTE::

        - :pep:`263` says that Python will interpret a UTF-8
          byte order mark as a declaration of UTF-8 encoding, but I don't
          think we do that; this function only sees a Python string so it
          can't account for a BOM.

        - We default to UTF-8 encoding even though PEP 263 says that
          Python files should default to ASCII.

        - Also see https://docs.python.org/ref/encodings.html.

    AUTHORS:

    - Lars Fischer
    - Dan Drake (2010-12-08, rewrite for :trac:`10440`)
    """
    lines = contents.splitlines()
    for num, line in enumerate(lines[:2]):
        if re.search(r"coding[:=]\s*([-\w.]+)", line):
            out.write(line + '\n')
            return '\n'.join(lines[:num] + lines[(num+1):])

    # If we didn't find any encoding hints, use utf-8. This is not in
    # conformance with PEP 263, which says that Python files default to
    # ascii encoding.
    out.write("# -*- coding: utf-8 -*-\n")
    return contents

def preparse_file_named_to_stream(name, out):
    r"""
    Preparse file named \code{name} (presumably a .sage file), outputting to
    stream \code{out}.
    """
    name = os.path.abspath(name)
    with open(name) as f:
        contents = f.read()
    contents = handle_encoding_declaration(contents, out)
    parsed = preparse_file(contents)
    out.write('#'*70+'\n')
    out.write('# This file was *autogenerated* from the file %s.\n' % name)
    out.write('#'*70+'\n')
    out.write(parsed)

def preparse_file_named(name):
    r"""
    Preparse file named \code{name} (presumably a .sage file), outputting to a
    temporary file.  Returns name of temporary file.
    """
    from sage.misc.temporary_file import tmp_filename
    tmpfilename = tmp_filename(os.path.basename(name)) + '.py'
    with open(tmpfilename, 'w') as out:
        preparse_file_named_to_stream(name, out)
    return tmpfilename


"""
Silvio Severino - malware classification
"""
import main
from sklearn.svm import SVC
from sklearn import linear_model
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, f1_score
from sklearn.metrics import accuracy_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
import numpy as np
import matplotlib.pyplot as plt
from prettytable import PrettyTable

print ("Reading data ...")
x_all, y_all = main.read(LOAD_DATA=False)
x_train, x_test, y_train, y_test = train_test_split(x_all, y_all, test_size=0.3, random_state=42)

modelLogistic = [
        linear_model.LogisticRegression(),
        linear_model.LogisticRegression(C=10, class_weight=None, dual=False, fit_intercept=True,
                                        intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,
                                        penalty='l1', random_state=None, solver='liblinear', tol=0.0001,
                                        verbose=0, warm_start=False),
        linear_model.LogisticRegression(C=1, class_weight=None, dual=False, fit_intercept=True,
                                        intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,
                                        penalty='l2', random_state=None, solver='lbfgs', tol=0.0001,
                                        verbose=0, warm_start=False)]
for model in modelLogistic:
    model.fit(x_train, y_train)
    y_pred = model.predict(x_test)
    accuLog = accuracy_score(y_test, y_pred)
    fLog = f1_score(y_test, y_pred)

modelSvm = [SVC(),
              SVC(C=1000, cache_size=200, class_weight=None, coef0=0.0,
                  decision_function_shape='ovr', degree=3, gamma=0.01, kernel='rbf',
                  max_iter=-1, probability=False, random_state=None, shrinking=True,
                  tol=0.001, verbose=False)
              ]
for model in modelSvm:
    model.fit(x_train, y_train)
    y_pred = model.predict(x_test)
    accuSvm = accuracy_score(y_test, y_pred)
    fSvm = f1_score(y_test, y_pred)

modelRf = [RandomForestClassifier(),
          RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
                                 max_depth=None, max_features='sqrt', max_leaf_nodes=None,
                                 min_impurity_decrease=0.0, min_impurity_split=None,
                                 min_samples_leaf=1, min_samples_split=2,
                                 min_weight_fraction_leaf=0.0, n_estimators=1000, n_jobs=1,
                                 oob_score=False, random_state=None, verbose=0,
                                 warm_start=False),
          RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
                                 max_depth=None, max_features='sqrt', max_leaf_nodes=None,
                                 min_impurity_decrease=0.0, min_impurity_split=None,
                                 min_samples_leaf=1, min_samples_split=2,
                                 min_weight_fraction_leaf=0.0, n_estimators=1000, n_jobs=1,
                                 oob_score=False, random_state=None, verbose=0,
                                 warm_start=False)
          ]
for model in modelRf:
    model.fit(x_train, y_train)
    y_pred = model.predict(x_test)
    accuRf = accuracy_score(y_test, y_pred)
    fRf = f1_score(y_test, y_pred)

forest = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
                                max_depth=None, max_features='sqrt', max_leaf_nodes=None,
                                min_impurity_decrease=0.0, min_impurity_split=None,
                                min_samples_leaf=1, min_samples_split=2,
                                min_weight_fraction_leaf=0.0, n_estimators=1000, n_jobs=1,
                                oob_score=False, random_state=None, verbose=0,
                                warm_start=False)

#Show the result
table = PrettyTable()
print("Malware classification")
table.field_names = ["Algorithms", "Accuracy", "F1 score"]
table.add_row(["Logistic regression", accuLog, fLog])
table.add_row(["Support vector machine", accuSvm, fSvm])
table.add_row(["Random forest", accuRf, fRf])
print(table)

# Show the plot of features
forest = modelRf[0]
forest.fit(x_all, y_all)
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],axis=0)
indices = np.argsort(importances)[::-1]
plt.figure()
plt.title("Feature importances")
indices = sorted(indices)
indices_text = ['S0','S1','S2','S3','S4','S5','S6','S7','S8',]
plt.bar(range(x_all.shape[1]), importances[indices], color="g", yerr=std[indices], align="center")
plt.xticks(range(x_all.shape[1]), indices_text)
plt.xlim([-1, x_all.shape[1]])
plt.show()


from datetime import datetime
import shutil
import tempfile
import os
import zipfile

import floppyforms as forms

from ..core.conf import conf
from ..core.forms import BareTextarea
from .importer import CSVAddressImporter
from . import models, tasks

class AddressForm(forms.ModelForm):
    class Meta:
        model = models.Address
        widgets = {"notes": BareTextarea}
        fields = [
            "street_prefix", "street_number", "street_name", "street_type",
            "street_suffix", "edited_street",
            "city", "state", "multi_units", "complex_name", "notes"]

    def __init__(self, *args, **kwargs):
        super(AddressForm, self).__init__(*args, **kwargs)
        if conf.MLT_DEFAULT_STATE is not None:
            self.fields["state"].initial = conf.MLT_DEFAULT_STATE

    def clean(self):
        if not (
            self.cleaned_data["street_name"] or
            self.cleaned_data["edited_street"]
            ):
            raise forms.ValidationError("Please enter a street address.")

        return self.cleaned_data

    def save(self, user):
        address = super(AddressForm, self).save(commit=False)

        if address.pk is None:
            address.input_street = (
                address.parsed_street or address.edited_street)

        # default for new addresses, should be reset when an address is edited.
        address.geocode_failed = False
        address.save(user=user)

        return address

class AddressImportForm(forms.Form):
    file = forms.FileField()
    tag = forms.CharField()

    def clean_tag(self):
        tag = self.cleaned_data["tag"]
        if models.AddressBatch.objects.filter(tag=tag).exists():
            raise forms.ValidationError("This batch tag is already used.")
        return tag

    def save(self, user):
        i = CSVAddressImporter(
            user=user,
            timestamp=datetime.now(),
            tag=self.cleaned_data["tag"])

        return i.process_file(self.cleaned_data["file"])

class LoadParcelsForm(forms.Form):
    shapefile = forms.FileField()

    def clean_shapefile(self):
        try:
            z = zipfile.ZipFile(self.cleaned_data["shapefile"], 'r')
        except zipfile.BadZipfile:
            raise forms.ValidationError(
                "Uploaded file is not a valid zip file.")

        shapefile_path = None
        target_dir = tempfile.mkdtemp(suffix="-mlt-parcel-shapefile")
        try:
            for name in z.namelist():
                if name.startswith(os.path.sep) or os.path.pardir in name:
                    raise forms.ValidationError(
                        "Zip file contains unsafe paths (absolute or with ..).")
                z.extract(name, target_dir)
                if name.endswith(".shp"):
                    shapefile_path = os.path.join(target_dir, name)

            if shapefile_path is None:
                raise forms.ValidationError(
                    "Unable to find a .shp file in uploaded zip file.")
        except forms.ValidationError:
            shutil.rmtree(target_dir)
            raise

        self.cleaned_data["target_dir"] = target_dir
        self.cleaned_data["shapefile_path"] = shapefile_path

        return self.cleaned_data["shapefile"]

    def save(self):
        return tasks.load_parcels_task.delay(
            self.cleaned_data["target_dir"],
            self.cleaned_data["shapefile_path"])


# Bellman-Ford Algorithm
import sys

from collections import defaultdict, namedtuple
from math import inf, isinf, isfinite

Edge = namedtuple('Edge', ['node_from', 'node_to', 'weight'])

class DirectedGraph:
    def __init__(self, nr_nodes, edge_list):
        self._nr_nodes = nr_nodes
        self._adjacency_list = self._build_adjacency_list(edge_list)

    def __iter__(self):
        yield from self._adjacency_list.items()

    @staticmethod
    def _build_adjacency_list(edge_list):
        adjacency_list = defaultdict(list)
        for edge in edge_list:
            adjacency_list[edge.node_from].append(edge)
        return dict(adjacency_list)

    @property
    def nr_nodes(self):
        return self._nr_nodes

def convert_to_intlist(line):
    return tuple(int(item) for item in line.split())

def read_edges(reader, nr_edges):
    edges = []
    for _ in range(nr_edges):
        a, b, w = convert_to_intlist(next(reader))
        edges.append(Edge(node_from=a, node_to=b, weight=w))
    return edges

def update_distances(graph, distances):
    for node, neighbours in graph:
        dist_node = distances[node-1]
        if isfinite(dist_node):
            for _, neighbour, weight in neighbours:
                distance_through_node = dist_node + weight
                if distances[neighbour-1] > distance_through_node:
                    distances[neighbour-1] = distance_through_node

def bellman_ford_algorithm(graph, node):
    n = graph.nr_nodes
    distances = [inf] * n
    distances[node-1] = 0
    for _ in range(n):
        update_distances(graph, distances)
    return distances

def main():
    reader = sys.stdin
    nr_nodes, nr_edges = convert_to_intlist(next(reader))
    edge_list = read_edges(reader, nr_edges)
    graph = DirectedGraph(nr_nodes, edge_list)
    result = bellman_ford_algorithm(graph, node=1)
    print(' '.join(map(lambda x: 'x' if isinf(x) else str(x), result)))

if __name__ == '__main__':
    main()


import os
import sys
from collections import OrderedDict

import yaml
import yaml.representer

def dict_representer(dumper, data):
    return yaml.representer.SafeRepresenter.represent_dict(dumper, data.items())

def str_representer(dumper, data):
    if "\n" in data:
        style = "|"
    else:
        style = None

    return yaml.representer.SafeRepresenter.represent_scalar(
        dumper, "tag:yaml.org,2002:str", data, style=style
    )

yaml.add_representer(dict, dict_representer)
yaml.add_representer(OrderedDict, dict_representer)
yaml.add_representer(str, str_representer)

def load_yaml(file_path):
    with open(os.path.expanduser(file_path), "r", encoding="utf-8") as f:
        return yaml.safe_load(f)

def dump_yaml(data, file_path=None):
    if not file_path:
        return yaml.dump(
            data, default_flow_style=False, allow_unicode=True, width=sys.maxsize
        )

    with open(os.path.expanduser(file_path), "w", encoding="utf-8") as f:
        yaml.dump(
            data, f, default_flow_style=False, allow_unicode=True, width=sys.maxsize
        )


from django.db import models
from author.models import Author

# Create your models here
class Book(models.Model):
    id = models.AutoField
    title = models.CharField(max_length=100)
    image = models.ImageField(upload_to='books/')
    author = models.ForeignKey(Author, on_delete=models.CASCADE)

class Pages(models.Model):
    id = models.AutoField
    number = models.PositiveIntegerField()
    content = models.TextField(max_length=600)
    book = models.ForeignKey(Book, on_delete=models.CASCADE)


import os
import xml.etree.ElementTree
import cv2
import shutil
import time
import xml.etree.ElementTree
from bs4 import BeautifulSoup

start = time.time()

target_path = 'D:\신지영_업무\\0709_어트리뷰트인수인계\\attribute_crop_images\\common\Test_Set'
file_path = 'D:\신지영_업무\\0709_어트리뷰트인수인계\\attribute_crop_images\\common\\common_attribute_annotaion.txt'

xml_list = []
file_name_list = []
target_file_list = os.listdir(target_path)
for x in target_file_list:
    if x.endswith('xml'):
        xml_list.append(x)
        temp = x.split('.')
        file_name_list.append(temp[0])

def split_label(label):
    temp = str(label)
    temp0 = temp.split('<')
    temp1, result = temp0[1].split('>')
    return result

def split_name(name):
    temp = str(name)
    try:
        temp0 = temp.split("\"")
        result = temp0[1]
    except:
        return False
    return result

xml_count = 0

str_result = ''
print("xml을 읽어 속성 값 count 시작")
for xml in xml_list:
    if xml.endswith('xml'):
        xml_path = os.path.join(target_path, xml)
        file = open(xml_path, 'r', encoding='utf-8')
        lines = ''
        while True:
            line = file.readline()
            if not line: break
            lines += line
        file.close()
        bs = BeautifulSoup(lines, 'lxml')
        str_temp = '%s' % file_name_list[xml_count]
        temp = []
        temp_str = ''
        man = 0
        woman = 0
        genderdk = 0
        infant = 0
        child = 0
        teenager = 0
        adult = 0
        oldperson = 0
        backpack = 0
        totebag = 0
        shoulderbag = 0
        plasticbag = 0
        bagdk = 0
        bagnone = 0
        red = 0
        yellow = 0
        green = 0
        blue = 0
        brown = 0
        pink = 0
        gray = 0
        black = 0
        white = 0
        unknown = 0
        for box in bs.find_all('box'):
            label = 'head'
            lines = str(box).splitlines()
            for x in lines[1:]:
                result = split_label(x)
                if result != 'false':
                    label_name = split_name(x)
                    if label_name == 'gender':
                        if result == 'male':
                            man += 1
                        elif result == 'female':
                            woman += 1
                        else:
                            genderdk += 1
                    elif label_name == 'age':
                        if result == '8~13':
                            child += 1
                        elif result == '14~19':
                            teenager += 1
                        elif result == '20~70':
                            adult += 1
                        elif result == '70~':
                            oldperson += 1
                        else:
                            infant += 1
                    else:
                        if label_name == 'backpack':
                            backpack += 1
                        elif label_name == 'bagless':
                            bagnone += 1
                        elif label_name == 'unknown_bag':
                            bagdk += 1
                        elif label_name == 'plasticbag':
                            plasticbag += 1
                        elif label_name == 'shoulderbag':
                            shoulderbag += 1
                        elif label_name == 'handbag':
                            totebag += 1
                        elif label_name == 'bag_red':
                            red += 1
                        elif label_name == 'bag_yellow':
                            yellow += 1
                        elif label_name == 'bag_green':
                            green += 1
                        elif label_name == 'bag_blue':
                            blue += 1
                        elif label_name == 'bag_brown':
                            brown += 1
                        elif label_name == 'bag_pink':
                            pink += 1
                        elif label_name == 'bag_grey':
                            gray += 1
                        elif label_name == 'bag_black':
                            black += 1
                        elif label_name == 'bag_white':
                            white += 1
                        elif label_name == 'bag_color_unknown':
                            unknown += 1

        ALL_classes = "{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}".format(man, woman, genderdk, infant, child, teenager, adult, oldperson, backpack, totebag, shoulderbag, plasticbag, bagdk, bagnone, red, yellow, green, blue, brown, pink, gray, black, white, unknown)
        str_result += ALL_classes + '\n'

file = open(file_path, 'a')
for x in str_result:
    file.write(x)

file.close()


# from asyncio.windows_events import NULL
from importlib.resources import contents
import scrapy
from arabic_scrapper.helper import load_dataset_lists, parser_parse_isoformat, translate_text, datetime_now_isoformat

news_sites_list,categories_english,main_category,sub_category,platform,media_type,urgency = load_dataset_lists("alkuwaitia newspaper")
now = datetime_now_isoformat()

class AlkuwaitiaSpider(scrapy.Spider):
    name = 'alkuwaitia'
    start_urls = news_sites_list

    def start_requests(self):

        for i in range(len(self.start_urls)):
             yield scrapy.Request(url = self.start_urls[i], callback = self.parse, meta = {'category_english': categories_english[i],"main_category": main_category[i],"sub_category": sub_category[i],"platform": platform[i],"media_type": media_type[i],"urgency": urgency[i]})

    def parse(self, response):

        card_selector = "//div[@class='Caricature_item']/a/@href"
        for url in response.xpath(card_selector).extract():

            url = f'https://alkuwaityah.com{url}'
            yield scrapy.Request(url = url, callback = self.parse_page, meta = response.meta)

    def parse_page(self,response):

        contents = response.xpath("//div[@class='News_Details']/div[@class='desc']/div/text()").extract()
        if(not len(contents)):
            contents = response.xpath("//div[@class='News_Details']/div[@class='desc']/div/strong/text()").extract()
            if(not len(contents)):
                contents = response.xpath("//div[@class='News_Details']/div[@class='desc']/div/div/text()").extract()

        contents = "\n\n".join(contents)

        yield {
                "news_agency_name": "alkuwaitia newspaper",
                "page_url" : response.url,
                "category" : response.meta["category_english"],
                "title" : response.xpath("//div[@class='title']/h3/text()").extract_first(),
                "contents": contents,
                "date" :  parser_parse_isoformat(translate_text(response.xpath("//div[@class='date']/span/text()").extract_first())),
                "author_name" : "alkuwaitia newspaper",
                "image_url" : "https://alkuwaityah.com" + response.xpath("//div[@class='News_img']/img/@src").extract_first(),

                "main_category": response.meta["main_category"],
                "sub_category": response.meta["sub_category"],
                "platform": response.meta["platform"],
                "media_type": response.meta["media_type"],
                "urgency": response.meta["urgency"],
                "created_at": now,
                "updated_at": now,
                "deleted_at": None
         }