import sys
import os
sys.path.append(os.path.abspath('.'))

import torch
import utils as ut
from train import *
from dataset import load_train_data, load_test_data
import constants

def main(config):
    # Fixed random number seed
    torch.manual_seed(config.seed)
    torch.cuda.manual_seed_all(config.seed)

    # Initialize image evaluation metrics
    best_psnr = 0.0
    best_ssim = 0.0

    if config.train.checkpoint.is_log:
        ut.log_on_train_start(log_name=config.exp_name, config=config)

    checkpoint_dir = os.path.join(constants.ROOT, 'model', config.exp_name)
    ut.create_dir(checkpoint_dir)

    # Define basic elements for training
    netG, netD = define_model(config)

    # optimG = define_optimizer(netG, config)
    # optimD = define_optimizer(netD, config)

    optimG = optim.Adam(netG.parameters(),
                       lr=config.train.optim.lr,
                       betas=config.train.optim.betas)
    optimD = optim.Adam(netD.parameters(),
                       lr=config.train.optim.lr,
                       betas=config.train.optim.betas)

    schedulerG = define_scheduler(optimG, config)
    schedulerD = define_scheduler(optimD, config)

    if config.train.checkpoint.load_model:
        G_state_dict, optimG_state_dict, start_epoch = ut.load_checkpoint(config.train.checkpoint.gen)
        D_state_dict, optimD_state_dict, start_epoch = ut.load_checkpoint(config.train.checkpoint.disc)
        netG.load_state_dict(G_state_dict)
        netD.load_state_dict(D_state_dict)
        optimG.load_state_dict(optimG_state_dict)
        optimD.load_state_dict(optimD_state_dict)

    # Loss function
    content_criteria = nn.MSELoss()
    adversarial_criteria = nn.BCEWithLogitsLoss()
    feature_extractor = VGGLoss()
    feature_extractor = feature_extractor.to(constants.DEVICE)
    feature_extractor.eval()

    # Data loader
    print("Loading data ...")
    train_loader = load_train_data(root=config.train.dataset.data_dir, batch_size=config.train.hyp.batch_size)
    test_loader = load_test_data(hr_root=config.test.dataset.hr_dir, lr_root=config.test.dataset.lr_dir)
    print("Finish loading data")

    for epoch in range(config.train.hyp.num_epoch):
        netG.train()
        netD.train()
        D_loss, G_loss = train(
            train_loader,
            epoch,
            netG,
            netD,
            optimG,
            optimD,
            content_criteria,
            adversarial_criteria,
            feature_extractor,
            config)

        schedulerD.step()
        schedulerG.step()

        psnr, ssim = test(test_loader, netG)
        is_best = psnr > best_psnr and ssim > best_psnr
        best_psnr = max(psnr, best_psnr)
        best_ssim = max(ssim, best_ssim)

        print("D_loss: %.6f, G_loss: %.6f, psnr: %.6f, ssim: %.6f" % (D_loss, G_loss, psnr, ssim))

        ut.save_checkpoint(
            {
                "epoch": epoch + 1,
                "model": netD.state_dict(),
                "optimizer": optimD.state_dict(),
            },
            f'{checkpoint_dir}/disc_{epoch+1}.pth.tar',
            f'{checkpoint_dir}/disc_best.pth.tar',
            is_best)

        ut.save_checkpoint(
            {
                "epoch": epoch + 1,
                "model": netG.state_dict(),
                "optimizer": optimG.state_dict(),
            },
            f'{checkpoint_dir}/gen_{epoch+1}.pth.tar',
            f'{checkpoint_dir}/gen_best.pth.tar',
            is_best)

if __name__ == '__main__':
    main_config = ut.read_config(os.path.join(constants.ROOT,'config/config.yaml'))
    main(main_config)


import math

import matplotlib.pyplot as plt
import numpy as np
from matplotlib import gridspec
from scipy.special import factorial

from plot.plot_data import plot_matrixImage

def normalize(X):
    f_min, f_max = X.min(), X.max()
    return (X - f_min) / (f_max - f_min)

def gabor_kernel_2(frequency, sigma_x, sigma_y, theta=0, offset=0, ks=61):
    w = np.floor(ks / 2)
    y, x = np.mgrid[-w:w + 1, -w:w + 1]
    rotx = x * np.cos(theta) + y * np.sin(theta)
    roty = -x * np.sin(theta) + y * np.cos(theta)
    g = np.zeros(y.shape)
    g[:] = np.exp(-0.5 * (rotx ** 2 / sigma_x ** 2 + roty ** 2 / sigma_y ** 2))
    g /= 2 * np.pi * sigma_x * sigma_y
    g *= np.cos(2 * np.pi * frequency * rotx + offset)
    return g

def gabor_kernel_3(frequency, x_c, y_c, sigma_x, sigma_y, theta=0, offset=0, ks=61, scale=1):
    w = np.floor(ks / 2)
    y, x = np.mgrid[-w:w + 1, -w:w + 1]
    rotx = (x - x_c) * np.cos(theta) + (y - y_c) * np.sin(theta)
    roty = -(x - x_c) * np.sin(theta) + (y - y_c) * np.cos(theta)
    g = np.zeros(y.shape)
    g[:] = np.exp(-0.5 * (rotx ** 2 / sigma_x ** 2 + roty ** 2 / sigma_y ** 2))
    g /= 2 * np.pi * sigma_x * sigma_y
    g *= np.cos(2 * np.pi * frequency * rotx + offset)
    return g * scale

def poisson(k, lamb):
    """poisson pdf, parameter lamb is the fit parameter"""
    return (lamb ** k / factorial(k)) * np.exp(-lamb)

def negLogLikelihood(params, data):
    """ the negative log-Likelohood-Function"""
    lnl = - np.sum(np.log(poisson(data, params[0])))
    return lnl

# def tfm_poisson_pdf(x, mu):
#     y, J = transformation_and_jacobian(x)
#     # For numerical stability, compute exp(log(f(x)))
#     return np.exp(y * np.log(mu) - mu - gammaln(y + 1.)) * J

def plot_conv_weights(weights, model_name):
    length = weights.shape[0] * weights.shape[2]
    matrix = np.zeros([length, 0])
    for i in range(0, weights.shape[0]):
        row = np.empty([0, weights.shape[2]])
        for j in range(0, weights.shape[1]):
            row = np.concatenate((row, weights[i, j]), axis=0)
        # f_min, f_max = np.min(row), np.max(row)
        # row = (row - f_min) / (f_max - f_min)
        # row[0,0] = 0
        matrix = np.concatenate((matrix, row), axis=1)
        # matrix[0,0] = 1
    f_min, f_max = np.min(matrix), np.max(matrix)
    matrix = (matrix - f_min) / (f_max - f_min)
    plot_matrixImage(matrix, 'weights_' + model_name)

def plot_weights(weights, model_name, gs=None, name=None):
    show = False
    if gs is None:
        plt.figure(figsize=(10, 2), frameon=False)
        inner = gridspec.GridSpec(weights.shape[0], weights.shape[1], wspace=0.2, hspace=0.2)
        show = True
    else:
        inner = gridspec.GridSpecFromSubplotSpec(weights.shape[0], 8,
                                                 subplot_spec=gs, wspace=0.1, hspace=0.1)
    # gs = gridspec.GridSpec(, width_ratios=[1] * weights.shape[1],
    #                        wspace=0.5, hspace=0.5, top=0.95, bottom=0.05, left=0.1, right=0.95)
    idx = 0
    for i in range(0, weights.shape[0]):
        for j in range(0, weights.shape[1]):
            kernel1 = weights[i, j]
            ax_ = plt.subplot(inner[i, j])
            ax_.set_xticks([])
            ax_.set_yticks([])
            ax_.set_axis_off()
            ax_.imshow(kernel1, cmap='gray')
            #
            idx += 1
            if j == 0:
                ax_.set_title(name, pad=10, weight='semibold', size=16)
    if show:
        plt.tight_layout()
        plt.savefig(f'weights_{model_name}.png')
        plt.show()

def show_kernels(weights, func_name, gs=None):
    number = math.ceil(math.sqrt(weights.shape[0]))
    img = np.transpose(weights, (0, 2, 3, 1))
    idx = 0
    show = False
    if gs is None:
        plt.figure(figsize=(10, 10))
        inner = gridspec.GridSpec(1, weights.shape[0], wspace=0.2, hspace=0.2)
        show = True
    else:
        inner = gridspec.GridSpecFromSubplotSpec(1, 8,
                                                 subplot_spec=gs, wspace=0.1, hspace=0.1)

    # fig, axes = pyplot.subplots(ncols=weights.shape[0], figsize=(20, 4))
    for j in range(weights.shape[0]):  # in zip(axes, range(weights.shape[0])):
        # for i in range(number):
        ax_ = plt.subplot(inner[idx])
        ax_.set_xticks([])
        ax_.set_yticks([])
        ax_.set_axis_off()
        # ax.set_title(f'Kernel {idx}', pad=3)
        # imgs = img[range(j*8, (j*8)+number)]
        channel = img[idx]
        f_min, f_max = channel.min(), channel.max()
        channel = (channel - f_min) / (f_max - f_min)
        ax_.imshow(channel)
        if j == 0:
            ax_.set_title(func_name, pad=10, weight='bold', size=18)
        idx += 1

    if show:
        plt.tight_layout()
        plt.savefig(f'kernels_{func_name}.png')
        plt.show()

def similarity(m1, m2):
    sum = 0
    for i in range(m1.shape[0]):
        for j in range(m1.shape[1]):
            sum += np.abs(m1[i, j] - m2[i, j])
    return sum / (m1.shape[0] * m1.shape[1])


from django.http import HttpResponse
from django.shortcuts import render

def index(request):
    #params = {'name':'Tarbi'}
    return render(request,"index.html")

def analyze(request):
    #Get the text
    djtext = request.POST.get('text','default')

    #Operations
    removepunc = request.POST.get('removepunc','default')
    fullcaps = request.POST.get('fullcaps','default')
    count = request.POST.get('count','default')
    newlineremover = request.POST.get('newlineremover','default')
    spaceremover = request.POST.get('spaceremover','default')
    #Result text
    analyzed = ""

    if removepunc == "on":
        punctuations = '''!()-[]{};:'"\,<>./?@#$%^&*_~'''
        analyzed = ""
        for char in djtext:
            if char not in punctuations:
                analyzed = analyzed + char
        params = {'purpose': 'Remove Punctuations', 'analyzed_text': analyzed,'input_text': djtext}
        djtext = analyzed
        #return render(request, 'analyze.html', params)

    if fullcaps == "on":
        analyzed = djtext.upper()
        params = {'purpose': 'To Upper Case', 'analyzed_text': analyzed, 'input_text': djtext}
        djtext = analyzed
        #return render(request, 'analyze.html', params)

    if count == "on":
        cnt = 0
        analyzed = ""
        for x in djtext:
            if(x.isdigit()):
                continue
            analyzed+=x
        params = {'purpose': 'Number Remover', 'analyzed_text': analyzed, 'input_text': djtext}
        djtext = analyzed
        #return render(request, 'analyze.html', params)

    if newlineremover == "on":
        analyzed = ""
        for x in djtext:
            if x != '\n' and x!='\r':
                analyzed+=x
        params = {'purpose': 'New Line Remove', 'analyzed_text': analyzed, 'input_text': djtext}
        djtext = analyzed
        #return render(request, 'analyze.html', params)

    if spaceremover =="on":
        analyzed =""
        for x in djtext:
            if(x!=' '):
                analyzed+=x
        params = {'purpose': 'Space Remove', 'analyzed_text': analyzed, 'input_text': djtext}
        djtext = analyzed

    if spaceremover !="on" and newlineremover != "on" and count != "on" and fullcaps != "on" and removepunc != "on":
        return HttpResponse("Select any option and try again")
    return render(request, 'analyze.html', params)

def about(request):
    return render(request, 'about.html')
def contact(request):
    return render(request, 'contact.html')


import numpy as np

from inet.models.solvers.tf_lite import MultiTaskModel
from inet.models.tf_lite.tflite_methods import evaluate_interpreted_model

class TwoStageModel(MultiTaskModel):
    """
    Object detection model using dependent/sequential methods to solve the localization and classification tasks.
    A regressor predicts the location, the original input image gets cropped to a patch containing the extracted
    Bounding Box. Afterwards a classifier predicts the class label, based on the cropped input.

    [Similar to `IndependentModel`]

    Example:
        >>> from tensorflow.keras.applications.mobilenet import MobileNet
        >>> from inet.models.architectures.classifier import Classifier
        >>> from inet.models.architectures.bounding_boxes import BoundingBoxRegressor
        >>> clf_backbone = MobileNet(weights='imagenet', include_top=False, input_shape=(224, 224))
        >>> reg_backbone = MobileNet(weights='imagenet', include_top=False, input_shape=(224, 224))
        >>> regressor = BoundingBoxRegressor(reg_backbone)
        >>> classifier = Classifier(clf_backbone)
        >>> solver = TwoStageModel(regressor, classifier, (224, 224, 3), False)
    """
    ## Name of model architecture
    model_name = 'two-stage-model'

    def predict(self, X):
        """
        Performs dependent predictions on input `X`.

        Regressor receives raw `X` -> returns `c`
        `X` is cropped using `c` -> `X_hat`
        Classifier receives `X_hat` -> returns `y`

        :param X: vector of input images
        :return: Prediction Tuple [y, c]
        """
        if self.is_tflite:
            bbs = evaluate_interpreted_model(self.regressor, X)
            bbs = np.array(bbs).reshape((len(bbs), -1))
        else:
            bbs = self.regressor.predict(X)

        cropped_images = np.array([i for i in map(self.crop_image, zip(X.copy(), bbs.copy()))])

        if self.is_tflite:
            clf = evaluate_interpreted_model(self.classifier, cropped_images)
            clf = np.array(clf).reshape((len(clf), -1))
            return np.c_[clf, bbs]

        classifications = self.classifier.predict(cropped_images)
        return np.c_[classifications, bbs]


import csv
from dateutil.parser import parse
from decimal import *
import pandas as pd
import gc
import os
from multiprocessing import Process
def intersection(list1, list2):
    res = []
    idx1 = 0
    while idx1 < len(list1):
        if list1[idx1] in list2:
            res.append(list1[idx1])
        idx1 += 1
    return res

def get_project_info():
    snapshot_id = 60295045
    one_day = 86400
    one_week = 604800
    one_month = 2628000
    cnt = 0
    min_date = 1165524100
    total_commits = 0
    authors = []

    daily_cnt = 1
    daily_commits = []
    daily_contributors = []
    daily_temp_contrib = []

    weekly_cnt = 1
    weekly_commits = []
    weekly_contributors = []
    weekly_temp_contrib = []

    monthly_cnt = 1
    monthly_commits = []
    monthly_contributors = []
    monthly_temp_contrib = []
    for lines in pd.read_csv('/home/sv/big_snapshot_.csv', encoding='utf-8', header=None, chunksize=1000000):
        for line in lines.iterrows():
            author = int(line[1][1])
            date = int(line[1][0])
            cnt += 1
            print(cnt)
            total_commits += 1
            if author not in authors:
                authors.append(author)

            #daily
            if author not in daily_contributors and date <= min_date + one_day:
                daily_contributors.append(author)
            if date <= min_date + one_day*daily_cnt:
                if author not in daily_temp_contrib:
                    daily_temp_contrib.append(author)
            else:
                daily_contributors = intersection(daily_contributors, daily_temp_contrib)
                daily_temp_contrib = []
                daily_cnt += 1
                while date > min_date + one_day*daily_cnt:
                    daily_commits.append(0)
                    daily_contributors = []
                    daily_cnt += 1
                daily_temp_contrib.append(author)
                daily_commits.append(1)

            #weekly
            if author not in weekly_contributors and date <= min_date + one_week:
                weekly_contributors.append(author)
            if date <= min_date + one_week*weekly_cnt:
                if author not in weekly_temp_contrib:
                    weekly_temp_contrib.append(author)
            else:
                weekly_contributors = intersection(weekly_contributors, weekly_temp_contrib)
                weekly_temp_contrib = []
                weekly_cnt += 1
                while date > min_date + one_week*weekly_cnt:
                    weekly_commits.append(0)
                    weekly_contributors = []
                    weekly_cnt += 1
                weekly_temp_contrib.append(author)
                weekly_commits.append(1)

            #monthly
            if author not in monthly_contributors and date <= min_date + one_month:
                monthly_contributors.append(author)
            if date <= min_date + one_month*monthly_cnt:
                if author not in monthly_temp_contrib:
                    monthly_temp_contrib.append(author)
            else:
                monthly_contributors = intersection(monthly_contributors, monthly_temp_contrib)
                monthly_temp_contrib = []
                monthly_cnt += 1
                while date > min_date + one_month*monthly_cnt:
                    monthly_commits.append(0)
                    monthly_contributors = []
                    monthly_cnt += 1
                monthly_temp_contrib.append(author)
                monthly_commits.append(1)

        #writing to file
        daily_freq = Decimal(sum(daily_commits))/Decimal(len(daily_commits))
        weekly_freq = Decimal(sum(weekly_commits))/Decimal(len(weekly_commits))
        monthly_freq = Decimal(sum(monthly_commits))/Decimal(len(monthly_commits))
        df = pd.DataFrame({
            'snapshot_id': [snapshot_id],
            'total_commits': [total_commits],
            'total_authors': [len(authors)],
            'daily_freq': [daily_freq],
            'daily_contributors': [len(daily_contributors)],
            'weekly_freq': [weekly_freq],
            'weekly_contributors': [len(weekly_contributors)],
            'monthly_freq': [monthly_freq],
            'monthly_contributors': [len(monthly_contributors)]
        })
        df.to_csv('/home/sv/project-git-metrics-first.csv', mode = 'a', header = False, index = False)
if __name__ == '__main__':
    get_project_info()


import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

h, k_up1, k_up2 = np.loadtxt('./Reactions/Kup.dat',skiprows=3,usecols=(1,5799+1,5800+1),unpack=True)

h *= 1e-5

k_up = k_up1 + k_up2

plt.xscale('log')

plt.plot(k_up,h,'k-')

plt.savefig('./N2O-rates.pdf',bbox_inches='tight')


import math
import torch.nn as nn

class HRNET_NECK(nn.Module):
    def __init__(self, in_channels, feature_size=256):
        super(HRNET_NECK, self).__init__()
        C2_size, C3_size, C4_size, C5_size = in_channels

        # P2
        self.P2_1 = nn.Conv2d(C2_size, feature_size, kernel_size=1, stride=1, padding=0)
        self.P2_2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, stride=2, padding=1)

        # P3
        self.P3_1 = nn.Conv2d(C3_size, feature_size, kernel_size=1, stride=1, padding=0)
        self.P3_2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, stride=2, padding=1)

        # P4
        self.P4_1 = nn.Conv2d(C4_size, feature_size, kernel_size=1, stride=1, padding=0)
        self.P4_2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, stride=2, padding=1)

        # P5
        self.P5_1 = nn.Conv2d(C5_size, feature_size, kernel_size=1, stride=1, padding=0)

        # "P6 is obtained via a 3x3 stride-2 conv on C5"
        self.P6 = nn.Conv2d(feature_size, feature_size, kernel_size=3, stride=2, padding=1)

        # "P6 is computed by applying ReLU followed by a 3x3 stride-2 conv on P6"
        self.P7_1 = nn.ReLU()
        self.P7_2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, stride=2, padding=1)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def forward(self, inputs):

        C2, C3, C4, C5 = inputs

        P2_x = self.P2_1(C2)
        P2_downsample = self.P2_2(P2_x)

        P3_x = self.P3_1(C3)
        P3_x = P2_downsample + P3_x
        P3_downsample = self.P3_2(P3_x)

        P4_x = self.P4_1(C4)
        P4_x = P4_x + P3_downsample
        P4_downsample = self.P4_2(P4_x)

        P5_x = self.P5_1(C5)
        P5_x = P5_x + P4_downsample

        P6_x = self.P6(P5_x)

        P7_x = self.P7_1(P6_x)
        P7_x = self.P7_2(P7_x)

        return [P3_x, P4_x, P5_x, P6_x, P7_x]


import sys

input = sys.stdin.readline

n = int(input())
graph = []
move = [[-1, 0], [1, 0], [0, -1], [0, 1]]
ans = []
number = 0

for _ in range(n):
    graph.append([int(i) for i in (input().strip())])

def dfs(x, y, initial):
  global cnt
  cnt = initial
  graph[x][y] = 0

  for i in move:
    next_x = x + i[0]
    next_y = y + i[1]
    if 0 <= next_x < n and 0 <= next_y < n and graph[next_x][next_y] == 1:
      cnt += 1
      dfs(next_x, next_y, cnt)

for x in range(n):
    for y in range(n):
        if graph[x][y] == 1:
          dfs(x, y, 1)
          number += 1
          ans.append(cnt)

print(number)
print("\n".join(str(i) for i in sorted(ans)))

백준/Silver/2667. 단지번호붙이기/단지번호붙이기.py


"""
String reversing. What could be simpler?

Usage:
python string_reverse.py <string_to_reverse>
"""

import sys

def string_reverse(input_string):
    result = ''
    for i in input_string:
        result = i + result
    return result

if __name__ == '__main__':
    if len(sys.argv) < 2:
        sys.stderr.write('This util needs some input: ./string_reverse.py string_to_reverse\n')
        sys.exit()
    input_string = sys.argv[1]
    if len(sys.argv) > 2:
        sys.stderr.write('Warning: the second and the following arguments ignored\n')
    sys.stdout.write('Initial string: {}\n'.format(input_string))
    sys.stdout.write('Reversed string: {}\n'.format(string_reverse(input_string)))


from collections import defaultdict
import Policy as policy
import random
import numpy as np
import matplotlib.pyplot as plt
# import pytorch as torch

class Agent:

    def __init__(self, env) -> None:
        self.env = env

        # replay_buffer = {(state, action) : (state_, reward)}
        self.replay_buffer = defaultdict(lambda : tuple(list, float))

        # visisted_states = {state}
        self.visisted_states = set()

        self.state = self.env.reset()

    # Decide an action from a given state
    # Returns True if the environment is done (won or lost)
    def next(self) -> bool:
        return False

# Shallow agent for discrete environments or continuous environments with a small state space
class ShallowAgent(Agent):

    def __init__(self, env) -> None:
        super().__init__(env)

        # self.v = {state : value}
        self.V = defaultdict(lambda : 0)

        # self.q = {(state, action) : value}
        self.Q = defaultdict(lambda : 0)

# Deep agent for continuous environments or discrete environments with a large state space
class DeepAgent(Agent):

    def __init__(self, env, weight_size) -> None:
        super().__init__(env)

        # self.w = torch.rand(weight_size)

class DiscreteQLearningAgent(Agent):

    def __init__(self, env) -> None:
        super().__init__(env)

class DiscreteActionValueIterationAgent(Agent):

    def __init__(self, env, policy = policy.random) -> None:
        super().__init__(env)
        self.policy = policy

    def next(self) -> bool:
        # Action value iteration function
        q = defaultdict(lambda : 0)

        for state in self.env.get_states():
            for action in self.env.A(state):
                trans_prob = self.env.get_transistion_probabilities(state, action)

                temp = 0
                for next_state, reward in trans_prob:
                    pi = self.policy(self.env, next_state, self.q)
                    temp += trans_prob[(next_state, reward)]*(reward + 0.99*sum([pi[action_] * self.q[(next_state,action_)] for action_ in self.env.A(next_state)]))

                q[(state,action)] = temp

        self.q = q

        # Value iteration function
        v = defaultdict(lambda : 0)
        for state in self.env.get_states():
            actions = self.env.A(state)
            for action in actions:
                pi = 1/len(actions)
                temp = 0
                trans_prob = self.env.get_transistion_probabilities(state, action)

                # Get reward, next state
                for next_state, reward in trans_prob.keys():

                    temp += trans_prob[(next_state,reward)]*(reward + 0.99*self.v[next_state])

                pi = self.policy(self.env, state, self.q)
                v[state] += pi[action] * temp

        self.v = v

        return True

class ManualAgent(Agent):

    def __init__(self, env) -> None:
        super().__init__(env)

    def next(self):

        while(True):
            print("Current state: " + str(self.env.state))
            print("Total reward: " + str(self.env.get_accumulative_reward(self)))
            print("---------------")
            print("Enter next action")
            print("Avaliable actions: " + str(self.env.A(self.state)))
            try:
                action = input()

                if action == "exit":
                    return False

                action = int(action)

                print("\n")

                if(action <= self.env.A(self.state)[-1]):

                    next_state, reward = self.env.step(self.state, action)

                    self.state = next_state
                    self.visited_states.append(self.state)
                    self.previous_actions.append(action)
                    self.obtained_rewards.append(reward)

                    print("---------------")
                    print("Reward: " + str(reward))
                    print("---------------")
                    return True
                else:
                    print("Invalid action")

            except ValueError:
                print("The provided string is not a valid representation of an integer.\n"+
                        "Please enter a valid integer in the action space")


COUNT=0
count2=0
#history=[1,1,0,0]
def q1(history):
	def perm(n,begin,end):
		global COUNT
		global count2
		if begin>=end:
		    for i in range(0, end):
		        if n[i]==n[i-1]:
                            count2+=1
                           # print(n)
                            break#manage test statistic
		    #print (n)
		    COUNT +=1
		else:
		    i=begin
		    for num in range(begin,end):
		        n[num],n[i]=n[i],n[num]
		        perm(n,begin+1,end)
		        n[num],n[i]=n[i],n[num]
	perm(history, 0, len(history))
	p_value=float(count2)/float(COUNT)
	print('the p value is: ')
	print(p_value)
	return p_value

history=[1,0,0,1]
q1(history)
#perm(n,0,len(n))
#print (COUNT)
#print(count2)


from typing import List, Tuple

def create_adjacent_list(edges):
    adjacent_list = dict()
    for edge in edges:
        if adjacent_list.get(edge[0]):
            adjacent_list[edge[0]].append(edge[1])
        else:
            adjacent_list[edge[0]] = [edge[1]]
    return adjacent_list

def solution(n: int, m: int, edges: List[Tuple[int, int]]):
    adjacents_list = create_adjacent_list(edges)
    for i in range(1, n + 1):
        temp = [0] * n
        if adjacents_list.get(i):
            vertex = adjacents_list.get(i)
            for v in vertex:
                temp[v - 1] = 1
        print(' '.join(map(str, temp)))

def input_data():
    n, m = map(int, input().strip().split())
    rows = m
    edges = list()
    while rows:
        edges.append(tuple(map(int, input().strip().split())))
        rows -= 1
    return n, m, edges

if __name__ == '__main__':
    solution(*input_data())

"""
5 3
1 3
2 3
5 2

"""


from Word2Vec.Word2VecGenerator import Word2VecGenerator
import glob
from JsonParse.JsonParser import JsonParser
import json as Json

class TrainingComponentGenerator:
    __largest_n_words = 0
    __astNode2Vec_size = 0
    __number_of_vector_code2vec = 0

    def __init__(self, astNode2Vec_size, number_of_vector_code2vec):
        self.__astNode2Vec_size = astNode2Vec_size
        self.__number_of_vector_code2vec = number_of_vector_code2vec

    def generateTrainingComponent(self, dataFolderPath):
        dataset = []
        for file in glob.glob(dataFolderPath):
            dataset.append(file)

        commits = list()

        parser = JsonParser()
        for data in dataset:
            json = parser.openJson(data)
            commitData = json
            commits.extend(commitData)

        astSentences = list()
        sourceCodeSentences = list()
        astNodeDict = list()

        for commit in commits:
            self.__collectWord2VecData(commit, sourceCodeSentences, astSentences, astNodeDict)
        self.__word2vecModelGenerate(sourceCodeSentences, astSentences)
        astNodeDictSet = set(astNodeDict)  # convert it as set data type.
        astNodeDict = list(astNodeDictSet)
        jsonString = Json.dumps(astNodeDict)
        with open('Outcome/Models/AstNodeDictionary.json', 'w') as f:
            f.write(jsonString)

        print("Training Components are built")

    def __collectWord2VecData(self, commit, sourceCodeSentences, astSentences, astNodeDict):
        tasks = commit['tasks']
        commitAstNodeDic = commit['astNodeDic']
        astNodeDict.extend(commitAstNodeDic)
        for task in tasks:
            taskElementTreeSet = task['taskElementTreeSet']
            for taskElement in taskElementTreeSet:
                astNodeSentence = taskElement['astNodeSentence']
                astNodeSenAsList = self.__stringToList(astNodeSentence)
                astSentences.append(astNodeSenAsList)

                sourceCode = taskElement['sourceCode']
                sourceCodeAsList = self.__tokenizedCodes(sourceCode)
                sourceCodeSentences.append(sourceCodeAsList)
                if (self.__largest_n_words < len(sourceCodeAsList)):
                    self.__largest_n_words = len(sourceCodeAsList)

    def __word2vecModelGenerate(self, sourceCodeSentences, astSentences):
        # CODE2VEC
        code2Vec = Word2VecGenerator()
        code2Vec.generateModel(sourceCodeSentences, vector_size=self.__number_of_vector_code2vec, window=4, min_count=1,
                               Type='CodeType')
        print("Code2Vec is generated")
        # AST2Vec
        astNode2Vec = Word2VecGenerator()
        astNode2Vec.generateModel(astSentences, vector_size=self.__astNode2Vec_size, window=2, min_count=1,
                                  Type="AstType")
        print("AST2Vec is generated")
        return astNode2Vec, code2Vec

    def __stringToList(self, string):
        listRes = list(string.split(" "))
        return listRes

    def __tokenizedCodes(self, sourceCode):
        sourceCodeAsList = self.__stringToList(sourceCode)
        sourceCodeAsList = [x for x in sourceCodeAsList if x != '']
        return sourceCodeAsList

    def getMaximumNumberOfWord(self):
        return self.__largest_n_words


from project.Util.EMFAttributes import EMFAttributes
from project.Util.finalWrite import finalWrite

class FileInput:
    filePath = ''

    def readFile(self):
        while True:
            path = '/Users/shubhamjain/CS562/project/examples/example5'
            # path += input('Input the File Name with its path\n')
            try:
                file = open(path, "r")
                if file:
                    break
            except (Exception, FileExistsError) as error:
                print("Error while fetching data from file", error)
        return file

    def InputFile(self):
        print("Input File")
        attr = EMFAttributes()
        file = self.readFile()
        selectAttributes = []
        if file.readline().lower().__contains__('select'):
            selectAttributes =''
            selectAttributes += file.readline()
            selectAttributes = selectAttributes.strip().replace('  ',' ').split(',')

            for idx,selectAtt in enumerate(selectAttributes):
                selectAttributes[idx] = selectAtt.replace(',', '').replace(' ', '')
                if not selectAttributes[idx].isalpha():
                    selectAttributes[idx]  =  selectAtt.replace(',','').replace(' ','')
                    if not selectAttributes[idx].__contains__('_'):
                        print("You got trouble in select Attribute", selectAtt)

        n = 0
        if file.readline().lower().__contains__('variable'):
            n = int(file.readline().replace(' ',''))
        groupAttributes = []
        if file.readline().lower().__contains__('attributes'):
            groupAttributes = ''
            groupAttributes += file.readline()
            groupAttributes = groupAttributes.strip().replace(',', ' ').replace('  ', ' ').split(' ')
            for idx, groupAtt in enumerate(groupAttributes):
                if not groupAtt.isalpha():
                    groupAttributes[idx] = groupAtt.replace(',', '').replace(' ', '')
                    if not groupAttributes[idx].__contains__('_'):
                        print("You got trouble in group Attribute", groupAtt)
            # print(groupAttributes)
        f_vect = []
        if file.readline().lower().__contains__('vect'):
            f_vect = ''
            f_vect += file.readline()
            f_vect = f_vect.strip().replace(',', ' ').replace('  ', ' ').split(' ')
            for idx, vect in enumerate(f_vect):
                if not vect.isalpha():
                    f_vect[idx] = vect.replace(',', '').replace(' ', '')
                    if not f_vect[idx].__contains__('_'):
                        print("You got trouble in f-vect Attribute", vect)
        select = []

        if file.readline().lower().__contains__('select'):
            while True:
                conditions = []
                temp = file.readline()[:-1]
                if temp.lower().__contains__('having'):
                    break
                temp = temp.split('and')
                for cons in temp:
                    conditions.append(cons.strip())
                # print(conditions)
                select.append(conditions)
        # print(select)
        having = []
        if temp.lower().__contains__('having'):

            temp = file.readline()
            if not temp.lower().__contains__('where'):
                having = ''
                having += temp
                having = having.split('and')
                having = [val.strip() for val in having]
        # print(having)
        where = ''
        if temp.lower().__contains__('where'):
            where = file.readline().strip()

        attr.emfAttributes(selectAttributes, n, groupAttributes, f_vect, select, having, where)
        # print(attr.f_Vect)

        return attr

    def output(self, attributes, manage):

        file = open('/Users/shubhamjain/CS562/project/output/output.py', 'w+')
        final_write = finalWrite()
        final_write.setFileName(file.name)
        imports = ['from configparser import RawConfigParser','import psycopg2']
        final_write.setImports(imports)
        final_write.setStructDB(manage.getStructDB())
        final_write.setAttributes(attributes)
        final_write.outputFile(manage)
        for write in final_write.returns:
            file.write(write)
        # print()
        return file.name

#
# fileInput = FileInput()
# fileInput.InputFile()


import sys
import os
import ctypes
from ctypes import (
    c_double,
    c_int,
    c_float,
    c_char_p,
    c_int32,
    c_uint32,
    c_void_p,
    c_bool,
    POINTER,
    _Pointer,  # type: ignore
    Structure,
    Array,
    c_uint8,
    c_size_t,
)
import pathlib
from typing import List, Union

# Load the library
def _load_shared_library(lib_base_name: str):
    # Construct the paths to the possible shared library names
    _base_path = pathlib.Path(__file__).parent.resolve()
    # Searching for the library in the current directory under the name "libllama" (default name
    # for llamacpp) and "llama" (default name for this repo)
    _lib_paths: List[pathlib.Path] = []
    # Determine the file extension based on the platform
    if sys.platform.startswith("linux"):
        _lib_paths += [
            _base_path / f"lib{lib_base_name}.so",
        ]
    elif sys.platform == "darwin":
        _lib_paths += [
            _base_path / f"lib{lib_base_name}.so",
            _base_path / f"lib{lib_base_name}.dylib",
        ]
    elif sys.platform == "win32":
        _lib_paths += [
            _base_path / f"{lib_base_name}.dll",
        ]
    else:
        raise RuntimeError("Unsupported platform")

    cdll_args = dict()  # type: ignore
    # Add the library directory to the DLL search path on Windows (if needed)

    # Try to load the shared library, handling potential errors
    for _lib_path in _lib_paths:
        print("_lib_path = ", _lib_path)
        if _lib_path.exists():
            try:
                return ctypes.CDLL(str(_lib_path), **cdll_args)
            except Exception as e:
                raise RuntimeError(f"Failed to load shared library '{_lib_path}': {e}")

    raise FileNotFoundError(
        f"Shared library with base name '{lib_base_name}' not found"
    )

# Specify the base name of the shared library to load
_lib_base_name = "model"

# Load the library
_lib = _load_shared_library(_lib_base_name)

# LLAMA_API struct llama_context_params llama_context_default_params();
def inference(argv: c_char_p):
    return _lib.inference(argv)

#_lib.inference.argtypes = [c_int, c_char_p]
_lib.inference.restype = c_char_p

if __name__ == "__main__":
    inference(bytes( "stories15M.bin", encoding = 'utf-8'))


################ Henri Lahousse ################
# voice assistant
# 05/31/2022

# libraries
import struct
import pyaudio
import pvporcupine        # for wakeword
import pvrhino            # for situations

porcupine = None
pa = None
audio_stream = None
rhino = None

# documentation picovoice https://picovoice.ai/docs/
# create model wakeword   https://console.picovoice.ai/ppn
# create model situation  https://console.picovoice.ai/rhn

access_key = 'ENTER_KEY'          # find on picovoice website  https://console.picovoice.ai/access_key // my_key 0nevFcYH3LlyYTajYWkG44d+vLWdm5Njxe8tr6xNrj/Kn9/m2qOjeg==

def voice_ass():
    porcupine = pvporcupine.create(
        access_key=access_key,
        keyword_paths=['ENTER_PATH']  # download model from website and extract file for wakeword detection  // my_path /home/pi/Downloads/wakeword.ppn
    )

    # setup
    def setup(path):
        rhino = pvrhino.create(
            access_key=access_key,
            context_path=path)
        return rhino

    rhino_drive = setup('ENTER_PATH')               # download model from website and extract for situation recognission // /home/pi/Downloads/drive.rhn
    rhino_roof = setup('ENTER_PATH')                # =  // /home/pi/Downloads/roof.rhn
    rhino_smartlights = setup('ENTER_PATH')         # = // /home/pi/Downloads/smartlights.rhn

    pa = pyaudio.PyAudio()

    # prepare audio for processing
    audio_stream = pa.open(
        rate=porcupine.sample_rate,
        channels=1,
        format=pyaudio.paInt16,
        input=True,
        frames_per_buffer=porcupine.frame_length)

    # prepare audio for processing
    def audio(rhino):
        audio_stream_rhn = pa.open(
            rate=rhino.sample_rate,
            channels=1,
            format=pyaudio.paInt16,
            input=True,
            frames_per_buffer=rhino.frame_length)
        return audio_stream_rhn

    audio_sm = audio(rhino_smartlights)
    audio_rf = audio(rhino_roof)
    audio_dr = audio(rhino_drive)

    while True:
        pcm = audio_stream.read(porcupine.frame_length)
        pcm = struct.unpack_from("h" * porcupine.frame_length, pcm)

        keyword_index = porcupine.process(pcm)

        # finalizing audio
        def fin(aud, rhino):
            rh = aud.read(rhino.frame_length)
            rh = struct.unpack_from("h" * rhino.frame_length, rh)
            is_finalized = rhino.process(rh)
            return is_finalized

        is_fin_sm = fin(audio_sm, rhino_smartlights)
        is_fin_rf = fin(audio_rf, rhino_roof)
        is_fin_dr = fin(audio_dr, rhino_drive)

        # results, get the understood situation returned
        def rs(is_fin, rhino):
            if is_fin:
                inference = rhino.get_inference()  # if if_fin is True we get the inference
                if inference.is_understood:  # use intent and slots if it understands
                    intent = inference.intent  # intent is a string
                    slots = inference.slots  # slots is a dictionary
                    return intent, slots

        # returns wakeword
        if keyword_index == 0:
            return 1

        rs(is_fin_sm, rhino_smartlights)
        rs(is_fin_rf, rhino_roof)
        rs(is_fin_dr, rhino_drive)

    porcupine.delete()
    rhino.delete()


import serial
import serial
from time import sleep
import threading
import time
# sudo chmod 666 /dev/ttyACM0
device_port = "/dev/ttyACM0"
from multiprocessing.pool import ThreadPool
import settings

class uwb_data(threading.Thread):
    def __init__(self,file_name,device_port):
        threading.Thread.__init__(self)
        self.file_name = file_name
        self.serial = serial.Serial(device_port)
        self.running = True
        self.myval = []

    def create_csv_file(self):
        self.f = open(self.file_name, 'w+')
        self.f.write("timestamp,x,y,z \n")
        sleep(1)

    def store_uwb_data(self):
        val = str(self.serial.readline().decode().strip(' \r\n'))
        if val.startswith('+DPOS:'):
            val = val.strip('+DPOS:')
            val = val.split(',')
            self.myval =  [int(float(val[2])),int(float(val[3]))]

    def get_uwb_data(self):
        return self.myval

    def run(self):
        while self.running:
            self.store_uwb_data()
            settings.myList = self.get_uwb_data()

    def terminate(self):
         """clean stop"""
         self.running = False

if __name__ == "__main__":
    uwb_get_way = uwb_data('IDRdata.csv',"/dev/ttyACM0")
    uwb_get_way.start()
    pool = ThreadPool(processes=1)
    try:
        while True:
            async_result = pool.apply_async(uwb_get_way.get_uwb_data)
            return_val = async_result.get()
            print(settings.myList)

    except (KeyboardInterrupt, SystemExit):
        uwb_get_way.terminate()
    print("killed")


# 도시 분할 계획

import sys
input = sys.stdin.readline

def find_parent(parent, x):
    if parent[x] != x:
        parent[x] = find_parent(parent, parent[x])
    return parent[x]

def union_parent(parent, a, b):
    a = find_parent(parent, a)
    b = find_parent(parent, b)
    # 더 작은 노드를 루트 노드로 설정
    if a < b:
        parent[b] = a
    else:
        parent[a] = b

# 입력 및 초기화
num_node, num_edge = map(int, input().split())

parent_table = [0] * (num_node+ 1)  # 부모 테이블
edges = []                          # 간선 리스트
result_edges = []                          # 최종 비용
for i in range(1, num_node + 1):
    parent_table[i] = i

for _ in range(num_edge):
    a, b, cost = map(int, input().split())
    edges.append((cost, a, b))

# 간선을 비용 오름차순으로 정렬
edges.sort()

# 크루스칼 알고리즘
for edge in edges:
    cost, a, b = edge
    # 사이클이 발생하지 않는 경우
    if find_parent(parent_table, a) != find_parent(parent_table, b):
        union_parent(parent_table, a, b)
        # 신장 트리의 간선으로 선택(최종 비용에 포함)
        result_edges.append(cost)

# 마을을 2개로 분할하되 유지비의 비용을 최소로 해야하므로 가장 큰 유지비를 하나 제외하고 출력한다.
print(sum(result_edges) - max(result_edges))


#!/usr/bin/python3
"""
function that prints a text with 2 new lines after each of\
these characters: ., ? and :
"""

def text_indentation(text):
    """
    text_indentation -- print a text with 2 new lines after each of\
    these characters
    text -- recibe the Text
    """

    if type(text) is not str:
        raise TypeError("text must be a string")
    """
    removes space only when it finds a matching character
    """
    tok = 0
    for i in text:
        """
        if it found a character it removes the space continues
        """
        if tok == 1 and i is ' ':
            print('', end='')
            tok = 0
            continue
        if i is '.' or i is '?' or i is ':':
            print("{}\n".format(i))
            tok = 1
        else:
            print(i, end='')
            tok = 0


# To add a new cell, type '#%%'
# To add a new markdown cell, type '#%% [markdown]'

#%% [markdown]
#
# # HW06
# ## By: xxx
# ### Date: xxxxxxx
#

#%% [markdown]
# Let us improve our Stock exercise and grade conversion exercise with Pandas now.
#

#%%
import dm6103 as dm
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#%%
# Load the data frame from api
dfaapl = dm.api_dsLand('AAPL_daily', 'date')
print("\nReady to continue.")

dm.dfChk(dfaapl)

# What are the variables in the df?
# What are the data types for these variables?

#%%
# You can access pd dataframe columns using the dot notation as well as using column names
print(dfaapl.price, '\n')
# same as
print(dfaapl['price'])

#%%
# Step 1
# Create the Stock class
#

class Stock:
  """
  Stock class of a publicly traded stock on a major market
  """
  import dm6103 as dm
  import os
  import numpy as np
  import pandas as pd
  def __init__(self, symbol, name, init_tbname) :
    """
    :param symbol: stock symbol
    :param name: company name
    :param init_tbname: the initial table name on our DSLand API with historical data. Date is index, with eod price and vol as columns.
    """
    # note that the complete list of properties/attributes below has more than items than
    # the numnber of arguments of the constructor. That's perfectly fine.
    # Some property values are to be assigned later after instantiation.
    self.symbol = symbol.upper()
    self.name = name
    self.data = self.import_history(init_tbname) # this is a pandas df, make sure import_history() returns a pd dataframe
    # the pandas df self.data will have columns price, volume, delta1, delta2, and index is date
    self.init_delta1() # Calculate the daily change values from stock price itself, append to df
    self.init_delta2() # Calculate the daily values second derivative, append to df
    self.firstdate = self.data.index[-1]
    self.lastdate = self.data.index[0]

  def import_history(self, tbname):
    """
    import stock history from api_dsLand, with colunms date, eod_price, volume
    """
    return dm.api_dsLand( tbname, 'date' )  # use date as index

  def init_delta1(self):
    """
    compute the daily change from price_eod, append to data as new column as delta1
    """
    # notice that:
    # aapl['price'] returns a pandas series
    # aapl[['price']] returns a pandas dataframe
    # aapl['price'].values returns a numpy array of the values only
    self.data['delta1'] = 0  # initialize a new column with 0s
    self.data['delta1'] = self.data['price'][0:-1] - self.data.price.values[1:]   # self.data['price'] is same as self.price for df
    # the first term on the right is the full pd series with index attached. Second one is a simple numpy array without the date
    # index. That way, the broadcasting will not try to match the indices/indexes on the two df
    return # you can choose to return self

  def init_delta2(self):
    """
    compute the daily change for the entire list of delta1, essentially the second derivatives for price_eod
    """
    # essentially the same function as init_delta1.
    self.data['delta2'] = 0  # initialize a new column with 0s
    self.data['delta2'] = self.data.delta1[0:-1] - self.data.delta1.values[1:]   # self.data['price'] is same as self.price for df
    return # you can choose to return self

  def add_newday(self, newdate, newprice, newvolume):
    """
    add a new data point at the beginning of data df
    """
    # Make plans
    # insert a new row to self.data with
    # (date, price, volume, delta1, delta2) to the pandas df,
    # and also should update self.lastdate
    #
    # update self.lastdate
    self.lastdate = newdate

    # get ready a new row, in the form of a pandas dataframe.
    # Pandas dataframe does not have an insert function. The usual method is to use .append()
    # and .append() is most efficient to append a df to another df of the same columns.
    newRow = self.setNewRow(newdate, newprice, newvolume) # we do this quite a lot: assume it's done already, then implement it later.
    # need this function setNewRow() to return a dataframe

    self.data = newRow.append(self.data) # this will put the new row on top, and push self.data after the new data

    return self

  def setNewRow(self, newdate, newprice, newvolume):
    # first create a copy of the dataframe with a dummy first row
    # the correct newdate is set as the index value for this 1-row dataframe
    df = pd.DataFrame( dict( {'date': [ newdate ]}, **{ key: [0] for key in self.data.columns } ) )
    df.set_index( 'date', inplace=True )

    # df.index = [ newdate ] # this is already set properly above.
    df.price[0] = newprice
    df.volume[0] = newvolume
    df.delta1[0] = newprice - self.data.price[0]
    df.delta2[0] = df.delta1[0] - self.data.delta1[0]
    return df

  def nday_change_percent(self,n):
    """
    calculate the percentage change in the last n days, returning a percentage between 0 and 100
    """
    change = self.data.price[0]-self.data.price[n]
    percent = 100*change/self.data.price[n]
    print(self.symbol,": Percent change in",n,"days is {0:.2f}".format(percent))
    return percent

  def nday_max_price(self,n):
    """
    find the highest price within the last n days
    """
    return self.data.price[0:n].max()

  def nday_min_price(self,n):
    """
    find the lowest price within the last n days
    """
    return self.data.price[0:n].min()

#%%
# Try these:
filename = 'AAPL_daily'
aapl = Stock('AAPL','Apple Inc',filename)
aapl.data.head()
aapl.data.tail()
aapl.nday_max_price(333) # record the answer here
aapl.nday_min_price(500) # record the answer here
aapl.nday_change_percent(500)  # record the answer here

aapl.add_newday('9/13/19',218.42,12345678)
aapl.data.head()

# %%


# -*- encoding: utf-8 -*-
import logging
import os
import time

import numpy as np
import openpyxl
import pandas as pd
import xlrd
# 导入PyQt5模块
from PySide2.QtCore import *
from PySide2.QtWidgets import *

from dataImportModel import Ui_Form as dataImportFormEngine
from widgets import kwargs_to_str
from lib.comm import set_var, run_command

# 导入matlab加载模块

# 定义日志输出格式
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

class ImportDialog(QDialog):
    signal_data_change = Signal(str, dict, str, str, str, str, str)  # 自定义信号，用于传递文件路径
    extension_lib = None

    def __init__(self, parent=None):
        super(ImportDialog, self).__init__(parent)
        self.current_dataset: pd.DataFrame = None
        self.import_message = {"isError": False, "warningMessage": []}
        self.separator_char = [",", ";", "\\s", "\\t"]
        self.encode_type = ["utf8", "gb2312", "gbk", "ascii"]

    def importDatasetPreview(self):
        pass

    def getImportParam(self):
        pass

    def importDatasetReload(self):
        pass

    def updateTableView(self):
        pass

    def open_file(self, path: str):
        assert os.path.exists(path)
        self.lineEdit_filePath.setText(path)
        self.previewButton()

    def openfile(self):
        """
            选择文件，需要支持三种场景：
            （1）点击 “浏览” 按钮
            （2）点击 “预览” 按钮
        """
        path = self.lineEdit_filePath.text()
        self.import_param.update(choosefile=False)
        if not path:
            # 先判断传入的 path 是否有内容，
            path, openfile_type = QFileDialog.getOpenFileName(self, '选择文件', self.get_work_dir(),
                                                              "文件类型({})".format(self.file_types))
            self.lineEdit_filePath.setText(path)
        if path:
            # 如果没有选择文件就关闭窗口，这时候path还是没有路径，datasetName 则清空
            temp_name = (os.path.split(self.lineEdit_filePath.text())[1]).split(".")[0:-1]
            # 获取文件名称，并将文件名称作为导入的变量名称，如果文件名称为空，则使用 temp 作为变量名称
            dataset_name = "temp" if temp_name == [""] else ".".join(temp_name)
            self.lineEdit_datasetName.setText(dataset_name)
        else:
            self.lineEdit_filePath.setText(self.import_param["filepath"])

    def chooseFileButton(self):
        """选择文件按钮"""
        self.lineEdit_filePath.setText("")
        self.previewButton()

    def previewButton(self):
        """预览按钮"""
        self.import_param.update(ispreview=True)
        self.openfile()
        self.getImportParam()
        if self.import_message["isError"]:
            self.showWarningMessage()
        else:
            if self.lineEdit_filePath.text():
                self.importDatasetLoad()
        self.updateTableView()

    def importDatasetButton(self):
        """对发送钱的数据验证"""
        self.import_param.update(ispreview=False)
        self.getImportParam()
        if self.import_message["isError"]:
            self.showWarningMessage()
            return

        # if self.import_param["filepath"] == "" or len(self.current_dataset) == 0:
        if len(self.current_dataset) == 0:
            self.showWarningMessage(info="导入失败！\n提示：请提供正确数据集")
            return

        var_name_check = self.updateDatasetVarname()
        if var_name_check:
            import sys
            t0 = time.time()
            self.importDatasetLoad()
            self.sendDataset()
            t1 = time.time()
            logger.info("导入数据集所用时间: {t} s 大小 {m} MB".format(
                t=round(t1 - t0, 2), m=round(sys.getsizeof(self.current_dataset) / 1024, 2)
            ))
            self.current_dataset = None

    def importDatasetLoad(self):
        """获取数据并做检验"""
        error = ""
        self.import_param.update(status=False)
        try:
            self.importDatasetReload()
            self.import_param.update(status=True)

        except UnicodeDecodeError as e:
            encodetype = self.import_param["param"]["encoding"]
            self.updateWarningMessage(info="指定的编码方式“{}”无法解码要打开的文件,请尝试其他编码方式".format(encodetype))
            error = str(e)

        except MemoryError as e:
            self.updateWarningMessage(info="文件过大，超过内存上限，导入失败！")
            error = str(e)

        except Exception as e:
            self.updateWarningMessage(info="导入失败，错误详情：\n{}".format(str(e)))
            error = str(e)

        if self.import_message["isError"]:
            self.showWarningMessage()
        return (error)

    def getDatasetInfo(self, varname=""):
        """
            获取变量的名称、数据结构等信息
            目前暂不支持保留用户重新配置的字段数据类型方案
            varname = 变量统一命名
        """
        self.import_param.update(varname={}, dtypes={})
        for k in self.current_dataset:
            self.import_param["varname"][k] = varname if varname else k
            if type(self.current_dataset[k]) == pd.DataFrame:
                self.import_param["dtypes"][k] = self.current_dataset[k].dtypes
            else:
                self.import_param["dtypes"][k] = type(self.current_dataset[k])

    def updateDatasetVarname(self):
        """
            更新导入数据集时候的名称
            TODO: 重置数据集名称
            考虑到未来导入数据集时候需要重命名数据集的名称，可能会存在这几类场景：
            （1）导入后的变量名称更新
                【1】一个文件一个单变量（页面）导入
                【2】一个文件多变量（页面）导入，导入后可能以一个字典导入，或是多个变量名称，如果数据结构都一致情况下，
                    可能还有合并成一个变量导入
            （2）导入时候使用什么类型数据结构导入，数据框，字典，字符，列表等
            （3）导入时候的数据结构的调整
            （4）导入时候变量是否有存在，如果有存在，则需要提醒用户修改冲突的变量名称
            因此考虑将这部分独立出来进行处理。
        """
        # 使用当前“数据集名” / “页面” 的名称
        self.newdatasetname = {"varname": {}}
        e = self.import_param["datasetname"]
        while True:
            var_name, ok = QInputDialog.getText(self, "变量名", "输入新的变量名称:", QLineEdit.Normal, e)
            if ok:
                if len(var_name) == 0:
                    QMessageBox.warning(self, "提示", "请输入变量名称！")
                    continue
                elif self.extension_lib.Data.var_exists(var_name):
                    # 在变量名称冲突情况下，允许用户判断是否覆盖变量名称
                    isCover = QMessageBox().question(None, "提示", "变量 {} 已经存在，是否覆盖？".format(var_name),
                                                     QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
                    if isCover == QMessageBox.Yes:
                        break
                    else:
                        continue
                elif not var_name.isidentifier():
                    QMessageBox.warning(self, '提示', '变量名无效\n提示：\n1、不要以数字开头;\n2、不要包含除下划线外的所有符号。')
                else:
                    break
            else:
                ok = False
                self.import_param.update(ispreview=True, status=True)
                break
        if ok:
            self.newdatasetname["varname"][e] = var_name
            # if self.import_param["ismerge"]:
            #    self.newdatasetname["datasetname"] = var_name
            # self.import_param["datasetname"] = var_name
            # else:
            #    self.newdatasetname["varname"][e] = var_name
            # self.import_param["varname"][e] = var_name
        return (ok)

    def sendDataset(self):
        """
        这个方法与具体导入sas，spss还是excel数据都是无关的。
        其实意思就是把pandas数据加入到工作空间中。
        """
        if self.import_param["status"]:
            # if self.import_param["ismerge"]:
            #    set_var(self.newdatasetname["datasetname"], self.current_dataset)
            # else:
            for name_i, var_i in self.newdatasetname["varname"].items():
                set_var(var_i, self.current_dataset[name_i])  # 将数据导入工作空间
            QMessageBox.information(self, "{}导入结果".format(""), "数据导入完成！", QMessageBox.Yes)
            self.close()

    def clearImportParam(self):
        """重置数据集"""
        self.current_dataset = {}
        self.import_message = {"isError": False, "warningMessage": []}
        self.import_param = {
            "datasetname": "",  # 数据集名称
            "varname": {},  # 导入的变量名称，dict，用于后续存放更改变量名称后的结果
            "filepath": "",  # 文件路径
            "hasheader": True,  # 首行是否为列名称
            "dtypes": {},  # 字段数据类型，dict，用于后续存放更改数据类型后的结果
            "status": False,  # 导入结果状态：True = 导入成功，False = 导入失败
            "param": {},  # 导入面板上的参数，dict
            "ispreview": True,  # 是否预览
            "ismerge": False  # 多变量数据集是否合并成字典导入
        }

    def get_work_dir(self) -> str:
        """获取工作路径"""
        return self.extension_lib.Program.get_work_dir()

    def center(self):
        """将窗口置于中心"""
        qr = self.frameGeometry()
        cp = QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def keyPressEvent(self, e):
        """按键盘Escape退出当前窗口"""
        if e.key() == Qt.Key_Escape:
            button = QMessageBox.question(self, "Question", "是否退出当前窗口？",
                                          QMessageBox.Ok | QMessageBox.Cancel, QMessageBox.Ok)

            if button == QMessageBox.Ok:
                self.close()

    def showWarningMessage(self, info=""):
        """显示异常信息"""
        info = info if info else self.import_message["warningMessage"][0]
        if info:
            QMessageBox.warning(self, '警告：', info)
            logging.info("获取数据警告：\n" + info)

    def updateWarningMessage(self, info="", new=True):
        """更新导入状态"""
        if new:
            self.import_message["isError"] = True
            self.import_message["warningMessage"].append(info)
        else:
            self.import_message["isError"] = False
            self.import_message["warningMessage"] = []

    def checkFilePath(self, path):
        '''检查输入的文件路径是否合法'''
        if path:
            if not os.path.exists(path):
                self.updateWarningMessage(info="数据集路径不存在，\n请重新输入数据集路径！")

            if os.path.split(path)[-1].split(".")[-1].lower() not in self.file_types:
                self.updateWarningMessage(
                    info="数据文件格式有错：\n仅支持({})类型文件，\n请重新输入数据集路径！".format(self.file_types)
                )
        return (path)

    def checkRowsNumber(self, rows, types):
        '''检查行数是否为正整数或“全部”'''
        typesDict = {
            "limitRows": "“限定行数”必须是大于等于0的整数或“全部”",
            "skipRows": "“跳过行数”必须是大于等于0的整数"
        }
        if rows == "全部":
            row_number = None
        elif rows.isdigit():
            row_number = int(rows)
        else:
            row_number = 0
            self.updateWarningMessage(info="{}\n请重新输入！".format(typesDict[types]))
        if self.import_param["ispreview"] and types == "limitRows":
            # 判断是否为预览，或是限制行数
            row_number = min([100, row_number if row_number else 101])
        return (row_number)

    def headerAsColumns(self, data):
        """首行为列名"""
        colnames = pd.DataFrame([data.columns], index=[0], columns=data.columns.tolist())
        data.index += 1
        data = data.append(colnames, ignore_index=False)
        data.sort_index(inplace=True)
        data.columns = ["C" + str(i + 1) for i in range(data.shape[1])]
        return (data)

    def datasetUpdate(self, data, skiprow, limitrow):
        """对数据集的规模进行处理"""
        data = data[data.index >= skiprow]  # 跳过行数
        if limitrow:
            limitrows = min(data.shape[0], limitrow)
            data = data.head(limitrows)
        return (data)

    def showDatasetPreview(self, data, header=True):
        """导入的数据集可视化"""
        if not header:
            # 首行不为列名情况下的处理
            data = self.headerAsColumns(data)

        table_rows, table_colunms = data.head(100).shape
        table_header = [str(col_i) for col_i in data.columns.tolist()]

        self.tableWidget_previewData.setColumnCount(table_colunms)
        self.tableWidget_previewData.setRowCount(table_rows)
        self.tableWidget_previewData.setHorizontalHeaderLabels(table_header)

        # 数据预览窗口
        for i in range(table_rows):
            row_values = data.iloc[i].tolist()
            for j, element in enumerate(row_values):
                newItem = QTableWidgetItem(str(element))
                newItem.setTextAlignment(Qt.AlignHCenter | Qt.AlignVCenter)
                self.tableWidget_previewData.setItem(i, j, newItem)

    def updateDatasetNameLine(self, tag):
        """更新数据集显示标签"""
        new_datasetname = self.import_param["datasetname"] if tag == "（全部导入）" else tag
        self.lineEdit_datasetName.setText(new_datasetname)

    def clearPreviewDataTableWidget(self):
        """清理表格组件内容"""
        self.tableWidget_previewData.clear()
        self.showDatasetPreview(data=pd.DataFrame([]))

    def showHelp(self):
        from packages.pm_helpLinkEngine import helpLinkEngine as h
        h.helpLink.openHelp("dataio_sample_showhelp")

    # 数据库相关的方法

    def checkTextNotNull(self, dicts):
        """检验输入内容是否为空"""
        db_dict = {"host": "IP地址", "user": "用户名称", "passwd": "密码", "db": "数据库名称", "password": "密码",
                   "port": "IP端口", "charset": "数据类型", "table": "表格名称", "schema": "数据模式", "database": "数据库名称",
                   "server_name": "服务名称"}
        for k, v in dicts.items():
            if not v:
                self.updateWarningMessage(info="‘{tag}’不能为空，请重新输入！".format(tag=db_dict[k]))

    def updateDatabaseConnectStatusLabel(self, e=""):
        tag = {"label": "连接成功", "color": "color: blue;"}
        if e:
            tag.update(label='连接失败:' + e, color="color: rgb(255, 0, 0);")
        self.label_test.setHidden(False)
        self.label_test.setText(tag["label"])
        self.label_test.setStyleSheet(tag["color"])

    def dbConnectTestButton(self):
        """检查数据库连接是否有效"""
        self.import_param.update(ispreview=True)
        self.getImportParam()
        if self.import_message["isError"]:
            self.showWarningMessage()
            return

        error = self.importDatasetLoad()
        self.updateDatabaseConnectStatusLabel(error)

    def dbDatasetImportButton(self):
        """导入数据按钮"""
        self.import_param.update(ispreview=False)
        self.getImportParam()
        if self.import_message["isError"]:
            self.showWarningMessage()
            return
        var_name_check = self.updateDatasetVarname()
        if var_name_check:
            import sys
            t0 = time.time()
            error = self.importDatasetLoad()
            self.updateDatabaseConnectStatusLabel(error)
            self.sendDataset()
            t1 = time.time()
            logger.info("导入数据集所用时间: {t} s 大小 {m} MB".format(
                t=round(t1 - t0, 2), m=round(sys.getsizeof(self.current_dataset) / 1024, 2)
            ))
            self.current_dataset = None

    def getCurFetchData(self, cur):
        """获取数据库返回的分页数据"""
        temp = pd.DataFrame([])
        try:
            cur.execute(self.import_param["sql"])
            if cur.description:
                temp = pd.DataFrame(data=list(cur.fetchall()),
                                    columns=list(map(lambda x: x[0], cur.description)))
        except Exception as e:
            self.updateWarningMessage("导入失败，错误详情：\n{}" + str(e))
        return (temp)

    def updateChooseTagName(self, comboBox, tagname=[]):
        """ 加载导入文件变量名称 """
        comboBox.clear()
        if not self.import_param["status"]:
            return
        if not tagname:
            tagname = list(self.current_dataset)
        tagname = ["（全部导入）"] + tagname if len(tagname) > 1 else tagname
        for v in tagname:
            # 更新Excel导入界面中"数据位置"列表
            comboBox.addItem(v)

# 优化完成
class ImportTextForm(ImportDialog, dataImportFormEngine):
    """
    "导入Text"窗口，包含方法：
    （1）getImportParam：获取面板中的配置信息
    （2）importDatasetReload：重新加载文件数据内容
    （3）updateTableView：更新视图呈现数据
    """

    def __init__(self, parent=None):
        self.file_types = "*.csv *.txt *.tsv"
        self.IconPath = ":/resources/icons/txt.svg"
        super().__init__(parent)
        self.setupUi(self)
        self.center()
        self.clearImportParam()
        self.updateUIForm()

    def AddUIFormActivity(self):
        """增加界面中的操作操作响应"""
        self.checkBox_ifColumns.stateChanged.connect(self.updateTableView)  # 选择首行是否为列名
        self.checkBox_asString.stateChanged.connect(self.previewButton)  # 是否以文本形式导入
        self.comboBox_encode.currentTextChanged.connect(self.previewButton)  # 选择编码方式
        self.comboBox_separator.currentTextChanged.connect(self.previewButton)  # 选择分割符号

    def updateUIForm(self):
        """ImportTextForm配置参数部分"""
        separator_char = ["\\n"] + self.separator_char
        self.comboBox_encode = self.updateForm_ComboBox(self.comboBox_encode, self.encode_type)
        self.comboBox_separator = self.updateForm_ComboBox(self.comboBox_separator, separator_char)

        self.horizontalLayoutAddUI(self.checkBox_asString)
        self.horizontalLayoutAddUI(self.checkBox_ifColumns)

        self.verticalLayoutAddUI(self.lineEdit_datasetName, "left")
        self.verticalLayoutAddUI(self.lineEdit_limitRow, "left")
        self.verticalLayoutAddUI(self.lineEdit_skipRow, "left")
        self.verticalLayoutAddUI(self.comboBox_separator, "right")
        self.verticalLayoutAddUI(self.comboBox_encode, "right")
        self.publicUIFormActivity()
        self.AddUIFormActivity()

    def getImportParam(self):
        """
            获取界面中的配置信息
            （1）首行列名（2）数据集名称（3）跳过行数（4）限定行数（5）文件编码（6）分割符号
        """
        self.updateWarningMessage(new=False)
        self.import_param.update(
            datasetname=self.lineEdit_datasetName.text(),
            filepath=self.lineEdit_filePath.text(),
            hasheader=self.checkBox_ifColumns.isChecked(),
            status=False, varname={}, dtypes={},
            asString=self.checkBox_asString.isChecked(),
            param={
                "filepath_or_buffer": self.checkFilePath(self.lineEdit_filePath.text()),
                "engine": "python",
                "header": 'infer' if self.checkBox_ifColumns.isChecked() else None,
                "sep": self.comboBox_separator.currentText(),
                "encoding": self.comboBox_encode.currentText(),
                "nrows": self.checkRowsNumber(self.lineEdit_limitRow.text(), "limitRows"),
                "skiprows": self.checkRowsNumber(self.lineEdit_skipRow.text(), "skipRows")
            }
        )

    def importDatasetReload(self):
        """
            刷新导入的数据
            file_path: 导入路径
        """
        param = self.import_param["param"]
        self.current_dataset = {}
        varname = self.import_param["datasetname"]

        if self.import_param["asString"]:
            with open(file=param["filepath_or_buffer"], encoding=param["encoding"]) as f:
                size = param["nrows"] if param["nrows"] else -1
                temp = f.read(size)
            f.close()
        else:
            temp = pd.read_table(**param)
        # 文本一次只导入一个文件，因此默认变名称即为数据集名称
        self.current_dataset[varname] = temp
        self.getDatasetInfo()
        self.import_param.update(status=True)

    def updateTableView(self):
        """
        刷新预览数据
        """
        # 处理需要呈现的内容
        self.clearPreviewDataTableWidget()

        if not self.import_param["status"]:
            return

        dataset = self.current_dataset[self.import_param["datasetname"]]
        if self.checkBox_asString.isChecked():
            preview_data = pd.DataFrame({"文本": [dataset[:100]]})
            header = True
        else:
            preview_data = dataset.head(100)
            header = self.checkBox_ifColumns.isChecked()

        self.showDatasetPreview(data=preview_data, header=header)

# 优化完成
class ImportCsvForm(ImportDialog, dataImportFormEngine):
    """导入CSV窗口"""

    def __init__(self, parent=None):
        self.IconPath = ":/resources/icons/csv.svg"
        self.file_types = "*.csv"
        super().__init__(parent)
        self.setupUi(self)
        self.center()
        self.clearImportParam()
        self.updateUIForm()

    def AddUIFormActivity(self):
        """增加界面中的操作操作响应"""
        self.checkBox_ifColumns.stateChanged.connect(self.updateTableView)  # 选择首行是否为列名
        self.checkBox_ifColIndex.stateChanged.connect(self.previewButton)  # 首列是否为列名
        self.comboBox_encode.currentTextChanged.connect(self.previewButton)  # 选择编码方式
        self.comboBox_separator.currentTextChanged.connect(self.previewButton)  # 选择分割符号

    def updateUIForm(self):
        """ImportTextForm配置参数部分"""
        self.comboBox_separator = self.updateForm_ComboBox(self.comboBox_separator, self.separator_char)
        self.comboBox_encode = self.updateForm_ComboBox(self.comboBox_encode, self.encode_type)

        self.horizontalLayoutAddUI(self.checkBox_ifColumns)
        self.horizontalLayoutAddUI(self.checkBox_ifColIndex)

        self.verticalLayoutAddUI(self.lineEdit_datasetName, "left")
        self.verticalLayoutAddUI(self.lineEdit_limitRow, "left")
        self.verticalLayoutAddUI(self.lineEdit_skipRow, "left")
        self.verticalLayoutAddUI(self.comboBox_separator, "right")
        self.verticalLayoutAddUI(self.comboBox_encode, "right")
        self.publicUIFormActivity()
        self.AddUIFormActivity()

    def getImportParam(self):
        """
            获取界面中的配置信息
            （1）首行列名（2）数据集名称（3）跳过行数（4）限定行数（5）文件编码（6）分割符号
        """
        self.updateWarningMessage(new=False)
        self.import_param.update(
            datasetname=self.lineEdit_datasetName.text(),
            filepath=self.lineEdit_filePath.text(),
            hasheader=self.checkBox_ifColumns.isChecked(),
            status=False, varname={}, dtypes={},
            param={
                "filepath_or_buffer": self.checkFilePath(self.lineEdit_filePath.text()),
                "engine": "c",
                "header": 'infer' if self.checkBox_ifColumns.isChecked() else None,
                "sep": self.comboBox_separator.currentText(),
                "index_col": 0 if self.checkBox_ifColIndex.isChecked() else None,
                "encoding": self.comboBox_encode.currentText(),
                "nrows": self.checkRowsNumber(self.lineEdit_limitRow.text(), "limitRows"),
                "skiprows": self.checkRowsNumber(self.lineEdit_skipRow.text(), "skipRows")
            }
        )

    def importDatasetReload(self):
        """
            刷新导入的数据
            file_path: 导入路径
        """
        param = self.import_param["param"]
        self.current_dataset = {}
        varname = self.import_param["datasetname"]
        # CSV一次只导入一个文件，因此默认变名称即为数据集名称
        self.current_dataset[varname] = pd.read_csv(**param)
        run_command("", "pd.read_csv(%s)" % kwargs_to_str(param))
        self.getDatasetInfo()
        self.import_param.update(status=True)

    def updateTableView(self):
        """
        刷新预览数据
        """
        # 处理需要呈现的内容
        self.clearPreviewDataTableWidget()

        if not self.import_param["status"]:
            return

        dataset = self.current_dataset[self.import_param["datasetname"]]
        if self.comboBox_separator.currentText() == "(无)":
            preview_data = pd.DataFrame({"文本": [dataset[:100]]})
            header = True
        else:
            preview_data = dataset.head(100)
            header = self.checkBox_ifColumns.isChecked()

        self.showDatasetPreview(data=preview_data, header=header)

# 后续还需要进一步优化方案
class ImportExcelForm(ImportDialog, dataImportFormEngine):
    """打开excel导入窗口"""

    def __init__(self, parent=None):
        super().__init__(parent)
        self.IconPath = ":/resources/icons/excel.svg"
        self.setupUi(self)
        self.center()
        self.clearImportParam()
        self.new_import_filepath = ""
        self.file_types = "*.xls *.xlsx"
        self.sheetsname = []
        self.updateUIForm()

    def AddUIFormActivity(self):
        """增加界面中的操作操作响应"""
        self.checkBox_ifColumns.stateChanged.connect(self.updateTableView)  # 选择首行是否为列名
        self.checkBox_ifColIndex.stateChanged.connect(self.previewButton)  # 首列是否为列名
        self.comboBox_sheetname.currentTextChanged.connect(self.updateTableView)  # 切换页面

    def updateUIForm(self):
        """ImportTextForm配置参数部分"""
        self.horizontalLayoutAddUI(self.checkBox_ifColumns)  # 首行为列名
        self.horizontalLayoutAddUI(self.checkBox_ifColIndex)  # 首列为行名

        self.verticalLayoutAddUI(self.lineEdit_datasetName, "left")  # 数据集名称
        self.verticalLayoutAddUI(self.comboBox_sheetname, "right")  # 页面名称
        self.verticalLayoutAddUI(self.lineEdit_limitRow, "left")  # 限制行数
        self.verticalLayoutAddUI(self.lineEdit_skipRow, "right")  # 跳过行数
        self.publicUIFormActivity()
        self.AddUIFormActivity()

    def getImportParam(self):
        """
            获取Excel里头的页面信息
            （1）首行列名（2）数据集名称（3）跳过行数（4）限定行数（5）文件编码（6）分割符号
        """
        self.updateWarningMessage(new=False)
        # Excel 部分，默认都是全部数据导入后在内存中做处理, 因此 ispreview 都是 False
        self.import_param.update(
            datasetname=self.lineEdit_datasetName.text(),
            filepath=self.lineEdit_filePath.text(),
            hasheader=self.checkBox_ifColumns.isChecked(),
            status=False, varname={}, dtypes={}, loaddataset=False, ismerge=True,
            limitrows=self.checkRowsNumber(self.lineEdit_limitRow.text(), "limitRows"),
            skiprows=self.checkRowsNumber(self.lineEdit_skipRow.text(), "skipRows"),
            param={
                "io": self.checkFilePath(self.lineEdit_filePath.text()),
                "engine": "python",
                "sheet_name": "",
                "header": 'infer' if self.checkBox_ifColumns.isChecked() else None,
                "nrows": None,  # 默认全部加载，在内存中做处理
                "index_col": 0 if self.checkBox_ifColIndex.isChecked() else None,
                "skiprows": 0
            }
        )

        if self.import_message["isError"]:
            return

        if self.new_import_filepath != self.import_param["filepath"]:
            # 当前仅当文件路径发生变化时候进行重载，否则以内存中数据呈现对应变化
            self.import_param.update(loaddataset=True)
            self.LoadSheetname()

    def LoadSheetname(self):
        """预先加载 sheetname 信息"""
        ftype = os.path.split(self.import_param["filepath"])[1].endswith("xls")
        # 获取excel 工作簿中所有的sheet,设置 sheet 名
        if ftype:
            # 针对 xls 格式
            wb = xlrd.open_workbook(self.import_param["filepath"])
            self.sheetsname = wb.sheet_names()
        else:
            # 针对 xlsx 格式
            wb = openpyxl.load_workbook(self.import_param["filepath"], read_only=True)
            self.sheetsname = wb.sheetnames

        # 选择导入引擎
        self.import_param["param"].update(engine='xlrd' if ftype else 'openpyxl')

        # 如果存在多个页面时，需要考虑到将Excel文件中所有页面都导入，因此通过（全部导入）作为标识
        # self.updateChooseTagName(self.comboBox_sheetname, tagname = self.sheetsname)
        self.comboBox_sheetname.clear()
        tagname = ["（全部导入）"] + self.sheetsname if len(self.sheetsname) > 1 else self.sheetsname
        for v in tagname:
            # 更新Excel导入界面中"数据位置"列表
            self.comboBox_sheetname.addItem(v)

    def importDatasetReload(self):
        """
            刷新导入的数据
        """
        if self.import_param["loaddataset"]:
            param = self.import_param["param"]
            self.current_dataset = {}
            for sheet_i in self.sheetsname:
                # 默认都是全部加载后在处理
                param.update(sheet_name=sheet_i)
                self.current_dataset[sheet_i] = pd.read_excel(**param)
                run_command("", "pd.read_excel(%s)" % kwargs_to_str(param))

        if not self.import_param["ispreview"]:
            sheet_ind = self.comboBox_sheetname.currentText()
            if sheet_ind != "（全部导入）":
                self.import_param.update(ismerge=False)
                self.current_dataset = {sheet_ind: self.current_dataset[sheet_ind]}

            for name_i, temp in self.current_dataset.items():
                if not self.import_param["hasheader"]:
                    temp = self.headerAsColumns(temp)
                self.current_dataset[name_i] = self.datasetUpdate(
                    data=temp, limitrow=self.import_param["limitrows"], skiprow=self.import_param["skiprows"]
                )

        self.new_import_filepath = self.import_param["filepath"]
        self.getDatasetInfo()
        self.import_param.update(status=True, loaddataset=False)

    def updateTableView(self):
        """
            刷新预览数据
        """
        # 处理需要呈现的内容
        self.clearPreviewDataTableWidget()

        self.updateDatasetNameLine(tag=self.comboBox_sheetname.currentText())

        if not self.import_param["status"]:
            self.showDatasetPreview(data=pd.DataFrame([]))
            return

        # 首行是否为列名
        header = self.checkBox_ifColumns.isChecked()
        # 获取当前选择的表格信息
        load_sheet = self.comboBox_sheetname.currentText()
        l = self.import_param["limitrows"]
        s = self.import_param["skiprows"]

        if load_sheet == "（全部导入）":
            temp = []
            for name_i, data_i in self.current_dataset.items():
                if not header:
                    data_i = self.headerAsColumns(data_i)
                data_i = self.datasetUpdate(data_i, limitrow=l, skiprow=s)
                row_i, col_i = data_i.shape
                temp.append([name_i, row_i, col_i, data_i.columns.tolist()])
            header = True  # 避免呈现矩阵时候效果出现问题
            preview_data = pd.DataFrame(temp, columns=["表名称", "行数", "列数", "列名称"])
        else:
            preview_data = self.datasetUpdate(self.current_dataset[load_sheet], limitrow=l, skiprow=s)
        self.showDatasetPreview(data=preview_data, header=header)

# 优化完成
class ImportSpssForm(ImportDialog, dataImportFormEngine):
    """
    打开"从spss导入"窗口
    """

    def __init__(self, parent=None):
        super().__init__(parent)
        self.file_types = "*.sav"
        self.IconPath = ":/resources/icons/spss.svg"
        self.setupUi(self)
        self.center()
        self.clearImportParam()
        self.updateUIForm()

    def AddUIFormActivity(self):
        """增加界面中的操作操作响应"""
        self.checkBox_ifColumns.stateChanged.connect(self.updateTableView)  # 选择首行是否为列名
        self.comboBox_encode.currentIndexChanged.connect(self.previewButton)  # 选择编码方式

    def updateUIForm(self):
        """ImportTextForm配置参数部分"""
        self.encode_type = ["gbk", "utf8", "gb2312", "ascii"]
        self.comboBox_encode = self.updateForm_ComboBox(self.comboBox_encode, self.encode_type)

        self.horizontalLayoutAddUI(self.checkBox_ifColumns)
        self.verticalLayoutAddUI(self.lineEdit_datasetName, "left")
        self.verticalLayoutAddUI(self.comboBox_encode, "right")
        self.verticalLayoutAddUI(self.lineEdit_limitRow, "left")
        self.verticalLayoutAddUI(self.lineEdit_skipRow, "right")
        self.publicUIFormActivity()
        self.AddUIFormActivity()

    def getImportParam(self):
        """
            获取界面中的配置信息
            （1）首行列名（2）数据集名称（3）跳过行数（4）限定行数（5）文件编码（6）分割符号
        """
        self.updateWarningMessage(new=False)
        self.import_param.update(
            datasetname=self.lineEdit_datasetName.text(),
            filepath=self.lineEdit_filePath.text(),
            hasheader=self.checkBox_ifColumns.isChecked(),
            status=False, varname={}, dtypes={},
            limitrows=self.checkRowsNumber(self.lineEdit_limitRow.text(), "limitRows"),
            skiprows=self.checkRowsNumber(self.lineEdit_skipRow.text(), "skipRows"),
            param={
                "filename_path": self.checkFilePath(self.lineEdit_filePath.text()),
                "encoding": self.comboBox_encode.currentText()
            }
        )

    def importDatasetReload(self):
        """
            刷新导入的数据
        """
        import pyreadstat
        param = self.import_param["param"]
        self.current_dataset = {}
        varname = self.import_param["datasetname"]
        self.current_dataset[varname], meta = pyreadstat.read_sav(**param)
        # SPSS一次只导入一个文件，因此默认变名称即为数据集名称
        self.getDatasetInfo()
        self.import_param.update(status=True)

    def updateTableView(self):
        """
        刷新预览数据
        """
        # 处理需要呈现的内容
        self.clearPreviewDataTableWidget()
        if not self.import_param["status"]:
            return

        name = self.import_param["datasetname"]
        self.showDatasetPreview(data=self.current_dataset[name], header=True)

# 优化完成
class ImportSasForm(ImportDialog, dataImportFormEngine):
    """打开从sas导入窗口"""

    def __init__(self, parent=None):
        super().__init__(parent)
        self.file_types = "*.sas7bdat"
        self.IconPath = ":/resources/icons/sas.ico"
        self.setupUi(self)
        self.center()
        self.clearImportParam()
        self.updateUIForm()

    def AddUIFormActivity(self):
        # 导入窗口的相关事件
        # 在"导入"窗口，打开选择文件
        self.pushButton_choosefile.clicked.connect(self.chooseFileButton)

        # 帮助
        self.pushButton_help.clicked.connect(self.showHelp)

        # 配置更新数据
        self.checkBox_ifColumns.stateChanged.connect(self.updateTableView)  # 选择首行是否为列名
        self.comboBox_encode.currentIndexChanged.connect(self.previewButton)  # 选择编码方式

        # 按键更新数据
        self.pushButton_preview.clicked.connect(self.previewButton)  # 预览
        self.pushButton_ok.clicked.connect(self.importDatasetButton)  # 导入
        self.pushButton_cancel.clicked.connect(self.close)  # 取消

    def updateUIForm(self):
        """ImportTextForm配置参数部分"""
        self.comboBox_encode = self.updateForm_ComboBox(self.comboBox_encode, self.encode_type)

        self.horizontalLayoutAddUI(self.checkBox_ifColumns)
        self.verticalLayoutAddUI(self.lineEdit_datasetName, "left")
        self.verticalLayoutAddUI(self.comboBox_encode, "right")
        self.verticalLayoutAddUI(self.lineEdit_limitRow, "left")
        self.verticalLayoutAddUI(self.lineEdit_skipRow, "right")
        self.AddUIFormActivity()

    def getImportParam(self):
        """
            获取界面中的配置信息
            （1）首行列名（2）数据集名称（3）跳过行数（4）限定行数（5）文件编码（6）分割符号
        """
        self.updateWarningMessage(new=False)
        self.import_param.update(
            datasetname=self.lineEdit_datasetName.text(),
            filepath=self.lineEdit_filePath.text(),
            hasheader=self.checkBox_ifColumns.isChecked(),
            status=False, varname={}, dtypes={},
            param={
                "filepath_or_buffer": self.checkFilePath(self.lineEdit_filePath.text()),
                "format": "sas7bdat",
                "encoding": self.comboBox_encode.currentText()
            }
        )

    def importDatasetReload(self):
        """
            刷新导入的数据
        """
        param = self.import_param["param"]
        self.current_dataset = {}
        varname = self.import_param["datasetname"]
        self.current_dataset[varname] = pd.read_sas(**param)
        # SPSS一次只导入一个文件，因此默认变名称即为数据集名称
        self.getDatasetInfo()
        self.import_param.update(status=True)

    def updateTableView(self):
        """
        刷新预览数据
        """
        # 处理需要呈现的内容
        self.clearPreviewDataTableWidget()
        if not self.import_param["status"]:
            return

        name = self.import_param["datasetname"]
        self.showDatasetPreview(data=self.current_dataset[name], header=True)

# 优化完成
class ImportMatlabForm(ImportDialog, dataImportFormEngine):
    """打开matlab导入窗口"""

    def __init__(self, parent=None):
        super().__init__(parent)
        self.new_import_filepath = ""
        self.file_types = "*.mat"
        self.IconPath = ":/resources/icons/matlab.svg"
        self.setupUi(self)
        self.center()
        self.clearImportParam()
        self.updateUIForm()

    def AddUIFormActivity(self):
        """增加界面中的操作操作响应"""
        self.checkBox_asDataFrame.stateChanged.connect(self.updateTableView)  # 选择首行是否为列名
        self.comboBox_varname.currentTextChanged.connect(self.updateTableView)

    def updateUIForm(self):
        """ImportMatlabForm配置参数部分"""
        self.horizontalLayoutAddUI(self.checkBox_asDataFrame)
        self.verticalLayoutAddUI(self.lineEdit_datasetName, "left")
        self.verticalLayoutAddUI(self.comboBox_varname, "right")
        self.publicUIFormActivity()
        self.AddUIFormActivity()

    def getImportParam(self):
        """
            获取界面中的配置信息
            （1）首行列名（2）数据集名称（3）跳过行数（4）限定行数（5）文件编码（6）分割符号
        """
        self.updateWarningMessage(new=False)
        self.import_param.update(
            datasetname=self.lineEdit_datasetName.text(),
            filepath=self.lineEdit_filePath.text(), loaddataset=False,
            status=False, varname={}, dtypes={}, ismerge=True,
            asdataframe=self.checkBox_asDataFrame.isChecked(),
            param={
                "file_name": self.checkFilePath(self.lineEdit_filePath.text())
            }
        )

        if self.import_message["isError"]:
            return

        if self.new_import_filepath != self.import_param["filepath"]:
            # 当前仅当文件路径发生变化时候进行重载，否则以内存中数据呈现对应变化
            self.import_param.update(loaddataset=True)

    def importDatasetReload(self):
        """
            刷新导入的数据
        """
        if self.import_param["loaddataset"]:
            import scipy.io as sio
            param = self.import_param["param"]
            self.current_dataset = {}
            mat_dataset = sio.loadmat(**param)
            self.new_import_filepath = self.import_param["filepath"]
            for name_i, var_i in mat_dataset.items():
                if type(var_i) == np.ndarray and name_i[:2] != "__":
                    # 只保留数组类型的数据
                    # 由于部分非矩阵类型数据也是使用 ndarray 类型存储，因此只能使用 type 获取到的类型和 np.ndarray来比较
                    # 这样才能定位到需要的数组类型数据
                    # 注意：目前 scipy.io.loadmat 方法无法解析 matlab 的 table 类型数据！
                    # 预留一种场景：导入时候以 DataFrame 还是 ndarray 形式
                    self.current_dataset[name_i] = var_i
            self.import_param.update(status=True, loaddataset=False)
            self.updateChooseTagName(self.comboBox_varname)

        if not self.import_param["ispreview"]:
            for name_i, var_i in self.current_dataset.items():
                self.current_dataset[name_i] = pd.DataFrame(var_i) if self.import_param["asdataframe"] and len(
                    var_i.shape) <= 2 else var_i
            varname = self.comboBox_varname.currentText()
            if varname != "（全部导入）":
                self.import_param.update(ismerge=False)
                self.current_dataset = {varname: self.current_dataset[varname]}
        self.getDatasetInfo()  # 更新当前数据集的信息

    def updateTableView(self):
        """
        刷新预览数据
        """
        # 处理需要呈现的内容

        varname = self.comboBox_varname.currentText()
        self.clearPreviewDataTableWidget()
        self.updateDatasetNameLine(tag=varname)
        if not self.import_param["status"]:
            self.showDatasetPreview(data=pd.DataFrame([]))
            return

        if varname == "（全部导入）":
            temp = []
            for name_i, data_i in self.current_dataset.items():
                temp.append([name_i, data_i.shape, type(data_i)])
            preview_data = pd.DataFrame(temp, columns=["表名称", "大小", "数据格式"])
        elif not varname:
            return
        else:
            temp = self.current_dataset[varname]
            if len(self.current_dataset[varname].shape) > 2:
                temp = pd.DataFrame([{
                    "变量": varname, "数据类型": type(temp), "数据格式": temp.dtype,
                    "大小": self.current_dataset[varname].shape
                }])
            else:
                temp = pd.DataFrame(self.current_dataset[varname][0:100])
                temp.columns = ["C" + str(i + 1) for i in range(temp.shape[1])]
            preview_data = temp
        self.showDatasetPreview(data=preview_data, header=True)

# 优化完成
class ImportStataForm(ImportDialog, dataImportFormEngine):
    """打开stata导入窗口"""

    def __init__(self, parent=None):
        super().__init__(parent)
        self.file_types = "*.dta"
        self.IconPath = ":/resources/icons/stata.svg"
        self.setupUi(self)
        self.center()
        self.clearImportParam()
        self.updateUIForm()

    def AddUIFormActivity(self):
        """增加界面中的操作操作响应"""
        self.checkBox_ifColIndex.stateChanged.connect(self.updateTableView)  # 选择首行是否为列名

    def updateUIForm(self):
        """ImportMatlabForm配置参数部分"""
        self.horizontalLayoutAddUI(self.checkBox_ifColIndex)
        self.verticalLayoutAddUI(self.lineEdit_datasetName, "left")
        self.publicUIFormActivity()
        self.AddUIFormActivity()

    def getImportParam(self):
        """
            获取界面中的配置信息
            （1）首行列名（2）数据集名称（3）跳过行数（4）限定行数（5）文件编码（6）分割符号
        """
        self.updateWarningMessage(new=False)
        self.import_param.update(
            datasetname=self.lineEdit_datasetName.text(),
            filepath=self.lineEdit_filePath.text(),
            hasheader=True,
            status=False, varname={}, dtypes={},
            param={
                "filepath_or_buffer": self.checkFilePath(self.lineEdit_filePath.text()),
                "index_col": 0 if self.checkBox_ifColIndex.isChecked() else None
            }
        )

    def importDatasetReload(self):
        """
            刷新导入的数据
        """
        param = self.import_param["param"]
        self.current_dataset = {}
        varname = self.import_param["datasetname"]
        self.current_dataset[varname] = pd.read_stata(**param)
        # Stata一次只导入一个文件，因此默认变名称即为数据集名称
        self.getDatasetInfo()
        self.import_param.update(status=True)

    def updateTableView(self):
        """
        刷新预览数据
        """
        # 处理需要呈现的内容
        self.clearPreviewDataTableWidget()
        if not self.import_param["status"]:
            return

        name = self.import_param["datasetname"]
        self.showDatasetPreview(data=self.current_dataset[name], header=True)


from bs4 import BeautifulSoup
from collections import defaultdict, Counter

class Parser:
    @staticmethod
    def getWordsArticle(file):
        words = []
        with open(file, encoding='utf-8') as f:
            for line in f:
                line = line.split(" => ")
                word = line[0].replace("#", "")
                count = int(line[1].replace("\n", ""))
                words.append((word, count))
        return words

    @staticmethod
    def getAllPages(file):
        handler = open(file).read()
        soup = BeautifulSoup(handler, 'xml')
        return soup.find_all('page')

    @staticmethod
    def getText(page):
        soup = BeautifulSoup(page, 'html.parser')
        return soup.title

    @staticmethod
    def getRefs(text):
        soup = BeautifulSoup(text, 'html.parser')
        return soup.find_all('ref')

    @staticmethod
    def solveMatches(matches):
        count = Counter([x[0] for x in matches]).most_common()

        # If there is no tie
        if count[0][1] != count[1][1]:
            return count[0][0]

        # We only want to look at categories with the same numbers of appearance as the most common one
        max_cats = [match[0] for match in count if match[1] == count[0][1]]

        # If there is a tie we sum the distances and take the shortest
        # If there is a tie between the summed distances, we just take the last one
        distance_dict = defaultdict(int)
        for k, v in matches:
            if k in max_cats:
                distance_dict[k] += v
        return Counter(distance_dict).most_common()[-1][0]


import requests
from requests import HTTPError
import yaml
import json
import os

def load_config():
    config_path = 'api_config.yaml'
    with open(os.path.join(os.getcwd(), config_path), mode='r') as yaml_file:
        config = yaml.safe_load(yaml_file)
        return config

def auth():
    conf = load_config()['api_handle']
    url = conf['url']+conf['endpoint_auth']
    data = json.dumps(conf['credentials'])
    headers = {"content-type": "application/json"}
    try:
        result = requests.post(url, data=data, headers=headers)
        result.raise_for_status()
        token = "JWT " + result.json()['access_token']
        return request(token)

    except HTTPError:
        print('Exception with:')
        print(conf['url']+conf['endpoint'])

def get(url, date, headers):
    try:
        result = requests.get(url
                              , data=json.dumps({"date": date})
                              , headers=headers
                              , timeout=10)

        return result.json()

    except HTTPError:
        print('Error')

def save(inp):
    name = inp[0]['date']
    path = os.path.join(os.getcwd(), f'data/{name}')

    if not os.path.exists(path):
        try:
            os.makedirs(path)
        except OSError:
            print("Creation of the directory %s failed" % path)
        else:
            print("Successfully created the directory %s" % path)

    with open(f'{path}/out_{name}.json', 'w') as json_file:
        json.dump(inp, json_file)

def request(token):
    conf = load_config()['api_handle']
    url = conf['url'] + conf['endpoint_get']
    start_date = conf['start_date']
    headers = conf['headers']
    headers['authorization'] = token

    result = None
    if isinstance(result, dict) \
            and (result['message'] == 'No out_of_stock items for this date'):
        print('Empty Date')
    else:
        result = get(url, start_date, headers)

    save(result)

if __name__ == '__main__':
    auth()