#!/usr/bin/env python
# coding: utf-8

# In[9]:

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
get_ipython().run_line_magic('matplotlib', 'inline')
from tqdm.auto import tqdm
import concurrent.futures
from multiprocessing import Pool
import copy,os,sys,psutil
from collections import Counter

# In[35]:

import tensorflow as tf
import datetime
import gensim
import numpy as np
import pandas as pd
from zac_pyutils.ExqUtils import zprint
from zac_pyutils import ExqUtils
from collections import deque
from tqdm.auto import tqdm
from collections import Counter
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.preprocessing import OneHotEncoder
# from gensim.models.wrappers import FastText
import fasttext
import json
import re
import inspect

# # Config

# In[45]:

class TrainingConfig(object):
    epoches = 5
    batchSize = 128
    evaluateEvery = 100
    checkpointEvery = 100
    learningRate = 0.001
    minWordCnt = 5

class ModelConfig(object):
    numFilters = 64

    filterSizes = [2, 3, 4, 5]
    dropoutKeepProb = 0.5
    l2RegLambda = 0.001

class Config(object):
    job = "taste"
    basePath = "/home/zhoutong/nlp/data/textcnn"
    dataSource = basePath + "/labeled_timeliness_region_taste_emotion_sample.json"
    # dataSource = dataSource + ".sample_h10k"
    summaryDir = basePath+"/summary"
    cnnmodelPath_pb = basePath + "/textcnn_model_pb"
    cnnmodelPath_ckpt = basePath+"/textcnn_model_ckpt/model.ckpt"

    weDim = 300
    ft_modelPath = basePath + '/cc.en.300.bin'

    padSize = 16
    pad = '<PAD>'
    pad_initV = np.zeros(weDim)
    unk = '<UNK>'
    unk_initV = np.random.randn(weDim)

    # numClasses = 4  # 二分类设置为1，多分类设置为类别的数目
    numClasses_dict = {"taste":4,"timeliness":9,"emotion":3}
    numClasses = numClasses_dict[job]  # 二分类设置为1，多分类设置为类别的数目

    testRatio = 0.2  # 测试集的比例

    training = TrainingConfig()

    model = ModelConfig()

class Utils():
    # 清理符号
    @staticmethod
    def clean_punctuation(inp_text):
        res = re.sub(r"[~!@#$%^&*()_+-={\}|\[\]:\";'<>?,./“”]", r' ', inp_text)
        res = re.sub(r"\\u200[Bb]", r' ', res)
        res = re.sub(r"\n+", r" ", res)
        res = re.sub(r"\s+", " ", res)
        return res.strip()
    @staticmethod
    def pad_list(inp_list,width,pad_const):
        if len(inp_list) >= width:
            return inp_list[:width]
        else:
            return inp_list+[pad_const]*(width-len(inp_list))

config = Config()

# In[49]:

data.trainReviews.shape[0]
527480
config.training.batchSize
527480//config.training.batchSize

# In[75]:

# fail
import timeout_decorator
@timeout_decorator.timeout(seconds=3, use_signals=False, exception_message="timeout")
def get_input(res_inpt):
    res_inpt=input("是否重新生成persist数据？(y/n)")

res = None
try:
    get_input(res)
except Exception as e:
    res = "N"
print(res)

# In[ ]:

# fail
class StoppableThread(Thread):
    def __init__(self,target,args,timeout=1):
        super(Thread,self).__init__()
        self.timeout = timeout
        self.stopped = False
        self.t = Thread(target=target,args=args)
        self.t.setDaemon(True)

    def start(self):
        self.t.start()
        while not self.stopped:
            self.t.join(self.timeout)
        print("thread stopped.")

    def stop(self):
        self.stopped=True

res = "n"
t1 = StoppableThread(target=get_input,args=(res,))
t1.start()
time.sleep(3)
t1.terminate()
print(res)

# In[ ]:

class StoppableThread(Thread):
    def __init__(self,target,args,timeout=1):
        super(Thread,self).__init__()
        self.timeout = timeout
        self.stopped = False
        self.t = Thread(target=target,args=args)
        self.t.setDaemon(True)

    def start(self):
        self.t.start()
        while not self.stopped:
            self.t.join(self.timeout)
        print("thread stopped.")

    def stop(self):
        self.stopped=True

def func():
    pass

t1 = StoppableThread(target=func)
t1.start()
for i in range(5):
    time.sleep(1)
    print(i)
t1.terminate()

# In[1]:

print("abc")

# In[12]:

from threading import Thread
import time
import queue

class StoppableThread(Thread):
    class TimeoutException(Exception):
        pass

    def __init__(self,target,args=None,time_limit=1,delta=0.05):
        super(Thread,self).__init__()
        self.delta = delta
        self.stopped = False
        if args is None:
            self.t = Thread(target=target)
        else:
            self.t = Thread(target=target,args=args)
        self.t.setDaemon(True)

        self.timing_thread = Thread(target=self.timing,args=(time_limit,))
        self.timing_thread.setDaemon(True)

    def timing(self,timeout):
        time.sleep(timeout)
        self.stopped=True

    def start(self):
        self.t.start()
        self.timing_thread.start()
        while not self.stopped:
            self.t.join(self.delta)
            time.sleep(0.05)
        raise TimeoutException("thread timeout")

    def stop(self):
        self.stopped=True

q = queue.Queue()
def func(inp):
    print("input: ",inp)
    for i in range(3):
        time.sleep(1)
        print(i)
    inp += 1
    print("processed: ",inp)
    q.put(inp)

to_inp = 2
t1 = StoppableThread(target=func,args=(to_inp,), time_limit=1)
t1.start()
print("to_inp: ",to_inp)
print("result in queue:",q.queue[0])

# In[19]:

len(q.queue)

# In[14]:

print("result in queue:",q.queue[0])

# In[9]:

print(q.queue[0])
dir(q)

# In[39]:

zprint("各数据路径：")
print("basePath路径：{}\n样本数据来源: {}\nsummary目录：{}\n".format(config.basePath,config.dataSource,config.summaryDir))
zprint("模型参数如下：")
for k,v in inspect.getmembers(config.model):
    if not k.startswith("_"):
        print(k,v)
zprint("训练参数如下：")
for k,v in inspect.getmembers(config.training):
    if not k.startswith("_"):
        print(k,v)

# # TextCNN

# In[3]:

class TextCNN(object):
    """
    Text CNN 用于文本分类
    """

    def __init__(self, config, wordEmbedding):

        # 定义模型的输入
        self.inputX = tf.placeholder(tf.int32, [None, config.padSize], name="inputX")
        self.inputY = tf.placeholder(tf.int32, [None], name="inputY")

        self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb")

        # 定义l2损失
        l2Loss = tf.constant(0.0)

        # 词嵌入层
        with tf.name_scope("embedding"):
            # 利用预训练的词向量初始化词嵌入矩阵
            self.W = tf.Variable(tf.cast(wordEmbedding, dtype=tf.float32, name="word2vec"), name="W")
            # 利用词嵌入矩阵将输入的数据中的词转换成词向量，维度[batch_size, sequence_length, embedding_size]
            self.embeddedWords = tf.nn.embedding_lookup(self.W, self.inputX)
            # 卷积的输入是思维[batch_size, width, height, channel]，因此需要增加维度，用tf.expand_dims来增大维度
            self.embeddedWordsExpanded = tf.expand_dims(self.embeddedWords, -1)

        # 创建卷积和池化层
        pooledOutputs = []
        # 有三种size的filter，2, 3， 4， 5，textCNN是个多通道单层卷积的模型，可以看作三个单层的卷积模型的融合
        for i, filterSize in enumerate(config.model.filterSizes):
            with tf.name_scope("conv-maxpool-%s" % filterSize):
                # 卷积层，卷积核尺寸为filterSize * embeddingSize，卷积核的个数为numFilters
                # 初始化权重矩阵和偏置
                # 4-D (height,width,in_channels,out_channels) | out_channels是多少就用多少个 Height*Width 的卷积核
                filterShape = [filterSize, config.weDim, 1, config.model.numFilters]
                W = tf.Variable(tf.truncated_normal(filterShape, stddev=0.1), name="W")
                b = tf.Variable(tf.constant(0.1, shape=[config.model.numFilters]), name="b")
                convRes = tf.nn.conv2d(
                    input=self.embeddedWordsExpanded,
                    filter=W,
                    strides=[1, 1, 1, 1],
                    padding="VALID",
                    name="conv")

                # relu函数的非线性映射
                h = tf.nn.relu(tf.nn.bias_add(convRes, b), name="relu")

                # 池化层，最大池化，池化是对卷积后的序列取一个最大值
                pooled = tf.nn.max_pool(
                    h,
                    ksize=[1, config.padSize - filterSize + 1, 1, 1],
                    # ksize shape: [batch, height, width, channels]
                    strides=[1, 1, 1, 1],
                    padding='VALID',
                    name="pool")
                pooledOutputs.append(pooled)  # 将三种size的filter的输出一起加入到列表中

        # 得到CNN网络的输出长度
        numFiltersTotal = config.model.numFilters * len(config.model.filterSizes)

        # 池化后的维度不变，按照最后的维度channel来concat
        self.hPool = tf.concat(pooledOutputs, 3)

        # 摊平成二维的数据输入到全连接层
        self.hPoolFlat = tf.reshape(self.hPool, [-1, numFiltersTotal])

        # dropout
        with tf.name_scope("dropout"):
            self.hDrop = tf.nn.dropout(self.hPoolFlat, self.dropoutKeepProb)

        # 全连接层的输出
        with tf.name_scope("output"):
            outputW = tf.get_variable(
                "outputW",
                shape=[numFiltersTotal, config.numClasses],
                initializer=tf.contrib.layers.xavier_initializer())
            outputB = tf.Variable(tf.constant(0.1, shape=[config.numClasses]), name="outputB")
            l2Loss += tf.nn.l2_loss(outputW)
            l2Loss += tf.nn.l2_loss(outputB)
            self.logits = tf.nn.xw_plus_b(self.hDrop, outputW, outputB, name="logits")
            if config.numClasses == 1:
                self.predictions = tf.cast(tf.greater_equal(self.logits, 0.0), tf.int32, name="predictions")
            elif config.numClasses > 1:
                self.predictions = tf.argmax(self.logits, axis=-1, name="predictions")

            print(self.predictions)

        # 计算二元交叉熵损失
        with tf.name_scope("loss"):

            if config.numClasses == 1:
                losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits,
                                                                 labels=tf.cast(tf.reshape(self.inputY, [-1, 1]),
                                                                                dtype=tf.float32))
            elif config.numClasses > 1:
                losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY)

            self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss

# In[4]:

class MetricUtils():
    """
    定义各类性能指标
    """
    @staticmethod
    def mean(item: list) -> float:
        """
        计算列表中元素的平均值
        :param item: 列表对象
        :return:
        """
        res = sum(item) / len(item) if len(item) > 0 else 0
        return res

    @staticmethod
    def accuracy(pred_y, true_y):
        """
        计算二类和多类的准确率
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :return:
        """
        if isinstance(pred_y[0], list):
            pred_y = [item[0] for item in pred_y]
        corr = 0
        for i in range(len(pred_y)):
            if pred_y[i] == true_y[i]:
                corr += 1
        acc = corr / len(pred_y) if len(pred_y) > 0 else 0
        return acc

    @staticmethod
    def binary_precision(pred_y, true_y, positive=1):
        """
        二类的精确率计算
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param positive: 正例的索引表示
        :return:
        """
        corr = 0
        pred_corr = 0
        for i in range(len(pred_y)):
            if pred_y[i] == positive:
                pred_corr += 1
                if pred_y[i] == true_y[i]:
                    corr += 1

        prec = corr / pred_corr if pred_corr > 0 else 0
        return prec

    @staticmethod
    def binary_recall(pred_y, true_y, positive=1):
        """
        二类的召回率
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param positive: 正例的索引表示
        :return:
        """
        corr = 0
        true_corr = 0
        for i in range(len(pred_y)):
            if true_y[i] == positive:
                true_corr += 1
                if pred_y[i] == true_y[i]:
                    corr += 1

        rec = corr / true_corr if true_corr > 0 else 0
        return rec

    @staticmethod
    def binary_f_beta(pred_y, true_y, beta=1.0, positive=1):
        """
        二类的f beta值
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param beta: beta值
        :param positive: 正例的索引表示
        :return:
        """
        precision = MetricUtils.binary_precision(pred_y, true_y, positive)
        recall = MetricUtils.binary_recall(pred_y, true_y, positive)
        try:
            f_b = (1 + beta * beta) * precision * recall / (beta * beta * precision + recall)
        except:
            f_b = 0
        return f_b

    @staticmethod
    def multi_precision(pred_y, true_y, labels):
        """
        多类的精确率
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param labels: 标签列表
        :return:
        """
        if isinstance(pred_y[0], list):
            pred_y = [item[0] for item in pred_y]

        precisions = [MetricUtils.binary_precision(pred_y, true_y, label) for label in labels]
        prec = MetricUtils.mean(precisions)
        return prec

    @staticmethod
    def multi_recall(pred_y, true_y, labels):
        """
        多类的召回率
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param labels: 标签列表
        :return:
        """
        if isinstance(pred_y[0], list):
            pred_y = [item[0] for item in pred_y]

        recalls = [MetricUtils.binary_recall(pred_y, true_y, label) for label in labels]
        rec = MetricUtils.mean(recalls)
        return rec

    @staticmethod
    def multi_f_beta(pred_y, true_y, labels, beta=1.0):
        """
        多类的f beta值
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param labels: 标签列表
        :param beta: beta值
        :return:
        """
        if isinstance(pred_y[0], list):
            pred_y = [item[0] for item in pred_y]

        f_betas = [MetricUtils.binary_f_beta(pred_y, true_y, beta, label) for label in labels]
        f_beta = MetricUtils.mean(f_betas)
        return f_beta

    @staticmethod
    def get_binary_metrics(pred_y, true_y, f_beta=1.0):
        """
        得到二分类的性能指标
        :param pred_y:
        :param true_y:
        :param f_beta:
        :return:
        """
        acc = MetricUtils.accuracy(pred_y, true_y)
        recall = MetricUtils.binary_recall(pred_y, true_y)
        precision = MetricUtils.binary_precision(pred_y, true_y)
        f_beta = MetricUtils.binary_f_beta(pred_y, true_y, f_beta)
        return acc, recall, precision, f_beta

    @staticmethod
    def get_multi_metrics(pred_y, true_y, labels, f_beta=1.0):
        """
        得到多分类的性能指标
        :param pred_y:
        :param true_y:
        :param labels:
        :param f_beta:
        :return:
        """
        acc = MetricUtils.accuracy(pred_y, true_y)
        recall = MetricUtils.multi_recall(pred_y, true_y, labels)
        precision = MetricUtils.multi_precision(pred_y, true_y, labels)
        f_beta = MetricUtils.multi_f_beta(pred_y, true_y, labels, f_beta)
        return acc, recall, precision, f_beta

# # Class Dataset

# In[5]:

class Dataset(object):
    def __init__(self, config):
        self.config = config
        self._dataSource = config.dataSource

        self.testRatio = config.testRatio
        self._we_fp = config.basePath+"/wordEmbeddingInfo"  # \t分割 词,idx,embedding
        self._tokens_arr_fp = config.basePath+"/tokens_arr.npy"
        self._labels_arr_fp = config.basePath+"/labels_arr.npy"
        self._emb_arr_fp = config.basePath+"/emb_arr.npy"
        self.ft_modelPath = config.ft_modelPath
        self.ft_model = None

        self.trainReviews = np.array([])
        self.trainLabels = np.array([])

        self.evalReviews = np.array([])
        self.evalLabels = np.array([])

        self.token2idx = {}
        self.wordEmbedding = None
        self.labelSet = []
        self.totalWordCnt = 0

    def _readData(self, filePath):
        f_iter = ExqUtils.load_file_as_iter(filePath)
        tokens_list = deque()
        label_list = deque()
        zprint("loading data from: "+filePath)
        for l in tqdm(f_iter,desc="readData"):
            info = json.loads(l)
            text,label = info['title'],info[self.config.job]
            tokens = Utils.pad_list(Utils.clean_punctuation(text).split(" "),width=self.config.padSize,pad_const=self.config.pad)
            tokens_list.append(tokens)
            label_list.append(label)
        return np.array(tokens_list), np.array(label_list)

    def _initStopWord(self, stopWordPath):
        with open(stopWordPath, "r") as fr:
            self._stopWordSet = set(fr.read().splitlines())

    def _initFasttextModel(self):
        if self.ft_model is None:
            self.ft_model = fasttext.load_model(self.ft_modelPath)

    def _tokens2idx(self,tokens_arr):
        tokensSet = set(np.unique(tokens_arr))

        pass

    def dataGen_persist(self):
        """
                初始化训练集和验证集
                """
        zprint("init fasttext model")
        self._initFasttextModel()

        # 初始化数据集
        tokens_arr, label_arr = self._readData(self._dataSource)
        self.labelSet = set(np.unique(label_arr))
        tokensSet = set(np.unique(tokens_arr))

        self.totalWordCnt = len(tokensSet)
        wordEmb = np.zeros(shape=[self.totalWordCnt, self.ft_model.get_dimension()])
        # (idx,token,emb)保存到文本文件
        zprint("预测词向量总计: {} , 词向量存入文件: {}".format(self.totalWordCnt, self._we_fp))
        with open(self._we_fp, "w") as fw:
            # 加上 <PAD> 和 <UNK> 及其初始化
            for idx, token in tqdm(enumerate(tokensSet), total=self.totalWordCnt,desc="tokensSet"):
                if token == self.config.pad:
                    emb = self.config.pad_initV
                elif token == self.config.unk:
                    emb = self.config.unk_initV
                else:
                    emb = self.ft_model[token]
                self.token2idx.update({token:idx})
                wordEmb[idx] = emb
                fw.writelines(str(idx) + "\t" + token + "\t" + ",".join([str(i) for i in list(emb)]) + "\n")

        # tokens变为idx保存为npy
        zprint("tokens映射为索引保存到npy")
        tokensIdx_arr = np.zeros_like(tokens_arr, dtype=np.int64)
        for i,tokens in enumerate(tokens_arr):
            for j,token in enumerate(tokens):
                tokensIdx_arr[i][j] = self.token2idx[token]
        np.save(self._tokens_arr_fp,tokensIdx_arr)

        zprint("labels保存到npy")
        np.save(self._labels_arr_fp,label_arr)

        zprint("idx对应的emb保存到npy")
        np.save(self._emb_arr_fp,wordEmb)

    def loadData(self):
        self.wordEmbedding = np.load(self._emb_arr_fp)

        tokensIdx_arr = np.load(self._tokens_arr_fp)
        label_arr = np.load(self._labels_arr_fp)
        self.labelSet = set(np.unique(label_arr))
        # 初始化训练集和测试集
        sss = StratifiedShuffleSplit(n_splits=1, test_size=self.testRatio, random_state=2019)
        train_idx, test_idx = list(sss.split(np.zeros(label_arr.shape[0]), label_arr))[0]

        self.trainReviews = tokensIdx_arr[train_idx]
        self.trainLabels = label_arr[train_idx]

        self.evalReviews = tokensIdx_arr[test_idx]
        self.evalLabels = label_arr[test_idx]

# # Produce Data

# In[11]:

data = Dataset(config)
# data.dataGen_persist()
data.loadData()
print("train data shape: {}".format(data.trainReviews.shape))
print("train label shape: {}".format(data.trainLabels.shape))
print("eval data shape: {}".format(data.evalReviews.shape))
print("eval data shape: {}".format(data.evalLabels.shape))
print("wordEmbedding info file: {}".format(data._we_fp))

# In[12]:

trainReviews = data.trainReviews
trainLabels = data.trainLabels
evalReviews = data.evalReviews
evalLabels = data.evalLabels

wordEmbedding = data.wordEmbedding
labelList = data.labelSet

# In[13]:

trainReviews
trainLabels

# # 开始构建计算图

# In[11]:

evalLabels.shape
batchY = np.array(evalLabels[10: 100], dtype="float32")
batchY.shape

# In[15]:

def nextBatch(x, y, batchSize):
    """
    生成batch数据集，用生成器的方式输出
    """
    perm = np.arange(len(x))
    np.random.shuffle(perm)
    x,y = x[perm],y[perm]

    numBatches = len(x) // batchSize

    for i in range(numBatches):
        start = i * batchSize
        end = start + batchSize
        batchX = np.array(x[start: end], dtype="int64")
        batchY = np.array(y[start: end], dtype="float32")

        yield batchX, batchY

# In[55]:

tf.reset_default_graph()

# In[56]:

self = cnn
# 定义模型的输入
self.inputX = tf.placeholder(tf.int32, [None, config.padSize], name="inputX")
self.inputY = tf.placeholder(tf.int32, [None], name="inputY")
self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb")
# 定义l2损失
l2Loss = tf.constant(0.0)
# 词嵌入层
with tf.name_scope("embedding"):
    # 利用预训练的词向量初始化词嵌入矩阵
    self.W = tf.Variable(tf.cast(wordEmbedding, dtype=tf.float32, name="word2vec"), name="W")
    # 利用词嵌入矩阵将输入的数据中的词转换成词向量，维度[batch_size, sequence_length, embedding_size]
    self.embeddedWords = tf.nn.embedding_lookup(self.W, self.inputX)
    # 卷积的输入是思维[batch_size, width, height, channel]，因此需要增加维度，用tf.expand_dims来增大维度
    self.embeddedWordsExpanded = tf.expand_dims(self.embeddedWords, -1)
# 创建卷积和池化层
pooledOutputs = []
# 有三种size的filter，2, 3， 4， 5，textCNN是个多通道单层卷积的模型，可以看作三个单层的卷积模型的融合
for i, filterSize in enumerate(config.model.filterSizes):
    with tf.name_scope("conv-maxpool-%s" % filterSize):
        # 卷积层，卷积核尺寸为filterSize * embeddingSize，卷积核的个数为numFilters
        # 初始化权重矩阵和偏置
        filterShape = [filterSize, config.weDim, 1, config.model.numFilters]
        W = tf.Variable(tf.truncated_normal(filterShape, stddev=0.1), name="W")
        b = tf.Variable(tf.constant(0.1, shape=[config.model.numFilters]), name="b")
        convRes = tf.nn.conv2d(
            input=self.embeddedWordsExpanded,
            filter=W,
            strides=[1, 1, 1, 1],
            padding="VALID",
            name="conv")
        # relu函数的非线性映射
        h = tf.nn.relu(tf.nn.bias_add(convRes, b), name="relu")
        # 池化层，最大池化，池化是对卷积后的序列取一个最大值
        pooled = tf.nn.max_pool(
            h,
            ksize=[1, config.padSize - filterSize + 1, 1, 1],
            # ksize shape: [batch, height, width, channels]
            strides=[1, 1, 1, 1],
            padding='VALID',
            name="pool")
        pooledOutputs.append(pooled)  # 将三种size的filter的输出一起加入到列表中
# 得到CNN网络的输出长度
numFiltersTotal = config.model.numFilters * len(config.model.filterSizes)
# 池化后的维度不变，按照最后的维度channel来concat
self.hPool = tf.concat(pooledOutputs, 3)
# 摊平成二维的数据输入到全连接层
self.hPoolFlat = tf.reshape(self.hPool, [-1, numFiltersTotal])
# dropout
with tf.name_scope("dropout"):
    self.hDrop = tf.nn.dropout(self.hPoolFlat, self.dropoutKeepProb)
# 全连接层的输出
with tf.name_scope("output"):
    outputW = tf.get_variable(
        "outputW",
        shape=[numFiltersTotal, config.numClasses],
        initializer=tf.contrib.layers.xavier_initializer())
    outputB = tf.Variable(tf.constant(0.1, shape=[config.numClasses]), name="outputB")
    l2Loss += tf.nn.l2_loss(outputW)
    l2Loss += tf.nn.l2_loss(outputB)
    self.logits = tf.nn.xw_plus_b(self.hDrop, outputW, outputB, name="logits")
    if config.numClasses == 1:
        self.predictions = tf.cast(tf.greater_equal(self.logits, 0.0), tf.int32, name="predictions")
    elif config.numClasses > 1:
        self.predictions = tf.argmax(self.logits, axis=-1, name="predictions")
    print(self.predictions)
# 计算二元交叉熵损失
with tf.name_scope("loss"):
    if config.numClasses == 1:
        losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits,
                                                         labels=tf.cast(tf.reshape(self.inputY, [-1, 1]),
                                                                        dtype=tf.float32))
    elif config.numClasses > 1:
        losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY)
    self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss

# In[82]:

self.logits.shape
self.inputY.shape
self.hDrop.shape,outputW.shape,outputB.shape
self.hPoolFlat.shape
inpX.shape,inpY.shape
filterShape
convRes.shape
pooled.shape
pooledOutputs
numFiltersTotal,"=",config.model.numFilters,"*",len(config.model.filterSizes)
self.hPool.shape
tf.concat(pooledOutputs, 3).shape,tf.concat(pooledOutputs, -1).shape
self.hPoolFlat.shape
self.hDrop.shape
self.logits.shape

# In[94]:

with tf.Session() as sess:
    fd={self.inputX:inpX,self.inputY:inpY,self.dropoutKeepProb:0.8}
    sess.run(tf.global_variables_initializer())
    sess.run(self.embeddedWordsExpanded,feed_dict=fd).shape
    filterShape
    sess.run(W).shape
    sess.run(convRes,feed_dict=fd).shape
    sess.run(pooled,feed_dict=fd).shape
    sess.run(outputW,fd).shape
    sess.run(outputB,fd).shape
    sess.run(self.logits,fd).shape
    sess.run(self.predictions,fd).shape
    sess.run(self.inputY,fd).shape
    sess.run(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY),fd)

# In[16]:

tf.reset_default_graph()

# In[21]:

trainReviews.shape
config.batchSize

# In[61]:

from zac_pyutils import TFUtils

# In[73]:

import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="-1"
tf.reset_default_graph() # 防止运行多次后session创建、运行会变慢
ckpt_fp="/home/zhoutong/nlp/data/textcnn/textcnn_model_ckpt/model.ckpt-20480"
pb_fp = "/home/zhoutong/nlp/data/textcnn/tmp_model.pb.sample"
saver = tf.train.import_meta_graph(ckpt_fp + '.meta', clear_devices=True)
with tf.Session() as sess:
    saver.restore(sess, ckpt_fp)
    output_tensor = sess.graph.get_tensor_by_name('output/predictions:0')
    output_tensor
    output_graph_def = tf.graph_util.convert_variables_to_constants(
            sess=sess,
            input_graph_def=sess.graph_def,
            output_node_names=['output/predictions'])
    with tf.gfile.GFile(pb_fp, "wb") as f:  # 保存模型
        f.write(output_graph_def.SerializeToString())  # 序列化输出

# In[ ]:

# In[ ]:

# In[ ]:

# import os
# os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
# os.environ["CUDA_VISIBLE_DEVICES"]="-1"
TFUtils.convert_ckpt2pb(ckpt_fp="/home/zhoutong/nlp/data/textcnn/textcnn_model_ckpt/model.ckpt-20480",
                        pb_fp = "/home/zhoutong/nlp/data/textcnn/tmp_model.pb.sample",
                        output_name_list=['output/predictions'])

# In[ ]:


from tools.utils import execute
from os import chdir

class Deployer:
    def __init__(self, config):
        self.config = config

    def deploy(self):
        chdir(self.config.proxy_root)
        self._prepare_env_file()
        execute('yarn deploy-skale-manager-components')
        execute('yarn deploy-to-both-chains')

    def deploy_mainnet(self):
        chdir(self.config.proxy_root)
        self._prepare_env_file()
        execute('yarn deploy-skale-manager-components')
        execute('yarn deploy-to-mainnet')

    def deploy_schain(self, schain_name):
        chdir(self.config.proxy_root)
        self._prepare_env_file(schain_name)
        execute('yarn deploy-to-schain')

    def deploy_second_schain(self):
        self.deploy()
        self.deploy_schain(self.config.schain_name_2)

    # private

    def _prepare_env_file(self, schain_name=''):
        if schain_name == '' : schain_name = self.config.schain_name
        env_file = [f'NETWORK_FOR_ETHEREUM="{self.config.network_for_mainnet}"',
                    f'NETWORK_FOR_SCHAIN="{self.config.network_for_schain}"',
                    f'PRIVATE_KEY_FOR_ETHEREUM="{self.config.mainnet_key}"',
                    f'URL_W3_ETHEREUM="{self.config.mainnet_rpc_url}"',
                    f'PRIVATE_KEY_FOR_SCHAIN="{self.config.schain_key}"',
                    f'URL_W3_S_CHAIN="{self.config.schain_rpc_url}"',
                    f'CHAIN_NAME_SCHAIN="{schain_name}"',
                    'NO_SIGNATURES=true']

        with open('.env', 'w') as dot_env:
            dot_env.write('\n'.join(env_file))


# %%

import json
import os
import shutil
from copy import deepcopy
from enum import Enum, auto
from pathlib import Path
from sys import platform
from typing import Any, Tuple
from zipfile import ZipFile
from deeppavlov import build_model, configs
from deeppavlov.core.commands.utils import parse_config

class ModelCodename(Enum):
    TEST = auto()
    TEST2 = auto()
    ORIGIN = auto()

class BaseDD:
    def __init__(self, codename: ModelCodename):
        self.codename = codename

class JsonDD(BaseDD):
    def __init__(self, codename: ModelCodename):
        super().__init__(codename)

class DDFactory:
    @staticmethod
    def create(dataset_codename: ModelCodename) -> BaseDD:
        return JsonDD(dataset_codename)

class BaseDataset:
    def __init__(self, data: Any, descriptor: BaseDD):
        self.data = data
        self.descriptor = descriptor

class JsonDataset:
    def __init__(self, data: list, view_config: str, descriptor: BaseDD):
        self.data = data
        self.view_config = view_config
        self.descriptor = descriptor

class DatasetKeeper:
    DATASET_DIR = "DS"
    DATASET_TASKS_FILENAME = "data.json"

    @staticmethod
    def load_dataset(dataset_descriptor: BaseDD) -> BaseDataset:
        filename = DatasetKeeper._get_filename(dataset_descriptor.codename)

        data, view_config = DatasetKeeper._load_json_dataset(filename)
        return JsonDataset(data, view_config, dataset_descriptor)

    @staticmethod
    def _load_json_dataset(filename) -> Tuple[list, str]:
        directory = os.path.dirname(filename)
        json_filename = os.path.join(directory, DatasetKeeper.DATASET_TASKS_FILENAME)
        with open(json_filename, "r", encoding="utf-8") as json_file:
            data = json.load(json_file)
        return data, None

    @staticmethod
    def _get_filename(dataset_codename: ModelCodename) -> str:
        path = os.path.join(DatasetKeeper.DATASET_DIR, f"{dataset_codename}")
        os.makedirs(path, exist_ok=True)
        filename = os.path.join(path, DatasetKeeper.DATASET_TASKS_FILENAME)
        return filename

class BaseMD:
    def __init__(self, model_codename: ModelCodename):
        self.model_codename = model_codename

class MDFactory:
    @staticmethod
    def create(model_codename: ModelCodename) -> BaseMD:
        return BaseMD(model_codename)

class BaseModel:
    def __init__(self, instance, descriptor: BaseMD):
        self.instance = instance
        self.descriptor = descriptor

    def __call__(self, *args, **kwargs):
        return self.instance(*args, **kwargs)

class ModelKeeper:
    MODEL_DIR = "models"
    TRAIN_PATH = "training"

    @staticmethod
    def load_model(model_descriptor: BaseMD) -> BaseModel:
        model_dir = ModelKeeper._make_model_dir(model_descriptor.model_codename)
        parsed_config = ModelKeeper._load_config(model_dir)
        instance = build_model(parsed_config)
        return BaseModel(instance, model_descriptor)

    @staticmethod
    def train_model(model_descriptor: BaseMD, dataset_descriptor: BaseDD, keep_train_files: bool) -> BaseModel:
        model_dir = ModelKeeper._make_model_dir(model_descriptor.model_codename)
        basic_config = ModelKeeper._load_config(model_dir)
        train_config = ModelKeeper._prepare_for_training(basic_config)
        dataset = DatasetKeeper.load_dataset(dataset_descriptor)

        from trainer import Trainer

        trainer = Trainer(keep_train_files, train_config, dataset, ModelKeeper.TRAIN_PATH)
        model = trainer.fit()
        return BaseModel(model, model_descriptor)

    @staticmethod
    def _load_config(model_dir) -> dict:
        #используем модель ner_rus_bert
        with open(configs.ner.ner_rus_bert, "r") as raw_config:
            wrong_config = json.load(raw_config)
        wrong_config["metadata"]["variables"]["MODEL_PATH"] = model_dir
        parsed_config = parse_config(wrong_config)
        return parsed_config

    @staticmethod
    def _prepare_for_training(config: dict) -> dict:
        # TODO Magic numbers
        config = deepcopy(config)
        config["train"]["epochs"] = 10  # 45
        config["train"]["batch_size"] = 10  # 14
        config["dataset_reader"]["data_path"] = ModelKeeper.TRAIN_PATH
        return config

    @staticmethod
    def _backup_model_files(model_dir):
        ext = "zip"
        base_name = os.path.basename(model_dir)
        shutil.make_archive(base_name, ext, model_dir)
        shutil.move(f"{base_name}.{ext}", Path(model_dir).parent)

    @staticmethod
    def _make_model_dir(model_codename: ModelCodename) -> str:
        model_path = os.path.join(ModelKeeper.MODEL_DIR, f"{model_codename}")
        os.makedirs(model_path, exist_ok=True)
        return model_path

#------------------------------------------------------------------------------------------------------------------------------------------------
#запускаем здесь

if __name__ == "__main__":

    train_model = True  # True - обучаем модель, False - просмотриваем результат
    keep_train_files = True # True - не обновляем файлы обучения в папке training, False - обновляем файлы обучения в папке training из Label Studio
    model_path = ModelCodename.TEST2 # model_path - наименование папки с моделью
    dd = DDFactory.create(model_path)
    dd.codename='test2' # codename - наименование папки с обучающим json
    ds = DatasetKeeper.load_dataset(dd)
    md = MDFactory.create(model_path)
    if train_model:
        # Обучение новой модели
        model = ModelKeeper.train_model(md, dd, keep_train_files)
        model.instance.save()
    else:
        model = ModelKeeper.load_model(md)

    print(model.instance(['королевич и Шрек сыграли свадьбу']))


import os
import scipy.io
import numpy as np
import scipy.signal
import config
import pandas as pd
import scipy.stats
import plotting

# Load Dataset
def LoadDataset(file_name, parameters):
    # Read Data File and Extract Desired Signals
    data_file = scipy.io.loadmat('./data/training/' + file_name)
    data = data_file['dataset']['data'][0][0]
    tags = data_file['dataset']['tags'][0][0].T[0]  # shape into 1d array
    selected_signals_data = np.vstack([data[parameters.signals[i]][0][0][:, j]
                                       for i in range(len(parameters.signals))
                                       for j in parameters.channels[i]])
    # Generate Sliding Windows
    signal_windows = np.expand_dims(GenerateWindowSlides(selected_signals_data[0],
                                                         parameters.window_size, parameters.step_size), axis=0)
    for i in range(1, len(selected_signals_data)):
        signal_windows = np.append(signal_windows,
                                   np.expand_dims(GenerateWindowSlides(selected_signals_data[i],
                                                                       parameters.window_size,
                                                                       parameters.step_size), axis=0), axis=0)
    action_tag_windows = GenerateWindowSlides(tags, parameters.window_size, parameters.step_size)

    # Generate Input/Output Dataset
    input_signal_data = signal_windows.swapaxes(0, 1)
    if parameters.window_tag_method == "MODE":
        output_class_data = scipy.stats.mode(action_tag_windows, axis=1)[0].flatten()
    elif parameters.window_tag_method == "LAST":
        output_class_data = action_tag_windows[:, -1]
    else:
        raise ValueError("Window Tag Method " + parameters.window_tag_method + " does not match either MODE or LAST.")

    return input_signal_data, output_class_data

# Sliding Window Function
def GenerateWindowSlides(data, window, step):
    shape = data.shape[:-1] + (data.shape[-1] - window + 1, window)
    strides = data.strides + (data.strides[-1],)
    slided_windows = np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides)
    return slided_windows[::step]

# Generate X/Y Data
def GenerateInputOutputData(one_hot=True, sanity_plot=False):
    # Retrieve Parameters
    parameters = config.LoadDataProvisionParameters()

    # Retrieve Training Data Files
    data_files = os.listdir('./data/training')
    X, Y = LoadDataset(data_files[0], parameters)

    # Generate I/O Data
    for i in range(1, len(data_files)):
        x, y = LoadDataset(data_files[i], parameters)
        X = np.append(X, x, axis=0)
        Y = np.append(Y, y, axis=0)
    X = np.expand_dims(X, axis=0).swapaxes(0, 1).swapaxes(2, 3)     # conform to NHWC shape
    Y[Y == 0] = 2   # equalize move and idle

    if sanity_plot:
        plotting.PlotSequentialInputOutputWindowedData(X, Y, parameters.window_size,
                                                       parameters.window_size // parameters.step_size)

    if one_hot:
        return X, pd.get_dummies(Y).as_matrix()
    return X, Y

# Generate X/Y Data Split for Learning and Evaluation
def GenerateTrainTestData(holdout=0.3, sanity_plot=False):
    # Get Input/Output Data
    X, Y = GenerateInputOutputData(sanity_plot)

    # Split Data
    train_test_split = np.random.rand(len(Y)) > holdout     # true: training, false: testing
    train_x = X[train_test_split]
    train_y = Y[train_test_split]
    test_x = X[~train_test_split]
    test_y = Y[~train_test_split]

    return train_x, train_y, test_x, test_y


from django.shortcuts import render
from Electronics.models import ProductType
from Garments.models import GarmentsProductType
from Cosmetics.models import CosmeticsProductType
from DailyNeeds.models import DailyNeedsProductType
from Pharmaceuticals.models import MedicineType
from .models import partner, Quote, Contact, Career, ClientOwnerForm, EmployeeOwnerForm
from django.core.files.storage import FileSystemStorage
from django.contrib import messages

# Create your views here.

#index
def index(request):
    if request.method=="POST":
        name=request.POST['name']
        companyname=request.POST['companyname']
        email=request.POST['email']
        phone=request.POST['phone']
        country=request.POST['country']
        product=request.POST['product']
        quotes =request.POST['quotes']

        if len(name)<2 or len(email)<5 or len(phone)<9 or len(product)<2 or len(quotes)<10:
            messages.error(request, "Please fill the form correctly")
        else:
            quote=Quote(name=name, companyname=companyname, email=email, phone=phone, quotes=quotes, country=country, product=product)
            quote.save()
            messages.success(request, "Your quote has been sent successfully")

    Electronicsitems= ProductType.objects.all()
    Garmentsitems= GarmentsProductType.objects.all()
    Cosmeticsitems= CosmeticsProductType.objects.all()
    DailyNeedsitems= DailyNeedsProductType.objects.all()
    Pharmaceuticals= MedicineType.objects.all()

    context= {
        'Electronicsitems': Electronicsitems,
        'Garmentsitems' : Garmentsitems,
        'Cosmeticsitems' : Cosmeticsitems,
        'DailyNeedsitems' : DailyNeedsitems,
        'Pharmaceuticals': Pharmaceuticals,
        }

    return render(request, 'index.html',context)

#about
def about(request):
    return render(request, 'about.html',)

#mission
def mission(request):
    return render(request, 'mission.html',)

#vision
def vision(request):
    return render(request, 'vision.html',)

#career
def career(request):
    if request.method=="POST":
        firstname=request.POST['firstname']
        lastname=request.POST['lastname']
        email=request.POST['email']
        phone=request.POST['phone']
        city=request.POST['city']
        position=request.POST['position']
        message =request.POST['message']
        cv =request.FILES['cv']

        if len(firstname)<2 or len(lastname)<2 or len(email)<5 or len(phone)<9 or len(position)<2 or len(message)<10:
            messages.error(request, "Please fill the form correctly")
        else:
            career=Career(firstname=firstname, lastname=lastname, email=email, phone=phone, message=message, city=city, position=position, cv=cv)
            career.save()
            messages.success(request, "Your application has been submitted successfully")

    return render(request, 'career.html',)

#policy
def policy(request):
    return render(request, 'PrivacyPolicy.html',)

#partners
def partners(request):
    partners = partner.objects.all()
    contex = {'partners': partners}

    return render(request, 'partners.html', contex)

#faq
def faq(request):
    return render(request, 'faq.html',)

#clientform
def clientform(request):
    # owner form
    if request.method=="POST":
        if 'ownerform' in request.POST:
            ownerName = request.POST['ownerName']
            ownerCompanyname = request.POST['ownerCompanyname']
            ownerDesignation = request.POST['ownerDesignation']
            ownerBusinessLocation = request.POST['ownerBusinessLocation']
            ownerBusinessType = request.POST['ownerBusinessType']
            ownerAddress = request.POST['ownerAddress']
            ownerCity = request.POST['ownerCity']
            ownerState = request.POST['ownerState']
            ownerZip = request.POST['ownerZip']
            ownerEmail = request.POST['ownerEmail']
            ownerPhone =request.POST['ownerPhone']
            ownerFax = request.POST['ownerFax']
            ownerWebsite = request.POST['ownerWebsite']
            ownerMessage = request.POST['ownerMessage']

            if len(ownerName)<2 or len(ownerCompanyname)<2 or len(ownerEmail)<5 or len(ownerPhone)<9 or len(ownerAddress)<2 or len(ownerMessage)<10:
                messages.error(request, "Please fill the form correctly")
            else:
                clientform=ClientOwnerForm(ownerName=ownerName, ownerCompanyname=ownerCompanyname, ownerDesignation=ownerDesignation, ownerBusinessLocation=ownerBusinessLocation, ownerBusinessType=ownerBusinessType, ownerAddress=ownerAddress, ownerCity=ownerCity, ownerState=ownerState, ownerZip=ownerZip, ownerEmail=ownerEmail, ownerPhone=ownerPhone, ownerFax=ownerFax, ownerWebsite=ownerWebsite, ownerMessage=ownerMessage)
                clientform.save()
                messages.success(request, "Your message has been sent successfully")

    # employee form
        elif 'employeeform' in request.POST:
            EmployeeName = request.POST['EmployeeName']
            EmployeeCompanyname = request.POST['EmployeeCompanyname']
            EmployeeDesignation = request.POST['EmployeeDesignation']
            EmployeeBusinessLocation = request.POST['EmployeeBusinessLocation']
            EmployeeBusinessType = request.POST['EmployeeBusinessType']
            EmployeeAddress = request.POST['EmployeeAddress']
            EmployeeCity = request.POST['EmployeeCity']
            EmployeeState = request.POST['EmployeeState']
            EmployeeZip = request.POST['EmployeeZip']
            EmployeeEmail = request.POST['EmployeeEmail']
            EmployeePhone =request.POST['EmployeePhone']
            EmployeeFax = request.POST['EmployeeFax']
            EmployeeWebsite = request.POST['EmployeeWebsite']

            OwnerName = request.POST['OwnerName']
            OwnerDesignation = request.POST['OwnerDesignation']
            OwnerAddress = request.POST['OwnerAddress']
            OwnerCity = request.POST['OwnerCity']
            OwnerState = request.POST['OwnerState']
            OwnerZip = request.POST['OwnerZip']
            OwnerEmail = request.POST['OwnerEmail']
            OwnerPhone =request.POST['OwnerPhone']

            EmployeeMessage = request.POST['EmployeeMessage']

            if len(EmployeeName)<2 or len(EmployeeCompanyname)<2 or len(EmployeeEmail)<5 or len(EmployeePhone)<9 or len(EmployeeAddress)<2 or len(EmployeeMessage)<10:
                messages.error(request, "Please fill the form correctly")
            else:
                employeeform=EmployeeOwnerForm(EmployeeName=EmployeeName, EmployeeCompanyname=EmployeeCompanyname, EmployeeDesignation=EmployeeDesignation, EmployeeBusinessLocation=EmployeeBusinessLocation, EmployeeBusinessType=EmployeeBusinessType, EmployeeAddress=EmployeeAddress, EmployeeCity=EmployeeCity, EmployeeState=EmployeeState, EmployeeZip=EmployeeZip, EmployeeEmail=EmployeeEmail, EmployeePhone=EmployeePhone, EmployeeFax=EmployeeFax, EmployeeWebsite=EmployeeWebsite, EmployeeMessage=EmployeeMessage,
                OwnerName=OwnerName, OwnerDesignation=OwnerDesignation, OwnerAddress=OwnerAddress, OwnerCity=OwnerCity, OwnerState=OwnerState, OwnerZip=OwnerZip, OwnerEmail=OwnerEmail, OwnerPhone=OwnerPhone)
                employeeform.save()
                messages.success(request, "Your message has been sent successfully")

    return render(request, 'clientform.html',)

#contact
def contact(request):
    if request.method=="POST":
        firstname = request.POST['firstname']
        lastname = request.POST['lastname']
        companyname = request.POST['companyname']
        email = request.POST['email']
        phoneno = request.POST['phoneno']
        address = request.POST['address']
        address2 = request.POST['address2']
        city = request.POST['city']
        state = request.POST['state']
        zipcode = request.POST['zipcode']
        message = request.POST['message']
        if len(firstname)<2 or len(lastname)<2 or len(email)<5 or len(phoneno)<9 or len(address)<2 or len(message)<10:
            messages.error(request, "Please fill the form correctly")
        else:
            contact=Contact(firstname=firstname, lastname=lastname, companyname=companyname, email=email, phoneno=phoneno, city=city, state=state, zipcode=zipcode, address=address, address2=address2, message=message)
            contact.save()
            messages.success(request, "Your message has been sent successfully")

    return render(request, 'contact.html',)

#quote
def quote(request):
    if request.method=="POST":
        name=request.POST['name']
        companyname=request.POST['companyname']
        email=request.POST['email']
        phone=request.POST['phone']
        country=request.POST['country']
        product=request.POST['product']
        quotes =request.POST['quotes']

        if len(name)<2 or len(email)<5 or len(phone)<9 or len(product)<2 or len(quotes)<10:
            messages.error(request, "Please fill the form correctly")
        else:
            quote=Quote(name=name, companyname=companyname, email=email, phone=phone, quotes=quotes, country=country, product=product)
            quote.save()
            messages.success(request, "Your quote has been sent successfully")
    return render(request, 'quote.html',)


from random import randint
from time import sleep

computador = randint(0,5) #computador "PENSA UM NÚMERO"

print('-=-' * 25)
print('Estou pensando em um número entre 0 e 5. Tente adivinhar o número!')
print('-=-' * 25)

jogador = int(input('Em que número eu pensei? '))
print('PROCESSANDO...')
sleep(3)

if computador == jogador:
    print(f'Parabéns! Você venceu! O número {jogador} escolhido por você é o mesmo {computador} \n'
          f'escolhido pelo computador')

else:
    print(f'Você perdeu! O número {computador} escolhido pelo computador é diferente de seu número {jogador},\n'
          f'tente novamente!')


###################
#logristic regression test
####################

import numpy as np
import matplotlib.pyplot as pl
import pandas as pd
import csv
import random
from scipy.optimize import fmin_l_bfgs_b

maxtimesvalue = 1
realmaxtimes = 1
times = 1
size = 18090
moredimen = 0
pickfortest = 500
resettimes = times
accuracynewreate = 1
judgenumber = 0.0
traininglable = 'training_labels.csv'
labletonumberdic = {'Photography': 1, 'Media and Video': 22, 'Cards and Casino': 24, 'Lifestyle': 28, 'Tools': 9,
                    'Entertainment': 18, 'Education': 11, 'Finance': 27, 'Personalization': 13, 'Brain and Puzzle': 10,
                    'Health and Fitness': 5, 'Communication': 6, 'Music and Audio': 8, 'Comics': 21,
                    'Arcade and Action': 4,
                    'Travel and Local': 17, 'Shopping': 7, 'Medical': 15, 'Productivity': 19, 'Books and Reference': 20,
                    'Libraries and Demo': 3, 'Sports': 23, 'Racing': 14, 'Casual': 16, 'News and Magazines': 12,
                    'Sports Games': 30, 'Weather': 29, 'Social': 2, 'Transportation': 26, 'Business': 25}
########
#this dic was choice randomly. just to give a number to lables
##########
def logistic_func(theta, x):
    return float(1) / (1 + np.math.e**(-x.dot(theta)))
def log_gradient(theta, x, y):
    first_calc = logistic_func(theta, x) - np.squeeze(y)
    final_calc = first_calc.T.dot(x)
    return final_calc
def cost_func(theta, x, y):
    log_func_v = logistic_func(theta,x)
    y = np.squeeze(y)
    step1 = y * np.log(log_func_v)
    step2 = (1-y) * np.log(1 - log_func_v)
    final = -step1 - step2
    return np.mean(final)
def grad_desc(theta_values, X, y, lr=.001, converge_change=.00001):
    #normalize
    X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
    #setup cost iter
    cost_iter = []
    cost = cost_func(theta_values, X, y)
    cost_iter.append([0, cost])
    change_cost = 1
    i = 1
    while(change_cost > converge_change):
        old_cost = cost
        theta_values = theta_values - (lr * log_gradient(theta_values, X, y))
        cost = cost_func(theta_values, X, y)
        cost_iter.append([i, cost])
        change_cost = old_cost - cost
        i+=1
    return theta_values, np.array(cost_iter)
def pred_values(theta, X, hard):
    #normalize
    X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
    pred_prob = logistic_func(theta, X)
    pred_value = np.where(pred_prob >= .5, 1, 0)
    if hard:
        return pred_value
    return pred_prob

def hx(weight, x):
    z = weight.T
    Z = z.dot(x)
    result = 1.0 / (1 + np.math.exp(-Z))
    return result

def H(weight, dataarray):
    resulth = 0
    coloumnumber = len(dataarray)
    for i in range(coloumnumber):
        temp = hx(weight, dataarray[i])
        temp2 = dataarray[i].dot(dataarray[i].T)
        resulth = resulth + temp * (1 - temp) * temp2
    resultfinal = resulth / coloumnumber
    return resultfinal

def J(weight, dataarray, y):
    coloumnumber = len(dataarray)
    wordnumber = len(dataarray[0])
    resultJ = np.zeros((wordnumber, 1))
    for i in range(coloumnumber):
        temm = np.asarray([dataarray[i]])
        resultJ = resultJ + ((hx(weight, dataarray[i]) - y[i]) * temm).T
    resultfinal = resultJ / coloumnumber
    return resultfinal

def getlucky(num, list):
    choicelist = []
    if (len(list) < num):
        choicelist = list
    else:
        choicelist = list[0:num]
        return choicelist
    listnumber = len(list)
    while listnumber < num:
        numberinlist = False
        choicearraynumber = random.randint(0, choiceamount)
        for q in choicelist:
            if choicearraynumber == q:
                numberinlist = True

        if numberinlist == False:
            choicelist.append(choicearraynumber)
            listnumber = listnumber + 1
    return choicelist

def creatematrixfromlist(a, X):
    X2 = np.zeros([len(a), len(X[0])])
    for i in range(len(a)):
        X2[i] = X[a[i]]

    return X2

def calculat(weight, dataarray, ynew, times):
    j = 0
    for i in range(times):
        Hresult = 1 / H(weight, dataarray)

        Jresult = J(weight, dataarray, ynew)
        weight = weight - Hresult * Jresult

    return weight

def getlablearray(traininglableslocation, numberofarray2):
    readertrain = pd.read_csv(traininglableslocation, header=None, iterator=True)
    chunkSize2 = numberofarray2
    g = readertrain.get_chunk(chunkSize2).values
    numberlist = g[:, 1:2]
    return numberlist

def getlablesdic(traininglableslocation, numberofarray2):
    readertrain = pd.read_csv(traininglableslocation, header=None, iterator=True)
    labletonumberdic = {}
    numberoflable = 1
    chunkSize2 = numberofarray2
    numberlist = []
    g = readertrain.get_chunk(chunkSize2).values
    for G in g:
        y = G[1:2]
        jud = True
        for ele in labletonumberdic:
            if ele == y[0]:
                numberlist.append(labletonumberdic[ele])
                jud = False
        if jud:
            labletonumberdic[y[0]] = numberoflable
            numberlist.append(numberoflable)
            numberoflable = numberoflable + 1
    return labletonumberdic

################################

################################
def getaverage(X, m):
    for i in range(len(X)):
        X[i] = X[i] / m
    return X

def getdistance(X, central):
    distance = 0.0
    for i in range(len(X)):
        distance = distance + (X[i] - central[i]) ** 2
    return distance

def classfy(X, lable):
    diclable = {}
    diclablesum = {}
    lablecentral = {}
    jud = False
    for i in range(len(lable)):

        if len(diclable) == 0:
            diclable[lable[i][0]] = 1
            diclablesum[lable[i][0]] = X[i]
        else:
            for lableindic in diclable:
                if lable[i][0] == lableindic:
                    jud = True
            if jud:
                diclable[lable[i][0]] = diclable[lable[i][0]] + 1
                diclablesum[lable[i][0]] = diclablesum[lable[i][0]] + X[i]
            else:
                diclable[lable[i][0]] = 1
                diclablesum[lable[i][0]] = X[i]
        jud = False
    for lableindic in diclable:
        lablecentral[lableindic] = getaverage(diclablesum[lableindic], diclable[lableindic])

    return lablecentral

def lablebelong(X, lablecentral):
    devidelength = 0
    lable = ''
    for ele in lablecentral:
        temp = 1 / (getdistance(X, lablecentral[ele]) + 0.01)
        if temp > devidelength:
            lable = ele
            devidelength = temp
    return lable

################################
def returnwronglist(wronglist, lucyforest):
    returnlist = []
    for i in wronglist:
        returnlist.append(lucyforest[i])
    return returnlist

################################
def createlablefromlist(a, X):
    lables = []
    for i in range(len(a)):
        lables.append(X[a[i]])
    return lables

################################
def matrixadd(X, k):
    if k == 0:
        return X
    else:
        m = len(X)
        n = len(X[0])
        newn = n + k ** 2
        newX = np.zeros([m, newn])
        for i in range(m):
            for j in range(n):
                newX[i][j] = X[i][j]
        for i in range(m):
            for j in range(k):
                for j2 in range(k):
                    newX[i][n + j * k + j2] = X[i][j] * X[i][j2]
        return newX

################################
def addnormal(X):
    m = len(X)
    n = len(X[0])
    newX = np.ones([m, n + 1])
    for i in range(m):
        for j in range(1, n + 1):
            newX[i][j] = X[i][j - 1]
    return newX

################################

compfinalmatrix = [['' for i in range(31)] for j in range(31)]

for i in range(1, 31):
    for j in range(1, 31):
        compfinalmatrix[i][j] = 0
for i in range(1, 31):
    for ele in labletonumberdic:
        if labletonumberdic[ele] == i:
            compfinalmatrix[0][i] = ele
for i in range(1, 31):
    for ele in labletonumberdic:
        if labletonumberdic[ele] == i:
            compfinalmatrix[i][0] = ele

for trainsetnumber in range(10):
    trainfile = trainsetnumber + 1
    combinetime = 0
    for i in range(10):
        if (i + 1) != trainfile:
            I = str(i + 1)
            trainfiledataread = '..\\temp\\train_data_reduce' + I + '.csv'
            d = pd.read_csv(trainfiledataread, header=None, delimiter=',').values
            trainpart = np.float_(d[:])
            if combinetime == 0:
                new_datamatrix = trainpart

            else:
                new_datamatrix = np.vstack([new_datamatrix, trainpart])

            trainfilelablesread = '..\\temp\\train_lables_' + I + '.csv'
            Y = getlablearray(trainfilelablesread, 2100)
            if combinetime == 0:
                new_labelsmatrix = Y
                combinetime = combinetime + 1
            else:
                new_labelsmatrix = np.vstack([new_labelsmatrix, Y])
        else:
            I = str(i + 1)
            testfiledataread = '..\\temp\\train_data_reduce' + I + '.csv'
            dt = pd.read_csv(testfiledataread, header=None, delimiter=',').values
            testdatamatrix = np.float_(dt[:])
            testfilelablesread = '..\\temp\\train_lables_' + I + '.csv'
            testlablematrix = getlablearray(testfilelablesread, 2100)
    if trainfile == 11:
        testfiledataread = '..\\temp\\test_data_reduce.csv'
        dt = pd.read_csv(testfiledataread, header=None, delimiter=',').values
        testdatamatrix = np.float_(dt[:])

    X = new_datamatrix
    print(len(X))
    # X = matrixadd(X, moredimen)
    # X = addnormal(X)
    numberofarray = len(X)
    choiceamount = numberofarray - 1
    weightlenth = len(X[0])
    Y = new_labelsmatrix
    Weightfinalt = np.zeros([weightlenth, len(labletonumberdic)])
    weight = np.zeros((weightlenth, 1))
    listofweight = []
    finifhprecent = 0
    outputsequence = 0
    for ele in labletonumberdic:
        Ytrain = np.zeros([numberofarray, 1])
        for i in range(numberofarray):
            if Y[i] == ele:
                Ytrain[i] = 1
            else:
                Ytrain[i] = 0

        betas = weight.T[0]
        fitted_values, cost_iter = grad_desc(betas, X, Ytrain)
        weight = np.array(fitted_values).reshape(len(fitted_values), 1)
        ele2 = labletonumberdic[ele]
        outputsequence = outputsequence + 1
        Weightfinalt[:, ele2 - 1:ele2] = weight
        weight = np.zeros((weightlenth, 1))

    X2 = testdatamatrix
    #X2 = matrixadd(X2, moredimen)
    #X2 = addnormal(X2)
    Y3 = testlablematrix
    finalresultoflr=np.zeros([len(X2),30])
    for unique in range(30):
        fitted=Weightfinalt[:,unique:unique+1].T[0]
        result = pred_values(fitted, X2, hard=False)
        finalresultoflr[:,unique:unique+1]=np.array(result).reshape(len(result), 1)

    static = 0
    resultassry=[]
    for array in range(len(finalresultoflr)):
        maxnumber = judgenumber
        maxnumbervalue = 0
        for number in range(len(finalresultoflr[array])):
            if finalresultoflr[array][number] > maxnumber:
                maxnumber = finalresultoflr[array][number]
                maxnumbervalue = number

        for ele in labletonumberdic:
            if labletonumberdic[ele] == (maxnumbervalue + 1):
                resultassry.append(ele)
                if ele == Y3[array]:
                    static = static + 1

    hahahaha = static / len(finalresultoflr)
    print(static)
    print(trainfile,'finalresult=', hahahaha)

    comparematrix = np.zeros([30, 30])
    print(resultassry)
    for i in range(len(Y3)):
        k = labletonumberdic[resultassry[i]] - 1
        temp = Y3[i][0]
        k2 = labletonumberdic[temp] - 1
        comparematrix[k2][k] = comparematrix[k2][k] + 1

    for i in range(1, 31):
        for j in range(1, 31):
            compfinalmatrix[i][j] =compfinalmatrix[i][j]+comparematrix[i - 1][j - 1]

for i in range(1, 31):
    for j in range(1, 31):
        compfinalmatrix[i][j] = compfinalmatrix[i][j]/10

trainfilewrite = '..\\temp\\result_matrix_of_GDLR.csv'
with open(trainfilewrite, 'w', newline='') as csvfile:
    spamwriter = csv.writer(csvfile, delimiter=',', quotechar='\n', quoting=csv.QUOTE_MINIMAL)
    spamwriter.writerows(compfinalmatrix)


import os
import pickle
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from generators.gpt3_5_turbo import ChatGPT
from generators.openAiParameters import Parameters
from detectors.turnitin_detector import TurnitIn
from detectors.open_ai_classifier import OpenAiClassifier
import logging

class ParameterOptimizer:
    def __init__(self, generator, turnitin, gpt2_detector, openai_classifier, max_tokens, param_name, param_bounds, param_step, texts_dir_path, result_dir_path,
                 log_dir_path, train_prompts):
        self.generator = generator
        self.turnitin = turnitin
        self.gpt2_detector = gpt2_detector
        self.openai_classifier = openai_classifier
        self.max_tokens = max_tokens
        self.param_name = param_name
        self.param_bounds = param_bounds
        self.param_step = param_step
        self.texts_dir_path = texts_dir_path
        self.result_dir_path = result_dir_path
        self.log_dir_path = log_dir_path
        self.train_prompts = train_prompts
        self.file_paths_dict = {}
        logging.basicConfig(filename=os.path.join(self.log_dir_path, 'parameter_optimization.log'), level=logging.INFO,
                            format='%(asctime)s %(message)s')

    def generate_and_save_texts(self):
        param_values = np.arange(self.param_bounds[0], self.param_bounds[1] + self.param_step, self.param_step)
        for param_value in param_values:
            param_value = round(param_value.item(), 2)
            parameters = Parameters(max_tokens=self.max_tokens)
            parameters.__dict__[self.param_name] = param_value
            print(parameters.__dict__)
            logging.info(parameters.__dict__)

            for idx, prompt in enumerate(self.train_prompts):
                file_name = f"{self.param_name}_{param_value}_prompt_{idx}.txt"
                self.generator.generate_prompt_save_txt(prompt, parameters, self.texts_dir_path, file_name)
                logging.info(prompt)
                file_path = os.path.join(self.texts_dir_path, file_name)
                # Add the file path to the dictionary
                self.file_paths_dict.setdefault(f"{self.param_name}_{param_value}", []).append(file_path)

        # Return file paths
        return self.file_paths_dict

    def submit_and_calculate_scores_turnitin(self, file_paths_dict, prompts_list, waittime, single_file_waittime):
        average_scores_dict = {}
        null_counts_dict = {}  # New dictionary to count null values
        null_present = True  # to keep the loop going until no nulls found

        while null_present:
            null_present = False  # reset for each complete run
            scores_dict = self.turnitin.submit_and_scrape_existing_files(file_paths_dict, waittime)
            # Creating a temporary dictionary to hold file paths with None scores
            file_paths_dict_temp = {}
            for param_value, scores in scores_dict.items():
                logging.info(f"Scores for {self.param_name} = {param_value}: {scores}")
                new_scores = [round(score / 100, 2) if score is not None else None for score in scores]

                if param_value not in null_counts_dict:
                    null_counts_dict[param_value] = 0  # Initialize count if not exist

                # Regenerate text for null scores and re-check
                for i, score in enumerate(new_scores):
                    if score is None:
                        null_present = True  # null found, another run required
                        null_counts_dict[param_value] += 1  # Increase the count
                        file_path = file_paths_dict[param_value][i]
                        directory, filename = os.path.split(file_path)
                        prompt_index = int(filename.split('_')[-1].split('.')[0])  # Extract prompt index from filename
                        prompt = prompts_list[prompt_index]  # Retrieve the prompt from the prompt list
                        print(f"Null text: {file_path}, Score value: {score}, Prompt index: {prompt_index}")

                        # Prepare parameters
                        parameters = Parameters(max_tokens=self.max_tokens)
                        parameters.__dict__[self.param_name] = float(param_value.split('_')[-1])

                        # Generate new texts
                        print(
                            f"generating new text for {filename} with filepath={file_path} and parameters: {parameters}")
                        self.generator.generate_prompt_save_txt(prompt, parameters, directory, filename)

                        # Add the regenerated file's path to the temporary dictionary
                        if param_value not in file_paths_dict_temp:
                            file_paths_dict_temp[param_value] = []
                        file_paths_dict_temp[param_value].append(file_path)

                        # Check the scores again
                        single_file_dict = {param_value: [file_path]}  # Adjust the dict for the specific file
                        new_score_dict = self.turnitin.submit_and_scrape_existing_files(single_file_dict,
                                                                                single_file_waittime)
                        new_scores[i] = new_score_dict[param_value][0]

                total = sum(score for score in new_scores if score is not None)
                average_scores_dict[param_value] = total / len(new_scores)
            # Update file_paths_dict to include only file paths which had None scores in the current iteration
            file_paths_dict = file_paths_dict_temp

        return average_scores_dict, null_counts_dict

    def submit_and_calculate_scores_openai(self, file_paths_dict):
        scores_dict = {}
        for param, file_paths in file_paths_dict.items():
            scores_list = []
            for file_path in file_paths:
                print(f"OpenAI: {param}: {file_path}")
                with open(file_path, 'r', encoding='utf-8') as file:  # Open each file
                    file_content = file.read()  # Read the content of the file
                    score = self.openai_classifier.submit_and_scrape(file_content)  # Execute the submit_and_scrape function
                    scores_list.append(score)  # Append the score to the scores list
            new_scores = [round(score, 2) for score in scores_list]
            scores_dict[param] = sum(new_scores) / len(scores_list)  # Assign the scores list to the corresponding key in the dictionary
        return scores_dict

    def submit_and_calculate_scores_gpt2(self, file_paths_dict):
        scores_dict = {}
        for param, file_paths in file_paths_dict.items():
            scores_list = []
            for file_path in file_paths:
                print(f"GPT2: {param}: {file_path}")
                with open(file_path, 'r', encoding='utf-8') as file:  # Open each file
                    file_content = file.read()  # Read the content of the file
                    score = self.gpt2_detector.submit_and_scrape(
                        file_content).fake_probability  # Execute the submit_and_scrape function
                    scores_list.append(score)  # Append the score to the scores list
            new_scores = [round(score, 2) for score in scores_list]
            scores_dict[param] = sum(new_scores) / len(scores_list)
        return scores_dict

    def save_scores_to_csv(self, scores_dict, detector_name, param_name):
        new_scores_dict = {float(key.split('_')[-1]): value for key, value in scores_dict.items()}
        df = pd.DataFrame(list(new_scores_dict.items()), columns=[param_name, 'Score'])
        df.to_csv(f'{self.result_dir_path}/{self.param_name}_{detector_name}_scores.csv', index=False)

    def load_scores_from_csv(self, detector_name, param_name):
        # Define the path to the csv file
        file_path = f'{self.result_dir_path}/{self.param_name}_{detector_name}_scores.csv'

        # Read the csv file into a DataFrame
        df = pd.read_csv(file_path)

        # Convert the DataFrame back into a dictionary
        scores_dict = {f"{self.param_name}_{row[param_name]}": row['Score'] for index, row in df.iterrows()}

        return scores_dict

    def plot_scores(self, scores_dict, detector_name, param_name):
        new_scores_dict = {float(key.split('_')[-1]): value for key, value in scores_dict.items()}
        plt.plot(list(new_scores_dict.keys()), list(new_scores_dict.values()), label=f'{detector_name} score')
        plt.legend()
        plt.xlabel(param_name)
        plt.ylabel(f'{detector_name} score')
        plt.title('Parameter tuning results')
        plt.savefig(f'{self.result_dir_path}/{self.param_name}_{detector_name}_scores.png')
        plt.show()

    def save_scores_to_combined_csv(self, scores_dicts, detector_names, param_name):
        df_combined = None
        for scores_dict, detector_name in zip(scores_dicts, detector_names):
            new_scores_dict = {float(key.split('_')[-1]): value for key, value in scores_dict.items()}
            df = pd.DataFrame(list(new_scores_dict.items()), columns=[param_name, detector_name])
            if df_combined is None:
                df_combined = df
            else:
                df_combined = df_combined.merge(df, on=param_name)
        df_combined.to_csv(f'{self.result_dir_path}/{self.param_name}_combined_scores.csv', index=False)

    def plot_combined_scores(self, scores_dicts, detector_names, param_name):
        plt.figure(figsize=(5, 4))
        for scores_dict, detector_name in zip(scores_dicts, detector_names):
            new_scores_dict = {float(key.split('_')[-1]): value for key, value in scores_dict.items()}
            plt.plot(list(new_scores_dict.keys()), list(new_scores_dict.values()), label=f'{detector_name} score')
        plt.legend()
        plt.xlabel(param_name)
        plt.ylabel('Score')
        plt.title('Parameter tuning results')
        plt.savefig(f'{self.result_dir_path}/{self.param_name}_combined_scores.png')
        plt.show()

    def save_null_counts_to_csv(self, null_counts_dict, param_name):
        df = pd.DataFrame(list(null_counts_dict.items()), columns=[param_name, 'Null Counts'])
        df.to_csv(f'{self.result_dir_path}/{self.param_name}_null_counts.csv', index=False)

    def plot_null_counts(self, null_counts_dict, param_name):
        plt.bar(list(null_counts_dict.keys()), list(null_counts_dict.values()), label='Null counts')
        plt.xlabel(param_name)
        plt.ylabel('Null counts')
        plt.title('Number of documents unclassified by Turnitin')
        plt.savefig(f'{self.result_dir_path}/{self.param_name}_null_counts.png')
        plt.show()


import os
import shutil
import tempfile
import unittest
from unittest.mock import call, patch

import pytest
from clinv.inventory import Inventory
from yaml import YAMLError

class ClinvBaseTestClass(object):
    """
    Base class to setup the setUp and tearDown methods for the test cases.
    """

    def setUp(self):
        self.tmp = tempfile.mkdtemp()
        self.inventory_dir = self.tmp
        self.logging_patch = patch("clinv.inventory.logging", autospect=True)
        self.logging = self.logging_patch.start()
        self.print_patch = patch("clinv.inventory.print", autospect=True)
        self.print = self.print_patch.start()

    def tearDown(self):
        self.logging_patch.stop()
        self.print_patch.stop()
        shutil.rmtree(self.tmp)

class InventoryBaseTestClass(ClinvBaseTestClass):
    """
    Base class to setup the setUp and tearDown methods for the Inventory test
    cases.
    """

    def setUp(self):
        super().setUp()
        self.source_data_path = os.path.join(self.inventory_dir, "source_data.yaml",)
        self.user_data_path = os.path.join(self.inventory_dir, "user_data.yaml",)

    def tearDown(self):
        super().tearDown()

class TestInventory(InventoryBaseTestClass, unittest.TestCase):
    """
    Test class to assess that the Inventory class works as expected
    """

    def setUp(self):
        super().setUp()
        self.source_plugins = []

        self.inv = Inventory(self.inventory_dir, self.source_plugins)

    def tearDown(self):
        super().tearDown()

    def test_init_sets_inventory_dir(self):
        self.assertEqual(self.inv.inventory_dir, self.inventory_dir)

    def test_init_sets_source_plugins(self):
        self.assertEqual(self.inv._source_plugins, self.source_plugins)

    def test_init_sets_source_data_path(self):
        self.assertEqual(self.inv.source_data_path, self.source_data_path)

    def test_init_sets_user_data_path(self):
        self.assertEqual(self.inv.user_data_path, self.user_data_path)

    def test_yaml_saving(self):
        save_file = os.path.join(self.tmp, "yaml_save_test.yaml")
        dictionary = {"a": "b", "c": "d"}
        self.inv._save_yaml(save_file, dictionary)
        with open(save_file, "r") as f:
            self.assertEqual("a: b\nc: d\n", f.read())

    def test_load_yaml(self):
        with open(self.source_data_path, "w") as f:
            f.write("test: this is a test")
        yaml_content = self.inv._load_yaml(self.source_data_path)
        self.assertEqual(yaml_content["test"], "this is a test")

    @patch("clinv.inventory.os")
    @patch("clinv.inventory.yaml")
    def test_load_yaml_raises_error_if_wrong_format(self, yamlMock, osMock):
        yamlMock.safe_load.side_effect = YAMLError("error")

        with self.assertRaises(YAMLError):
            self.inv._load_yaml(self.source_data_path)
        self.assertEqual(
            str(self.logging.getLogger.return_value.error.mock_calls),
            str([call(YAMLError("error"))]),
        )

    @patch("clinv.inventory.open")
    def test_load_yaml_raises_error_if_file_not_found(self, openMock):
        openMock.side_effect = FileNotFoundError()

        with self.assertRaises(FileNotFoundError):
            self.inv._load_yaml(self.source_data_path)
        self.assertEqual(
            str(self.logging.getLogger.return_value.error.mock_calls),
            str([call("Error opening yaml file {}".format(self.source_data_path))]),
        )

    @patch("clinv.inventory.Inventory._load_yaml")
    def test_load_loads_source_data(self, loadMock):
        self.inv.load()
        self.assertTrue(call(self.source_data_path) in loadMock.mock_calls)
        self.assertEqual(self.inv.source_data, loadMock())

    @patch("clinv.inventory.Inventory._load_yaml")
    def test_load_loads_user_data(self, loadMock):
        self.inv.load()
        self.assertTrue(call(self.user_data_path) in loadMock.mock_calls)
        self.assertEqual(self.inv.user_data, loadMock())

    @patch("clinv.inventory.Inventory._load_yaml")
    @patch("clinv.inventory.Inventory._load_plugins")
    @patch("clinv.inventory.Inventory._generate_inventory_objects")
    def test_load_loads_plugins(self, invMock, loadMock, yamlMock):
        self.inv.load()
        self.assertTrue(loadMock.called)

    @patch("clinv.inventory.Inventory._load_yaml")
    @patch("clinv.inventory.Inventory._generate_inventory_objects")
    def test_load_generates_inventory_objects(self, invMock, yamlMock):
        self.inv.load()
        self.assertTrue(invMock.called)

    @patch("clinv.inventory.Inventory._save_yaml")
    def test_save_saves_source_data(self, saveMock):
        self.inv.save()
        self.assertTrue(
            call(self.source_data_path, self.inv.source_data) in saveMock.mock_calls
        )

    @patch("clinv.inventory.Inventory._save_yaml")
    def test_save_saves_user_data(self, saveMock):
        self.inv.save()
        self.assertTrue(
            call(self.user_data_path, self.inv.user_data) in saveMock.mock_calls
        )

class TestInventoryPluginLoad(InventoryBaseTestClass, unittest.TestCase):
    """
    Test class to assess that the Inventory plugins methods work as expected
    """

    def setUp(self):
        super().setUp()
        self.source_patch = patch("clinv.inventory.aws.Route53src", autospect=True)
        self.source = self.source_patch.start()
        self.source.return_value.id = "source_id"
        self.source_plugins = [self.source]

        self.inv = Inventory(self.inventory_dir, self.source_plugins)
        self.inv._load_plugins()

    def tearDown(self):
        super().tearDown()
        self.source_patch.stop()

    def test_load_plugins_creates_expected_list_if_user_data(self):
        self.inv.user_data = {"source_id": {"user": "data"}}

        self.inv._load_plugins()

        self.assertEqual(
            self.source.assert_called_with(source_data={}, user_data={"user": "data"},),
            None,
        )
        self.assertEqual(self.inv.sources, [self.source.return_value])

    def test_load_plugins_creates_expected_list_if_no_user_data(self):
        self.inv.user_data = {}

        self.inv._load_plugins()

        self.assertEqual(
            self.source.assert_called_with(source_data={}, user_data={}), None
        )
        self.assertEqual(self.inv.sources, [self.source.return_value])

    def test_generate_source_data_loads_data_from_plugins(self):
        self.inv._generate_source_data()

        self.assertEqual(
            self.inv.source_data,
            {"source_id": self.source.return_value.generate_source_data.return_value},
        )

    def test_generate_user_data_loads_data_from_plugins(self):
        self.inv._generate_user_data()

        self.assertEqual(
            self.inv.user_data,
            {"source_id": self.source.return_value.generate_user_data.return_value},
        )

    def test_generate_inventory_objects_loads_data_from_plugins(self):
        self.inv._generate_inventory_objects()

        self.assertEqual(
            self.inv.inv,
            {"source_id": self.source.return_value.generate_inventory.return_value},
        )

    @patch("clinv.inventory.Inventory._generate_inventory_objects")
    @patch("clinv.inventory.Inventory._generate_user_data")
    @patch("clinv.inventory.Inventory._generate_source_data")
    @patch("clinv.inventory.Inventory.save")
    @patch("clinv.inventory.Inventory._load_yaml")
    def test_generate_loads_data_from_plugins(
        self, loadMock, saveMock, sourceMock, userMock, inventoryMock,
    ):
        self.inv.generate()

        self.assertTrue(saveMock.called)
        self.assertTrue(sourceMock.called)
        self.assertTrue(userMock.called)
        self.assertTrue(inventoryMock.called)
        self.assertTrue(call(self.user_data_path) in loadMock.mock_calls)

    @pytest.mark.skip("Wait till we don't use mocks ")
    def test_generate_inventory_accepts_resource_type(self):

        pass


#!/usr/bin/env python3
def combinations_array(num_array, r):
    import itertools
    return list(itertools.combinations(num_array, r))

def main():
    A, B, C, D, E = map(int,input().split())
    ans = set()

    for itm in combinations_array([A,B,C,D,E], 3):
        ans.add(sum(list(itm)))

    ans = list(ans)
    ans.sort(reverse=True)
    return ans[2]

print(main())

# self AC


from Service import CameraManager
import cv2
import time

cv2.startWindowThread()
CameraManager.createCamera(name="TEST", url=0, initial_mode="MODE")
isThis = False
while True:
    if CameraManager.getStatus("TEST"):
        CameraManager.resumeCamera("TEST")
        time.sleep(1)
        isThis = True
        break

while True:
    if isThis:
        frame = CameraManager.getCameraFrame("TEST")
        cv2.imshow("Test", frame)

    if cv2.waitKey(1) == ord('p'):
        isThis = False
        CameraManager.pauseCamera("TEST")

    if cv2.waitKey(1) == ord('s'):
        isThis = True
        CameraManager.resumeCamera("TEST")

    if cv2.waitKey(1) == ord('q'):
        CameraManager.cleanCamera("TEST")
        cv2.destroyAllWindows()
        break


import numpy as np
import random
import pickle

import torch.nn.functional as functional
import torch
from torch.utils.data import Dataset,DataLoader
from torch.autograd import Variable
import torch.nn as nn
import sys

class Encoder(nn.Module):
    def __init__(self, input_size, hidden_size, word_len,embed_size):
        super(Encoder, self).__init__()
        self.embed = nn.Embedding(word_len, embed_size)
        self.gru = nn.GRU(embed_size, hidden_size, 1, batch_first = True,bidirectional=True)
    def forward(self, seq, hid):
        out = self.embed(seq)
        out, hid = self.gru(out, hid)
        return out, hid

class Decoder(nn.Module):
    def __init__(self, input_size, hidden_size, word_len, sent_limit,embed_size):
        super(Decoder, self).__init__()
        self.embed = nn.Embedding(word_len, embed_size)
        self.lstm = nn.GRU(hidden_size//2, hidden_size, 1, batch_first = True)
        self.fc1 = nn.Linear(hidden_size, hidden_size)
        self.fc2 = nn.Linear(hidden_size, word_len)

        self.attn = nn.Linear(hidden_size + embed_size, sent_limit)
        self.attn_com = nn.Linear(hidden_size + embed_size, hidden_size//2)

        self.sent_limit = sent_limit
        self.softmax = nn.LogSoftmax(dim = 1)
        self.word_len = word_len
        self.hidden_size = hidden_size
        self.embed_size = embed_size
    def forward(self, enco_out, emb, hid):
        attn_weight = torch.cat((emb, hid),2) #[1, 40, 800]
        #print(attn_weight.size())
        attn_weight = self.attn(attn_weight)
        attn_weight = functional.softmax(attn_weight,dim = 2) #[1, 40, 12]

#        print(attn_weight.size())
#        print(enco_out.size())
        attn_applied = torch.bmm(attn_weight.view(-1,1,self.sent_limit), enco_out) #torch.Size([40, 1, 400])

        #print('in decoder :',attn_applied.size())
        #exit()
        out = torch.cat((emb.view(-1,1,self.embed_size), attn_applied),2)
        out = self.attn_com(out)
        out = functional.relu(out) #[40,1,400]
        #print('in decoder :',out.size())

        out, hid = self.lstm(out, hid)
        #out = self.fc1(out)
        #out = functional.relu(out)
        out = self.fc2(out)
        out = out.view(-1, self.word_len)
        out = self.softmax(out)
        emb = self.embed(torch.max(out, 1)[1])
        emb = emb.view(-1,1,self.embed_size)
        #print(emb.size())
        #exit()
        return out, emb, hid


# -*- coding: utf-8 -*-
"""
Peter Nilson 2017 -- http://www.github.com/petenilson
"""
from setuptools import setup

packages = [
    'pexelpie',
]

with open('requirements.txt') as f:
    required = f.read().splitlines()

setup(
    name='pexelpie',
    version='0.0.1.1',
    description='python client for Pexel images',
    author='Peter Nilson',
    author_email='petenilson@gmail.com',
    url='https://github.com/petenilson/pexelpie/',
    license='MIT',
    packages=packages,
    include_package_data=True,
    install_requires=required,
)


import requests
import datetime as dt
from datetime import timedelta

USERNAME = "yanjun"
TOKEN = "<TOKEN>"
GRAPH_ID = "yjgraph1"
pixela_endpoint = "https://pixe.la/v1/users"

user_params = {
    "token": TOKEN,
    "username": USERNAME,
    "agreeTermsOfService": "yes",
    "notMinor": "yes"
}

# response = requests.post(url=pixela_endpoint, json=user_params)
# print(response.text)

graph_endpoint = f"{pixela_endpoint}/{USERNAME}/graphs"

graph_config = {
    "id": GRAPH_ID,
    "name": "Cycling Graph",
    "unit": "Km",
    "type": "float",
    "color": "ajisai"
}

headers = {
    "X-USER-TOKEN": TOKEN
}

# response = requests.post(url=graph_endpoint, json=graph_config, headers=headers)
# print(response.text)

post_pixel_endpoint = f"{pixela_endpoint}/{USERNAME}/graphs/{GRAPH_ID}"

today = str(dt.datetime.today().date().strftime("%Y%m%d"))
yesterday = str((dt.datetime.today().date() - timedelta(1)).strftime("%Y%m%d"))

post_pixel_params = {
    "date": today,
    "quantity": input("How many kilometres did you cycled today? ")
}

# response = requests.post(url=post_pixel_endpoint, json=post_pixel_params, headers=headers)
# print(response.text)

update_pixel_endpoint = f"{post_pixel_endpoint}/{yesterday}"

update_pixel_params = {
    "quantity": "20"
}

# response = requests.put(url=update_pixel_endpoint, json=update_pixel_params, headers=headers)
# response.raise_for_status()
# print(response.text)

delete_pixel_endpoint = f"{post_pixel_endpoint}/{yesterday}"

response = requests.delete(url=delete_pixel_endpoint, headers=headers)
print(response.text)


def get_tdm_links(link_list):
    tdm_links = []

    # sometimes the link list is a none value and needs to be skipped
    if link_list == None:
        return link_list
    else:
        # otherwise work through each dictionary
        for link_dict in link_list:
            # check the intendeed application, if the intendeed application allow it, we extend our list
            if (link_dict.get('intended-application') == 'text-mining')\
            or (link_dict.get('intended-application') == 'syndication')\
            or (link_dict.get('intended-application') == 'unspecified'):
                tdm_links.append(link_dict.get('URL'))
    if tdm_links == []:
        tdm_links = None
    else:
        tdm_links = list(set(tdm_links))
    return tdm_links


#7.3. Encrypt and decrypt by means of the RSA algorithm with the following system
#parameters:
#1. p = 3, q = 11, d = 7, x = 5
#2. p = 5, q = 11, e = 3, x = 9
#Only use a pocket calculator at this stage.

p1 = 3
q1 = 11
d1 = 7
x1 = 5

N1 = p1 * q1
phi_N1 = (p1 - 1) * (q1 - 1)

e1 = pow(d1, -1, phi_N1)
print(f"\nExponente e calculado -> {e1}")

encrypt1 = pow(x1, e1, N1)
print(f"Mensaje encriptado (Caso 1): ", encrypt1)

decrypt1 = pow(encrypt1, d1, N1)
print(f"Mensaje desencriptado -> {x1} (Caso 1): ", decrypt1)

print("\n")

p2 = 5
q2 = 11
e2 = 3
x2 = 9

N2= p2 * q2
phi_N2 = (p2 - 1) * (q2 - 1)

key=pow(e2, -1, phi_N2)
print(f"Clave d calculada -> {key}")

encrypt2 = pow(x2, e2, N2)
print("Mensaje encriptado (Caso 2): ",encrypt2)

decrypt2=pow(encrypt2, key, N2)
print(f"Mensaje desencriptado -> {x2} (Caso 2): ",decrypt2)


class Solution:
    def minCharacters(self, A: str, B: str) -> int:
        countA = Counter(ord(c) - ord('a') for c in A)
        countB = Counter(ord(c) - ord('a') for c in B)

        def operation1(a, b):
            # a < i, b >= i
            res = float('inf')
            for i in range(1,26):
                count = sum(a[j] for j in range(i))
                count += sum(b[j] for j in range(i, 26))

                res = min(res, count)

            return res

        def operation3(a, b):
            res = float('-inf')

            for i in range(26):
                res = max(res, a[i] + b[i])

            return len(A) + len(B) - res

        return min(operation1(countA, countB), operation1(countB, countA), operation3(countA, countB))


import ckan.lib.app_globals as app_globals
import ckan.lib.i18n as i18n
import ckan.model as model
import ckan.plugins.toolkit as t
import datetime as d
from ckan.lib.render import TemplateNotFound
from ckan.common import OrderedDict
from pylons import config

log = __import__('logging').getLogger(__name__)
namespace = 'ckanext.federagobes'

c = t.c

class RDFController(t.BaseController):

	def view(self):

		context = {'model': model, 'session': model.Session,
					'user': c.user, 'for_view': True,
					'auth_user_obj': c.userobj}

		# Get the plugin configuration
		site_url = config.get('ckan.site_url') + config.get('ckan.root_path').replace('{{LANG}}', '')
		date_created = config.get(namespace + '.create_date')
		date_modified = d.datetime.now().isoformat()
		publisher = config.get(namespace + '.publisher')
		theme_taxonomy = config.get(namespace + '.theme_taxonomy')
		license_url = config.get(namespace + '.license_url')
		catalog_title = {}
		catalog_desc = {}

		for locale in i18n.get_locales():
			catalog_title[locale] = config.get(namespace + '.catalog_title.' + locale)
			catalog_desc[locale] = config.get(namespace + '.catalog_desc.' + locale)

		# Get the dataset catalogue
		packages = t.get_action('package_search')(context, {
				'include_private': False,
				'rows': 1000
			})
		packages = packages['results']

		t.response.headers['Content-Type'] = 'application/rdf+xml; charset=utf-8'
		return t.render('federador.xml', extra_vars={
				'site_url': site_url,
				'date_created': date_created,
				'date_modified': date_modified,
				'publisher': publisher,
				'theme_taxonomy': theme_taxonomy,
				'license_url': license_url,
				'packages': packages,
				'catalog_title': catalog_title,
				'catalog_desc': catalog_desc
			})


import datetime
import re
from importlib import import_module

from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponseRedirect
from django.shortcuts import render

from lockdown import settings

def compile_url_exceptions(url_exceptions):
    """Return a list of compiled regex objects, containing the url exceptions.

    All URLs in that list returned won't be considered as locked.
    """
    return [re.compile(p) for p in url_exceptions]

def get_lockdown_form(form_path):
    """Return a form class for a given string pointing to a lockdown form."""
    if not form_path:
        raise ImproperlyConfigured('No LOCKDOWN_FORM specified.')
    form_path_list = form_path.split(".")
    new_module = ".".join(form_path_list[:-1])
    attr = form_path_list[-1]
    try:
        mod = import_module(new_module)
    except (ImportError, ValueError):
        raise ImproperlyConfigured('Module configured in LOCKDOWN_FORM (%s) to'
                                   ' contain the form class couldn\'t be '
                                   'found.' % new_module)
    try:
        form = getattr(mod, attr)
    except AttributeError:
        raise ImproperlyConfigured('The module configured in LOCKDOWN_FORM '
                                   ' (%s) doesn\'t define a "%s" form.'
                                   % (new_module, attr))
    return form

# pylint: disable=too-many-instance-attributes
class LockdownMiddleware(object):
    """Middleware to lock down a whole Django site."""

    # pylint: disable=too-many-arguments
    def __init__(self, get_response=None, form=None, until_date=None,
                 after_date=None, logout_key=None, session_key=None,
                 url_exceptions=None, remote_addr_exceptions=None,
                 extra_context=None, **form_kwargs):
        """Initialize the middleware, by setting the configuration values."""
        if logout_key is None:
            logout_key = settings.LOGOUT_KEY
        if session_key is None:
            session_key = settings.SESSION_KEY
        self.get_response = get_response
        self.form = form
        self.form_kwargs = form_kwargs
        self.until_date = until_date
        self.after_date = after_date
        self.logout_key = logout_key
        self.session_key = session_key
        self.url_exceptions = url_exceptions
        self.remote_addr_exceptions = remote_addr_exceptions
        self.extra_context = extra_context

    def __call__(self, request):
        """Handle calls to the class instance."""
        response = self.process_request(request)

        if not response:
            response = self.get_response(request)

        return response

    # pylint: disable=too-many-locals,too-many-return-statements
    # pylint: disable=too-many-statements,too-many-branches
    def process_request(self, request):
        """Check if each request is allowed to access the current resource."""
        try:
            session = request.session
        except AttributeError:
            raise ImproperlyConfigured('django-lockdown requires the Django '
                                       'sessions framework')

        # Don't lock down if django-lockdown is disabled altogether.
        if settings.ENABLED is False:
            return None

        # Don't lock down if the client REMOTE_ADDR matched and is part of the
        # exception list.
        if self.remote_addr_exceptions:
            remote_addr_exceptions = self.remote_addr_exceptions
        else:
            remote_addr_exceptions = settings.REMOTE_ADDR_EXCEPTIONS

        if remote_addr_exceptions and \
                request.META.get('REMOTE_ADDR') in remote_addr_exceptions:
            return None

        # Don't lock down if the URL matches an exception pattern.
        if self.url_exceptions:
            url_exceptions = compile_url_exceptions(self.url_exceptions)
        else:
            url_exceptions = compile_url_exceptions(settings.URL_EXCEPTIONS)
        for pattern in url_exceptions:
            if pattern.search(request.path):
                return None

        # Don't lock down if outside of the lockdown dates.
        if self.until_date:
            until_date = self.until_date
        else:
            until_date = settings.UNTIL_DATE

        if self.after_date:
            after_date = self.after_date
        else:
            after_date = settings.AFTER_DATE

        if until_date or after_date:
            locked_date = False
            if until_date and datetime.datetime.now() < until_date:
                locked_date = True
            if after_date and datetime.datetime.now() > after_date:
                locked_date = True
            if not locked_date:
                return None

        form_data = request.POST if request.method == 'POST' else None
        if self.form:
            form_class = self.form
        else:
            form_class = get_lockdown_form(settings.FORM)
        form = form_class(data=form_data, **self.form_kwargs)

        authorized = False
        token = session.get(self.session_key)
        if hasattr(form, 'authenticate'):
            if form.authenticate(token):
                authorized = True
        elif token is True:
            authorized = True

        if authorized and self.logout_key and self.logout_key in request.GET:
            if self.session_key in session:
                del session[self.session_key]
            querystring = request.GET.copy()
            del querystring[self.logout_key]
            return self.redirect(request)

        # Don't lock down if the user is already authorized for previewing.
        if authorized:
            return None

        if form.is_valid():
            if hasattr(form, 'generate_token'):
                token = form.generate_token()
            else:
                token = True
            session[self.session_key] = token
            return self.redirect(request)

        page_data = {'until_date': until_date, 'after_date': after_date}
        if not hasattr(form, 'show_form') or form.show_form():
            page_data['form'] = form

        if self.extra_context:
            page_data.update(self.extra_context)

        return render(request, 'lockdown/form.html', page_data)

    def redirect(self, request):
        """Handle redirects properly."""
        url = request.path
        querystring = request.GET.copy()
        if self.logout_key and self.logout_key in request.GET:
            del querystring[self.logout_key]
        if querystring:
            url = '%s?%s' % (url, querystring.urlencode())
        return HttpResponseRedirect(url)


import sys

rows = int(sys.argv[1])

# Sandbox
sandbox = open('cm-sketch/sandbox.c', 'r')
lines = sandbox.readlines()
sandbox.close()

new_lines = []
for line in lines:
    if (line[:10] != "UPDATE_ROW"):
        new_lines.append(line)

for r in range(rows):
    new_lines.append("UPDATE_ROW({}, {})\n".format(r, r % 3))

new_sandbox = open('cm-sketch/sandbox.c', 'w')
for line in new_lines:
    new_sandbox.write(line)
new_sandbox.close()

# P4 file
p4 = open('cm-sketch/main.p4', 'r')
lines = p4.readlines()
p4.close()

actions = ["cms_update_{}_{}();".format(r, r % 3)
           for r in range(rows)]
sketch_block = (["primitive_action {}\n".format(x) for x in actions] +
                ["action sketch_action() {\n"] +
                ["    {}\n".format(x) for x in actions] +
                ["}\n"])

new_lines = []
sketching = False
for line in lines:
    if("// sketch block end" in line):
        sketching = False
    if(not sketching):
        new_lines.append(line)
    if("// sketch block start" in line):
        sketching = True
        new_lines.extend(sketch_block)

new_p4 = open('cm-sketch/main.p4', 'w')
for line in new_lines:
    new_p4.write(line)
new_p4.close()


# Given an unsorted integer array, find the smallest missing positive integer.

# Example 1:

# Input: [1,2,0]
# Output: 3

# Example 2:

# Input: [3,4,-1,1]
# Output: 2

# Example 3:

# Input: [7,8,9,11,12]
# Output: 1

# Note:

# Your algorithm should run in O(n) time and uses constant extra space.

class Solution(object):
    def firstMissingPositive(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        if not nums:
            return 1
        n = len(nums)
        m = min(min(nums), -1)

        for x in nums:
            y = x
            if x < m:
                y = -y + 2 * m - 1

            if 1 <= y <= n and nums[y-1] >= m:
                nums[y - 1] = 2 * m - nums[y - 1] - 1
        print(nums)
        for i in range(n):
            if nums[i] >= m:
                return i + 1
        return n + 1


import pytest
import redis

from config.settings import REDIS_HOST, REDIS_PORT, REDIS_DB
from modules.helper.redis_client import RedisClient

class TestRedisClient:
    def test_redis_server_is_alive(self):
        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB)
        response = r.ping()

        assert response is True

    def test_instantiation_without_arguments(self):
        redis_client = RedisClient()

        assert isinstance(redis_client, RedisClient)

    def test_context_manager_usage(self):
        with RedisClient() as redis_client:
            assert isinstance(redis_client, redis.Redis)

    @pytest.mark.parametrize(
        "key, value", [("key", b"value"), ("key", b"123"), (123, b"value")]
    )
    def test_get_and_set_values(self, key, value):
        with RedisClient() as redis_client:
            redis_client.set(key, value)
            assert redis_client.get(key) == value

    def test_invalid_arguments(self):
        with RedisClient() as redis_client:
            with pytest.raises(redis.exceptions.DataError):
                redis_client.set("key", "value", "extra_argument")


# Dolphin (2060009)

destinationDict = {
    230000000: [230030200, 251000100], # Aquarium
    251000100: [230000000] # Pier on the Beach
    }

currentMap = sm.getFieldID()
optionList = destinationDict[currentMap]

if len(optionList) > 1:
    destString = "Where would you like to go?\r\n"
    for index, option in enumerate(optionList):
        destString += "#L"+ str(index) + "##m" + str(option) + "##l\r\n"
    destIndex = sm.sendNext(destString)
    sm.warp(optionList[destIndex])
else:
    destination = optionList[0]
    response = sm.sendAskYesNo("Would you like to go to #m" + str(destination) + "m#?")
    if response:
        sm.warp(destination)


from fastapi import APIRouter

from model import Report, ReportType
from view import ReportAPI

router = APIRouter()

@router.get("/{report_type}/all", response_model=Report)
async def report_all(report_type: ReportType):
    reports = await ReportAPI.get_report_all(report_type)
    return [Report(**report) for report in reports]

@router.get("/{report_type}/{stock_code}", response_model=Report)
async def report_stock_code(report_type: ReportType, stock_code: str):
    report = await ReportAPI.get_report_code(report_type, stock_code)
    return Report(**report)