import string, re, time, os

__author__ = 'Robin Friedrich   friedrich@pythonpros.com'
__version__ = '2.2.2'

#################
# CLASS LIBRARY #
#################

class StringTemplate:
    """Generate documents based on a template and a substitution mapping.

    Must use Python 1.5 or newer. Uses re and the get method on dictionaries.

    Usage:
       T = TemplateDocument('Xfile')
       T.substitutions = {'month': ObjectY, 'town': 'Scarborough'}
       T.write('Maine.html')

    A dictionary, or object that behaves like a dictionary, is assigned to the
    *substitutions* attribute which has symbols as keys to objects. Upon every
    occurance of these symbols surrounded by braces {} in the source template,
    the corresponding value is converted to a string and substituted in the output.

    For example, source text which looks like:
     I lost my heart at {town} Fair.
    becomes:
     I lost my heart at Scarborough Fair.

    Symbols in braces which do not correspond to a key in the dictionary remain
    unchanged.

    An optional third argument to the class is a list or two strings to be
    used as the delimiters instead of { } braces. They must be of the same
    length; for example ['##+', '##'] is invalid.
    """
    def __init__(self, template, substitutions=None, **kw):
        self.delimiters = ['{', '}']
        self.__dict__.update(kw)
        if len(self.delimiters) != 2:
            raise ValueError("delimiter argument must be a pair of strings")
        self.delimiter_width = len(self.delimiters[0])
        delimiters = map(re.escape, self.delimiters)
        self.subpatstr = delimiters[0] + "[\w_]+" + delimiters[1]
        self.subpat = re.compile(self.subpatstr)
        self.substitutions = substitutions or {}
        self.set_template(template)

    def set_template(self, template):
        self.source = template

    def keys(self):
        return self.substitutions.keys()

    def __setitem__(self, name, value):
        self.substitutions[name] = value

    def __getitem__(self, name):
        return self.substitutions[name]

    def __str__(self):
        return self._sub(self.source)

    def _sub(self, source, subs=None):
        """Perform source text substitutions.

        *source* string containing template source text
        *subs* mapping of symbols to replacement values
        """
        substitutions = subs or self.substitutions
        dw = self.delimiter_width
        i = 0
        output = []
        matched = self.subpat.search(source[i:])
        while matched:
            a, b = matched.span()
            output.append(source[i:i+a])
            # using the new get method for dicts in 1.5
            output.append(str(substitutions.get(
                   source[i+a+dw:i+b-dw], source[i+a:i+b])))
            i = i + b
            matched = self.subpat.search(source[i:])
        else:
            output.append(source[i:])
        return string.join(output, '')

    def write(self, filename = None):
        """Emit the Document HTML to a file or standard output.

        Will not overwrite file is it exists and is textually the same.
        In Unix you can use environment variables in filenames.
        Will print to stdout if no argument given.
        """
        if filename:
            if os.path.exists(filename):
                s = str(self)
                if compare_s2f(s, filename):
                    f = open(filename, 'w')
                    f.write(str(self))
                    f.close()
            else:
                f = open(filename, 'w')
                f.write(str(self))
                f.close()
        else:
            import sys
            sys.stdout.write(str(self))

class TemplateDocument(StringTemplate):
    """Generate documents based on a template and a substitution mapping.

    Must use Python 1.5 or newer. Uses re and the get method on dictionaries.

    Usage:
       T = TemplateDocument('Xfile')
       T.substitutions = {'month': ObjectY, 'town': 'Scarborough'}
       T.write('Maine.html')

    A dictionary, or object that behaves like a dictionary, is assigned to the
    *substitutions* attribute which has symbols as keys to objects. Upon every
    occurance of these symbols surrounded by braces {} in the source template,
    the corresponding value is converted to a string and substituted in the output.

    For example, source text which looks like:
     I lost my heart at {town} Fair.
    becomes:
     I lost my heart at Scarborough Fair.

    Symbols in braces which do not correspond to a key in the dictionary remain
    unchanged.

    An optional third argument to the class is a list or two strings to be
    used as the delimiters instead of { } braces. They must be of the same
    length; for example ['##+', '##'] is invalid.
    """
    def set_template(self, template):
        f = open(template)
        self.source = f.read()
        f.close()

def compare_s2f(s, f2):
    """Helper to compare a string to a file, return 0 if they are equal."""

    BUFSIZE = 8192
    i = 0
    fp2 = open(f2)
    try:
        while 1:
            try:
                b1 = s[i: i + BUFSIZE]
                i = i + BUFSIZE
            except IndexError:
                b1 = ''
            b2 = fp2.read(BUFSIZE)
            if not b1 and not b2: return 0
            c = cmp(b1, b2)
            if c: return c
    finally:
        fp2.close()


"""
Código antigos usados para comparar novas implementações
"""

import pandas as pd
from ad_main import settings
from dados.episem import episem
from sqlalchemy import create_engine

PSQL_URI = "postgresql://{}:{}@{}:{}/{}".format(
    settings.PSQL_USER,
    settings.PSQL_PASSWORD,
    settings.PSQL_HOST,
    settings.PSQL_PORT,
    settings.PSQL_DB,
)

db_engine = create_engine(PSQL_URI)

class OldReportState:
    diseases = ["dengue", "chik", "zika"]

    @classmethod
    def _read_disease_data(
        cls, cities: dict, station_id: str, year_week: int, var_climate: str
    ) -> pd.DataFrame:
        """
        :param cities:
        :param station_id:
        :param year_week:
        :param var_climate:
        :return:
        """

        k = ["casos", "p_inc100k", "casos_est", "p_rt1", "nivel", "level_code"]

        k = ["{}_{}".format(v, d) for v in k for d in cls.diseases]

        k.append(var_climate)
        k.append("n_tweets")
        k.append("geocode_dengue AS geocode")
        k.append('"SE_dengue" AS "SE"')
        k.append('epiweek2date("SE_dengue") AS init_date_week')

        general_param = {
            "year_week_start": year_week - 100,
            "year_week_end": year_week,
            "geocodes": ",".join(map(lambda v: "'{}'".format(v), cities)),
            "var_climate": var_climate,
            "station_id": station_id,
        }

        sql = ""
        previous_disease = ""
        for disease in cls.diseases:
            _param = dict(general_param)
            _param["disease"] = disease

            table_suffix = ""
            if disease != "dengue":
                table_suffix = "_{}".format(disease)

            _param["table_suffix"] = table_suffix

            sql_ = (
                """
            (SELECT
               hist."SE" AS "SE_%(disease)s",
               hist.casos AS casos_%(disease)s,
               hist.p_rt1 AS p_rt1_%(disease)s,
               hist.casos_est AS casos_est_%(disease)s,
               hist.p_inc100k AS p_inc100k_%(disease)s,
               hist.nivel AS level_code_%(disease)s,
               (CASE
                  WHEN hist.nivel=1 THEN 'verde'
                  WHEN hist.nivel=2 THEN 'amarelo'
                  WHEN hist.nivel=3 THEN 'laranja'
                  WHEN hist.nivel=4 THEN 'vermelho'
                  ELSE '-'
                END) AS nivel_%(disease)s,
                hist.municipio_geocodigo AS geocode_%(disease)s
            FROM
             "Municipio"."Historico_alerta%(table_suffix)s" AS hist
            WHERE
             hist."SE" BETWEEN %(year_week_start)s AND %(year_week_end)s
             AND hist.municipio_geocodigo IN (%(geocodes)s)
            ORDER BY "SE_%(disease)s" DESC
            ) AS %(disease)s
            """
                % _param
            )

            if not sql:
                sql = sql_
            else:
                sql += """
                    LEFT JOIN {0}
                    ON (
                      {1}."SE_{1}" = {2}."SE_{2}"
                      AND {1}.geocode_{1} = {2}.geocode_{2}
                    )
                """.format(
                    sql_, previous_disease, disease
                )
            previous_disease = disease

        tweet_join = (
            """
        LEFT JOIN (
           SELECT
             epi_week(data_dia) AS "SE_twitter",
             SUM(numero) as n_tweets,
             "Municipio_geocodigo"
           FROM "Municipio"."Tweet"
           WHERE
             "Municipio_geocodigo" IN (%(geocodes)s)
             AND epi_week(data_dia)
               BETWEEN %(year_week_start)s AND %(year_week_end)s
           GROUP BY "SE_twitter", "Municipio_geocodigo"
           ORDER BY "SE_twitter" DESC
        ) AS tweets
           ON (
             "Municipio_geocodigo"=dengue."geocode_dengue"
             AND tweets."SE_twitter"=dengue."SE_dengue"
           )
        """
            % general_param
        )

        climate_join = (
            """
        LEFT JOIN (
          SELECT
             epi_week(data_dia) AS epiweek_climate,
             AVG(%(var_climate)s) AS %(var_climate)s
          FROM "Municipio"."Clima_wu"
          WHERE "Estacao_wu_estacao_id" = '%(station_id)s'
          GROUP BY epiweek_climate
          ORDER BY epiweek_climate
        ) AS climate_wu
           ON (dengue."SE_dengue"=climate_wu.epiweek_climate)
        """
            % general_param
        )

        sql += climate_join + tweet_join

        sql = " SELECT {} FROM ({}) AS data".format(",".join(k), sql)

        df = pd.read_sql(sql, index_col="SE", con=db_engine)

        if not df.empty:
            dfs = []

            # merge with a range date dataframe to keep empty week on the data
            ts_date = pd.date_range(
                df["init_date_week"].min(),
                df["init_date_week"].max(),
                freq="7D",
            )
            df_date = pd.DataFrame({"init_date_week": ts_date})

            for geocode in df.geocode.unique():
                df_ = df[df.geocode == geocode].sort_values("init_date_week")

                df_date_ = df_date.set_index(
                    df_.init_date_week.map(
                        lambda x: int(episem(str(x)[:10], sep=""))
                    ),
                    drop=True,
                )

                df_.index.name = "SE"
                df_date_.index.name = None

                df_["init_date_week"] = pd.to_datetime(
                    df_["init_date_week"], errors="coerce"
                )
                # TODO: these parameters is not valid in the future releases.
                dfs.append(
                    pd.merge(
                        df_,
                        df_date_,
                        how="outer",
                        left_index=True,
                        right_index=True,
                    )
                )

            df = pd.concat(dfs)

        df.sort_index(ascending=True, inplace=True)

        for d in cls.diseases:
            k = "p_rt1_{}".format(d)
            df[k] = (df[k] * 100).fillna(0)
            k = "casos_est_{}".format(d)
            df[k] = df[k].fillna(0).round(0)
            k = "p_inc100k_{}".format(d)
            df[k] = df[k].fillna(0).round(0)

            df.rename(
                columns={
                    "p_inc100k_{}".format(d): "incidência {}".format(d),
                    "casos_{}".format(d): "casos notif. {}".format(d),
                    "casos_est_{}".format(d): "casos est. {}".format(d),
                    "p_rt1_{}".format(d): "pr(incid. subir) {}".format(d),
                },
                inplace=True,
            )

        df.n_tweets = df.n_tweets.fillna(0).round(0)

        return df.rename(
            columns={
                "umid_max": "umid.max",
                "temp_min": "temp.min",
                "n_tweets": "tweets",
            }
        )


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

from flask import Flask, request
from http import HTTPStatus
from flask import Flask, jsonify, redirect, render_template, request, url_for

app = Flask(__name__)

@app.route('/formality', methods=['POST'])
def formality_check():
    data = request.json['data']
    formality = is_formal(data)
    return jsonify({'formality': formality})

def is_formal(sentence):
    if len(sentence.split()) > 10 and re.search(r'\b(?:therefore|moreover|nevertheless)\b', sentence):
        return "Formal"
    else:
        return "Informal"

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=5000)

# In[ ]:

# In[ ]:


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May  6 15:46:59 2019

@author: quoctung
"""

import tensorflow as tf
import numpy as np
from alexnet import AlexNet
from adversarialMMD import AdversarialNet
from ResNet_datagenerator import ImageDataGenerator
from tensorflow.keras.preprocessing import image
from tensorflow.keras.layers import Dropout, Flatten, Dense
from tensorflow.keras.applications import ResNet50
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D
from tensorflow.keras.callbacks import TensorBoard, EarlyStopping
from tensorflow.keras import backend as K
from tensorflow.keras.applications.resnet50 import preprocess_input
import mutan_block as mb
from tensorflow.keras.applications.resnet50 import ResNet50
import time
import gc

Iterator = tf.data.Iterator
valmode = True # Use full taget set for evaluation
display_step = 15

proj_dim = 1024
config = [20, 20, 20, 20]
size = 224
height = 224
width = 224

source_file = './testtxt/amazon.txt'
target_file = './testtxt/webcam.txt'

class AdResNet(AdversarialNet):
    def __init__(self, source, target, beta = tf.placeholder(tf.float32), keep_prob = tf.placeholder(tf.float32), dimension = 256, lamda = tf.placeholder(tf.float32), skip_layer = ['fc8'], n_class = 31, weight_path = "bvlc_alexnet.npy"):
        self.source = source
        self.target = target
        self.n_source = tf.shape(source)[0]
        self.n_target = tf.shape(target)[0]
        self.beta = beta
        AlexNet.__init__(self, tf.concat([self.source, self.target], axis = 0), keep_prob, skip_layer, n_class, weight_path = "bvlc_alexnet.npy")
        self.rep_dim = dimension
        self.lamda = lamda
        self.KEEP_PROB_TRAINING = 0.5
        self.KEEP_PROB_VALIDATION = 1.0
        self.pretrain_weight = {}

    def create(self):
        self.model = ResNet50(include_top = False, weights = 'imagenet', pooling = 'avg', input_tensor = self.X)
        self.model.trainable = True
        for layer in self.model.layers:
            self.pretrain_weight[layer.name] = self.model.get_layer(layer.name).get_weights()
        print(self.model.get_layer('res5c_branch2a').get_weights())
        output_resnet = self.model(self.X)

        self.bottleneck = self.fc(output_resnet, 2048, self.rep_dim, name = "adapt/bottleneck")
        #self.dropout_bottleneck = self.dropout(self.bottleneck, self.keep_prob)
        self.logits = tf.layers.dense(self.bottleneck, units=self.n_class, kernel_initializer=tf.truncated_normal_initializer(stddev=0.01), use_bias=True, name="adapt/fc8")

        self.source_classify, self.target_classifiy = tf.split(self.logits, [self.n_source, self.n_target])

        ad_input = tf.matmul(tf.expand_dims(self.bottleneck, axis = -1), tf.expand_dims(tf.nn.softmax(self.logits, axis = -1), axis = -1), transpose_b = True)
        ad_input = tf.reshape(ad_input, [-1, self.rep_dim * self.n_class])
        self.discriminator = self.adversarial_net(ad_input, [self.rep_dim * self.n_class, 1024], name = "feature")

        print(output_resnet)

    def create_block(self, version = 0, cdan = 0):
        self.model = ResNet50(include_top = False, weights = 'imagenet', pooling = 'avg', input_tensor = self.X)
        self.model.trainable = True
        for layer in self.model.layers:
            self.pretrain_weight[layer.name] = self.model.get_layer(layer.name).get_weights()
        output_resnet = self.model(self.X)

        adapt = self.fc(output_resnet, 2048, self.rep_dim * 2, name = "adapt/main_block1")
        adapt = self.fc(adapt, self.rep_dim * 2, self.rep_dim, name = "adapt/main_block2")
        adapt_source, adapt_target = tf.split(adapt, [self.n_source, self.n_target])
        if version == 0:
            fc7_skip = self.fc(output_resnet, 2048, self.rep_dim, name = "adapt/skip_connection")

            fc7_skip_source, fc7_skip_target = tf.split(fc7_skip, [self.n_source, self.n_target])

            self.source_feature = fc7_skip_source + self.beta * adapt_source
            self.target_feature = fc7_skip_target + adapt_target
        else:
            fc7_source, fc7_target = tf.split(output_resnet, [self.n_source, self.n_target])

            adapt_source = self.fc(fc7_source, 2048, self.rep_dim, name = "adapt/skip_connection_source")
            adapt_feature = self.fc(fc7_target, 2048, self.rep_dim, name = "adapt/skip_connection_target")

            self.source_feature = adapt_source + adapt_source
            self.target_feature = adapt_feature + adapt_target

        self.bottleneck = tf.concat([self.source_feature, self.target_feature], axis = 0)
        self.dropout_bottleneck = self.dropout(self.bottleneck, self.keep_prob)
        self.logits = tf.layers.dense(self.dropout_bottleneck, units=self.n_class, kernel_initializer=tf.truncated_normal_initializer(stddev=0.01), use_bias=True, name="adapt/fc8")

        self.source_classify, self.target_classifiy = tf.split(self.logits, [self.n_source, self.n_target])

        if cdan == 0:
            ad_input = tf.matmul(tf.expand_dims(self.bottleneck, axis = -1), tf.expand_dims(tf.nn.softmax(self.logits, axis = -1), axis = -1), transpose_b = True)
            ad_input = tf.reshape(ad_input, [-1, self.rep_dim * self.n_class])
            self.discriminator = self.adversarial_net(ad_input, [self.rep_dim * self.n_class, 1024], name = "feature")
        else:
            self.discriminator = self.mutan_adversarial_net(self.bottleneck, self.logits, [proj_dim, 1024], name = "feature")

        #self.source_logits, self.target_logits = tf.split(discriminator, [self.n_source, self.n_target])

        print(self.bottleneck)

    def adversarial_net(self, x, dimension, name = "feature"):
        x = self.dropout(x, self.keep_prob)
        x = self.fc(x, dimension[0], dimension[1], "adversarial_" + name + "_layer_2")
        x = self.dropout(x, self.keep_prob)
        x = tf.layers.dense(x, 1, kernel_initializer=tf.truncated_normal_initializer(stddev=0.01), use_bias=True, name = "adversarial_" + name + "_layer_3")

        return x

    def mutan_adversarial_net(self, x, y, dimension, name = 'feature'):
        rep = mb.mutan_block(x, y, self.rep_dim, self.n_class, dimension[0], config)
        rep = self.adversarial_net(rep, dimension, name = name)
        return rep

    def create_mutan(self):
        self.model = ResNet50(include_top = False, weights = 'imagenet', pooling = 'avg', input_tensor = self.X)
        self.model.trainable = True
        for layer in self.model.layers:
            self.pretrain_weight[layer.name] = self.model.get_layer(layer.name).get_weights()
        print(self.model.get_layer('res5c_branch2a').get_weights())
        output_resnet = self.model(self.X)

        self.bottleneck = self.fc(output_resnet, 2048, self.rep_dim, name = "adapt/bottleneck")
        #self.dropout_bottleneck = self.dropout(self.bottleneck, self.keep_prob)
        self.logits = tf.layers.dense(self.bottleneck, units=self.n_class, kernel_initializer=tf.truncated_normal_initializer(stddev=0.01), use_bias=True, name="adapt/fc8")

        self.source_classify, self.target_classifiy = tf.split(self.logits, [self.n_source, self.n_target])

        self.discriminator = self.mutan_adversarial_net(self.bottleneck, self.logits, [proj_dim, 1024], name = "feature")

    #init_lr = 1e-3, init_lr= 0.0003
    def training(self, source_file, target_file, init_lr = 5e-4, training_epochs = 1000, batch_size = 30, batch_test = 60):

        sess = tf.Session()
        K.set_session(sess)
        #self.create()
        #self.create_block()
        #self.create_mutan()
        self.create_block(version = 1, cdan = 0)

        """
        ----------------
        MODEL DEFINITION
        ----------------
        """
        one_hot_labels = tf.placeholder(tf.float32, [None, self.n_class])
        lr = tf.placeholder(tf.float32)
        domain_label = tf.concat([tf.ones([self.n_source, 1]), tf.zeros([self.n_target, 1])], axis = 0)

        non_adversarial = [v for v in tf.trainable_variables() if not ('adversarial' in v.name)]
        adversarial_list = [v for v in tf.trainable_variables() if 'adversarial' in v.name]
        scratch_list = [v for v in tf.trainable_variables() if 'adapt' in v.name]
        finetuning_list = [v for v in tf.trainable_variables() if not (('adapt' in v.name) or ('adversarial' in v.name))]

        weight_decay_not_ad = 0.0005 / 2 * tf.add_n([tf.nn.l2_loss(v) for v in non_adversarial if ('weight' in v.name) or ('kernel' in v.name)])
        weight_decay_ad = 0.0005 / 2 * tf.add_n([tf.nn.l2_loss(v) for v in adversarial_list if ('weight' in v.name) or ('kernel' in v.name)])
        #prediction_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits = self.source_classify, labels = one_hot_labels))
        prediction_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits = self.source_classify, labels = tf.argmax(one_hot_labels, 1)))
        discriminator_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = self.discriminator, labels = domain_label))
        adver_loss = discriminator_loss + weight_decay_ad

        skip_source_w = [v for v in tf.trainable_variables() if ('adapt/skip_connection_source/weights' in v.name)][0]
        skip_source_b = [v for v in tf.trainable_variables() if ('adapt/skip_connection_source/biases' in v.name)][0]
        skip_target_w = [v for v in tf.trainable_variables() if ('adapt/skip_connection_target/weights' in v.name)][0]
        skip_target_b = [v for v in tf.trainable_variables() if ('adapt/skip_connection_target/biases' in v.name)][0]

        print(skip_source_w.name, skip_source_b.name, skip_target_w.name, skip_target_b.name)
        difference_loss = (tf.reduce_sum(tf.square(skip_source_w - skip_target_w)) + tf.reduce_sum(tf.square(skip_source_b - skip_target_b)))
        loss = prediction_loss + weight_decay_not_ad - self.lamda * discriminator_loss + difference_loss

        """
        --------------------
        OPTIMIZER DEFINITION
        --------------------
        """

        #print(adversarial_list)
        #print(scratch_list)
        #print(finetuning_list)

        min_var_list = scratch_list + finetuning_list
        adv_var_list = adversarial_list

        min_gradients = tf.gradients(loss, min_var_list)
        adv_gradients = tf.gradients(adver_loss, adv_var_list)
        #min_gradients, _ = tf.clip_by_global_norm(min_gradients, 5.0)
        #adv_gradients, _ = tf.clip_by_global_norm(adv_gradients, 5.0)
        min_gradients = list(zip(min_gradients, min_var_list))
        adv_gradients = list(zip(adv_gradients, adv_var_list))

        optimizer1 = tf.train.MomentumOptimizer(lr * 10, momentum = 0.9)
        optimizer2 = tf.train.MomentumOptimizer(lr, momentum = 0.9)
        optimizer3 = tf.train.MomentumOptimizer(lr * 10, momentum = 0.9)
        #optimizer3 = tf.train.AdamOptimizer(lr * 10)

        train_op1 = optimizer1.apply_gradients(grads_and_vars = min_gradients[:len(scratch_list)])
        train_op2 = optimizer2.apply_gradients(grads_and_vars = min_gradients[len(scratch_list):])
        train_op3 = optimizer3.apply_gradients(grads_and_vars = adv_gradients)
        train_op = tf.group(train_op1, train_op2, train_op3)

        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        optimize = tf.group([train_op, update_ops])

        """
        -----------------------------------------------
        DATA PREPARATION (REUSE FROM THE ALEXNET MODEL)
        -----------------------------------------------
        """

        """
        with tf.device('/cpu:0'):

            train_source_data = ImageDataGenerator(preprocessing_function = preprocess_input,
                                                   horizontal_flip = True,
                                                   rotation_range = 40,
                                                   shear_range=0.2)
            train_target_data = ImageDataGenerator(preprocessing_function = preprocess_input,
                                                   horizontal_flip = True,
                                                   rotation_range = 40,
                                                   shear_range=0.2)
            val_data = ImageDataGenerator(preprocessing_function = preprocess_input)

            train_source_generator = train_source_data.flow_from_directory(source_file,
                                                                           batch_size = batch_size,
                                                                           target_size = (size, size),
                                                                           class_mode = 'categorical',
                                                                           shuffle = True)

            train_target_generator = train_target_data.flow_from_directory(target_file,
                                                                           batch_size = batch_size,
                                                                           target_size = (size, size),
                                                                           class_mode = 'categorical',
                                                                           shuffle = True)

            val_generator = val_data.flow_from_directory(target_file,
                                                         batch_size = batch_test,
                                                         target_size = (size, size),
                                                         class_mode = 'categorical')

            #next_batch_sr = train_source_generator.next()
            #next_batch_tr = train_target_generator.next()
            #next_batch_val = val_generator.next()

            train_batches_per_epoch = int(np.floor(train_source_generator.n/ train_source_generator.batch_size))
            val_batches_per_epoch = int(np.ceil(val_generator.n / val_generator.batch_size))
        """

        with tf.device('/cpu:0'):

            train_source_data = ImageDataGenerator(source_file,
                                     mode='training',
                                     batch_size=batch_size,
                                     num_classes=self.n_class,
                                     shuffle=True, size=size)

            train_target_data = ImageDataGenerator(target_file,
                                     mode='training',
                                     batch_size=batch_size,
                                     num_classes=self.n_class,
                                     shuffle=True, size=size)

            val_data = ImageDataGenerator(target_file,
                                      mode='inference',
                                      batch_size=batch_size,
                                      num_classes=self.n_class,
                                      shuffle=False, fulval=valmode)

            iterator_source_train = Iterator.from_structure(train_source_data.data.output_types,
                                       train_source_data.data.output_shapes)
            iterator_target_train = Iterator.from_structure(train_target_data.data.output_types,
                                       train_target_data.data.output_shapes)
            iterator_val = Iterator.from_structure(val_data.data.output_types,
                                       val_data.data.output_shapes)

            next_batch_sr = iterator_source_train.get_next()
            next_batch_tr = iterator_target_train.get_next()
            next_batch_val = iterator_val.get_next()

        train_batches_per_epoch = int(np.floor(train_source_data.data_size/batch_size))
        val_batches_per_epoch = int(np.ceil(val_data.data_size / batch_size))

        training_source_init_op = iterator_source_train.make_initializer(train_source_data.data)
        training_target_init_op = iterator_target_train.make_initializer(train_target_data.data)
        validation_init_op = iterator_val.make_initializer(val_data.data)

        """
        training_source_init_op = train_source_generator.make_initializer(train_source_data.data)
        training_target_init_op = train_target_generator.make_initializer(train_target_data.data)
        validation_init_op = val_generator.make_initializer(train_target_data.data)
        """

        """
        ---------------------------
        VALIDATION SCORE DEFINITION
        ---------------------------
        """
        prob = tf.placeholder(tf.float32, [None, self.n_class])
        correct_pred = tf.equal(tf.argmax(prob, 1), tf.argmax(one_hot_labels, 1))
        accuracy = tf.reduce_sum(tf.cast(correct_pred, tf.float32))
        accunum = tf.shape(one_hot_labels)[0]

        """
        ------------------------------
        ACTUAL TRAINING AND EVALUATION
        ------------------------------
        """

        init = tf.global_variables_initializer()
        sess.run(init)

        for layer in self.model.layers:
            self.model.get_layer(layer.name).set_weights(self.pretrain_weight[layer.name])
        #self.load_initial_weight(sess)
        del self.pretrain_weight
        gc.collect()

        print(init_lr)
        print(train_batches_per_epoch)
        print(val_batches_per_epoch)
        print(self.model.get_layer('res5c_branch2a').get_weights())
        print(source_file, target_file)
        #print(train_source_generator.class_indices)
        #print(train_target_generator.class_indices)
        #print(val_generator.class_indices)
        best = 0

        #print(sess.run([v for v in tf.trainable_variables() if 'conv1/weights' in v.name]))
        for epochs in range(training_epochs):

            sess.run(training_source_init_op)
            sess.run(training_target_init_op)
            times_pass = 0
            #print(sess.run([tf.get_default_graph().get_tensor_by_name('conv1/weights:0')]))
            L_pred = 0
            L_total = 0
            L_MMD = 0
            L_diff = 0
            self.model.trainable = True

            for times in range(train_batches_per_epoch):
                if times_pass == val_batches_per_epoch - 1:
                    sess.run(training_target_init_op)
                    times_pass = 0
                else:
                    times_pass += 1

                p = 0.5 * epochs / training_epochs # + times/train_batches_per_epoch/training_epochs
                learning_rate = init_lr / (1. + 10.0 * p)**0.75
                b = 1.0 * epochs / training_epochs
                #print(learning_rate)
                alpha = 0.25#2.0 / (1.0 + np.exp(-10.0 * epochs / training_epochs)) - 1.0#min(0.3, 2.0 / (1.0 + np.exp(-10.0 * epochs / training_epochs)) - 1.0)
                #(1 - np.exp(-10.0 * epochs / training_epochs)) / (1 + np.exp(-10.0 * epochs / training_epochs))

                #begin = time.time()
                x1, y1 = sess.run(next_batch_sr)
                x2, y2 = sess.run(next_batch_tr)
                #end = time.time()
                #print("Loading: ", end - begin)

                #begin = time.time()
                #print(times, np.mean(x1), np.mean(x2))
                _, total_loss, pred_loss, M_loss, D_loss = sess.run([optimize, loss, prediction_loss, adver_loss, difference_loss], feed_dict = {self.source: x1, self.target : x2,
                                            one_hot_labels : y1, lr : learning_rate, self.keep_prob : self.KEEP_PROB_TRAINING, self.lamda : alpha, self.beta : b})
                #end = time.time()
                #print("Running: ",end - begin)

                #print(total_loss, pred_loss, M_loss)

                L_pred = L_pred + pred_loss
                L_total = L_total + total_loss
                L_MMD = L_MMD + M_loss
                L_diff = L_diff + D_loss

            gc.collect()

            if not (epochs % 10 == 9):
                continue

            self.model.trainable = False
            #print(sess.run([v for v in tf.trainable_variables() if 'conv1/weights' in v.name]))
            print(L_pred * 1.0 / train_batches_per_epoch)
            print(L_MMD / train_batches_per_epoch)
            print(L_diff * 1.0 / train_batches_per_epoch)
            print(alpha)
            print("TIMES: ", epochs, " LOSS: ", L_total * 1.0 / train_batches_per_epoch)

            sess.run(validation_init_op)

            test_acc = 0.
            num = 0.

            for _ in range(val_batches_per_epoch):
                val_input = 0
                img_batch, label_batch = sess.run(next_batch_val)
                #img_batch = self.crop_generator(img_batch, False)
                #img_batch = np.reshape(img_batch, [-1 ,height, width, 3])
                #print(img_batch.size)
                logits = sess.run([tf.nn.softmax(self.logits, axis = -1)], feed_dict={self.target: img_batch, self.source : np.zeros(shape = (0,height, width,3), dtype = np.float32),
                                       self.keep_prob : self.KEEP_PROB_VALIDATION, self.beta : 1.0})

                val_input += logits[0]

                acc, num_ = sess.run([accuracy, accunum], feed_dict={prob: val_input, one_hot_labels: label_batch})
                test_acc += acc
                num += num_

            print("TIMES: ", epochs, " VALIDATION: ", test_acc / num)
            best = max(best, test_acc / num)

        print (best)

if source_file == './testtxt/dslr.txt':
    training_epochs = 400
elif source_file == './testtxt/webcam.txt' :
    training_epochs = 400
else:
    training_epochs = 400

net = AdResNet(tf.placeholder(dtype = tf.float32, shape = [None, height, width, 3]), tf.placeholder(dtype = tf.float32, shape = [None, height, width, 3]))
net.training(source_file, target_file, training_epochs = training_epochs)


"""
Ejercicio 10: Precedencia del operador

Escriba una función llamada precedencia que devuelve un número entero que representa la precedencia de un operador matemático. Una cadena
que contiene el operador se pasará a la función como su único parámetro. Su función debe devolver 1 para + y -, 2 para * y /, y 3 para ˆ.
Si la cadena que se pasa a la función no es uno de estos operadores, la función debería devolver -1. Incluya un programa principal que lea un
operador del usuario y muestre la precedencia del operador o un mensaje de error que indique que la entrada no era un operador.

En este ejercicio, se usa ˆ para representar la exponenciación, en lugar de la elección de Python de **, para facilitar el desarrollo de la solución.
"""

#Funciones
def precedencia(cadena):
	aux = cadena.lstrip(" ")
	aux = aux.rstrip(" ")
	if (len(aux) == 1):
		if (aux == "+" or aux == "-"):
			salida = 1
		elif(aux == "*" or aux == "/"):
			salida = 2
		elif(aux == "ˆ"):
			salida = 3
		else:
			salida = -1
	else:
		if (aux.lower() == "suma" or aux.lower() == "resta" or aux.lower() == "sumataria" or aux.lower() == "sustraccion"):
			salida = 1
		elif(aux.lower() == "multiplicacion" or aux.lower() == "division" or aux.lower()== "producto" or aux.lower() == "cociente"):
			salida = 2
		elif(aux.lower() == "potencia" or aux.lower() == "potenciacion"):
			salida = 3
		else:
			salida = -1

	return salida

#Programa
print("Bienvendos a mi programa!!!!!!!!")

cad = input("Ingrese una palabra\t")

if (precedencia(cad) == 1):
	print("La palabra ingresada tiene precedencia en una suma o resta")
elif(precedencia(cad) == 2):
	print("La palabra ingresada tiene precedencia en una multiplcacion o division")
elif(precedencia(cad) == 3):
	print("La palabra ingresada tiene precedencia en una potencia")
elif(precedencia(cad) == -1):
	print("La palabra indicada no es un operador")


# Author: Michiel Sandra, Lund University
import numpy as np

def zch(u, Nzc, q = 0, dtype=np.complex64):
    n = np.arange(Nzc)
    cf = np.mod(Nzc, 2)
    return np.exp(-1j * np.pi * u * n * (n + cf + 2*q)/Nzc, dtype=dtype)

dt = np.dtype([('re', np.int16), ('im', np.int16)])

# Parameters
f = 813 # amount of frequency points # floor(400 MHz / 500 MHz *1024 )
l = 1024 # L
p = 2048 # p must be a multiple of l (for the current version of this code)
m = 16 # M
ntx = 4
ntx_offset = 0
period = 2500000
r = period - ntx * ((m*l) + p)

# Generate ZCH sequence in freq domain and convert to time domain
yzch = zch(1, f)
y = np.concatenate((yzch, np.zeros(l-f)))
y = np.roll(y, -int(np.floor(f/2)))
y = np.fft.ifft(y)

# Some scaling
factor = 0.85 * 1/max(np.max(abs(y.real)),np.max(abs(y.imag)))
y = factor*y

# Change data format to sc16
y16 = np.zeros(l, dt)
y16['re'] = (y.real * (2**15))
y16['im'] = (y.imag * (2**15))

for i in range(ntx_offset, ntx):
    out = np.array([],dtype=dt)
    for j in range(ntx_offset, ntx):
        if j == i:
            out = np.append(out, np.tile(y16,m+int(p/l)))
        else:
            out = np.append(out, np.zeros(m*l+p,dt))

    out = np.append(out, np.zeros(r,dt))

    out.tofile('tx{}.dat'.format(i))


from cffi import FFI

ffi = FFI()
ffi.set_source('mylib',
r"""
#include <stdio.h>
int a = 1;
""")
ffi.cdef(r"""
void puts(const char *);
int a = 1;
""")
ffi.compile(verbose=True)

import mylib
mylib.lib.puts(b"Hello World!")
print(mylib.lib.a)


# coding=utf-8
# python基本语法(系统包导入)
#
#

import os
import time

def main():
    '''
    主函数
    :return:
    '''

    os.system('ls -al')     #执行linux命令ls -al，列出当前目录下的所有文件
    os.system('touch /tmp/aa.txt')  #执行系统命令 touch /tmp/aa.txt， 创建一个文件/tmp/aa.txt
    print(os.path.exists('/tmp/aa.txt'))    #判断文件/tmp/aa.txt是否存在

    cur_time = int(time.time())
    print('当前系统时间戳:{}'.format(int(cur_time)))

if __name__ == '__main__':
	main()


#encode=utf8
'''
Created on 2015年7月27日

@author:  李文强   liwenqiang

@Email ：liwenqiang2410@gmail.com

F:/workspace/lee/PyTest/sixteen/getweb.py

(
'''
import socket
if __name__ =='__main__':
    s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
    s.connect(('www.baidu.com',80))
    s.send("GET /index.html HTTP/1.0\r\n\r\n".encode())
    data=s.recv(4096).decode()
    print(data)


from django.urls import path
from django.views.generic import TemplateView

from .views import *
# from appointment.views import *

app_name = "accounts"

urlpatterns = [
    path('patient/register', RegisterPatientView.as_view(), name='patient-register'),
    path('person/', RegisterPersonView.as_view(), name='person'),
    # path('patient/profile/update/', EditPatientProfileView.as_view(), name='patient-profile-update'),
    path('doctor/register', RegisterDoctorView.as_view(), name='doctor-register'),
    # path('doctor/profile/update/', EditDoctorProfileView.as_view(), name='doctor-profile-update'),
    path('login', LoginView.as_view(), name='login'),
    path('logout', LogoutView.as_view(), name='logout'),
    path("result/", TemplateView.as_view(template_name="accounts/results.html"), name="result"),

]


# Databricks notebook source
# MAGIC %md # Transforming Complex Data Types in Spark SQL
# MAGIC
# MAGIC In this notebook we're going to go through some data transformation examples using Spark SQL. Spark SQL supports many
# MAGIC built-in transformation functions in the module `pyspark.sql.functions` therefore we will start off by importing that.

# COMMAND ----------

from pyspark.sql.functions import *
from pyspark.sql.types import *

# Convenience function for turning JSON strings into DataFrames.
def jsonToDataFrame(json, schema=None):
  # SparkSessions are available with Spark 2.0+
  reader = spark.read
  if schema:
    reader.schema(schema)
  return reader.json(sc.parallelize([json]))

# COMMAND ----------

# MAGIC %md Selecting from nested columns - Dots (`"."`) can be used to access nested columns for structs and maps.

# COMMAND ----------

# Using a struct
schema = StructType().add("a", StructType().add("b", IntegerType()))

events = jsonToDataFrame("""
{
  "a": {
     "b": 1
  }
}
""", schema)

display(events.select("a.b"))

# COMMAND ----------

# Using a map
schema = StructType().add("a", MapType(StringType(), IntegerType()))

events = jsonToDataFrame("""
{
  "a": {
     "b": 1
  }
}
""", schema)

display(events.select("a.b"))

# COMMAND ----------

# MAGIC %md Flattening structs - A star (`"*"`) can be used to select all of the subfields in a struct.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": {
     "b": 1,
     "c": 2
  }
}
""")

display(events.select("a.*"))

# COMMAND ----------

# MAGIC %md Nesting columns - The `struct()` function or just parentheses in SQL can be used to create a new struct.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": 1,
  "b": 2,
  "c": 3
}
""")

display(events.select(struct(col("a").alias("y")).alias("x")))

# COMMAND ----------

# MAGIC %md Nesting all columns - The star (`"*"`) can also be used to include all columns in a nested struct.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": 1,
  "b": 2
}
""")

display(events.select(struct("*").alias("x")))

# COMMAND ----------

# MAGIC %md Selecting a single array or map element - `getItem()` or square brackets (i.e. `[ ]`) can be used to select a single element out of an array or a map.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": [1, 2]
}
""")

display(events.select(col("a").getItem(0).alias("x")))

# COMMAND ----------

# Using a map
schema = StructType().add("a", MapType(StringType(), IntegerType()))

events = jsonToDataFrame("""
{
  "a": {
    "b": 1
  }
}
""", schema)

display(events.select(col("a").getItem("b").alias("x")))

# COMMAND ----------

# MAGIC %md Creating a row for each array or map element - `explode()` can be used to create a new row for each element in an array or each key-value pair.  This is similar to `LATERAL VIEW EXPLODE` in HiveQL.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": [1, 2]
}
""")

display(events.select(explode("a").alias("x")))

# COMMAND ----------

# Using a map
schema = StructType().add("a", MapType(StringType(), IntegerType()))

events = jsonToDataFrame("""
{
  "a": {
    "b": 1,
    "c": 2
  }
}
""", schema)

display(events.select(explode("a").alias("x", "y")))

# COMMAND ----------

# MAGIC %md Collecting multiple rows into an array - `collect_list()` and `collect_set()` can be used to aggregate items into an array.

# COMMAND ----------

events = jsonToDataFrame("""
[{ "x": 1 }, { "x": 2 }]
""")

display(events.select(collect_list("x").alias("x")))

# COMMAND ----------

# using an aggregation
events = jsonToDataFrame("""
[{ "x": 1, "y": "a" }, { "x": 2, "y": "b" }]
""")

display(events.groupBy("y").agg(collect_list("x").alias("x")))

# COMMAND ----------

# MAGIC %md Selecting one field from each item in an array - when you use dot notation on an array we return a new array where that field has been selected from each array element.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": [
    {"b": 1},
    {"b": 2}
  ]
}
""")

display(events.select("a.b"))

# COMMAND ----------

# MAGIC %md Convert a group of columns to json - `to_json()` can be used to turn structs into json strings. This method is particularly useful when you would like to re-encode multiple columns into a single one when writing data out to Kafka. This method is not presently available in SQL.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": {
    "b": 1
  }
}
""")

display(events.select(to_json("a").alias("c")))

# COMMAND ----------

# MAGIC %md Parse a column containing json - `from_json()` can be used to turn a string column with json data into a struct. Then you may flatten the struct as described above to have individual columns. This method is not presently available in SQL.
# MAGIC **This method is available since Spark 2.1**

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": "{\\"b\\":1}"
}
""")

schema = StructType().add("b", IntegerType())
display(events.select(from_json("a", schema).alias("c")))

# COMMAND ----------

# MAGIC %md Sometimes you may want to leave a part of the JSON string still as JSON to avoid too much complexity in your schema.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": "{\\"b\\":{\\"x\\":1,\\"y\\":{\\"z\\":2}}}"
}
""")

schema = StructType().add("b", StructType().add("x", IntegerType())
                            .add("y", StringType()))
display(events.select(from_json("a", schema).alias("c")))

# COMMAND ----------

# MAGIC %md Parse a set of fields from a column containing json - `json_tuple()` can be used to extract a fields available in a string column with json data.

# COMMAND ----------

events = jsonToDataFrame("""
{
  "a": "{\\"b\\":1}"
}
""")

display(events.select(json_tuple("a", "b").alias("c")))

# COMMAND ----------

# MAGIC %md Parse a well formed string column - `regexp_extract()` can be used to parse strings using regular expressions.

# COMMAND ----------

events = jsonToDataFrame("""
[{ "a": "x: 1" }, { "a": "y: 2" }]
""")

display(events.select(regexp_extract("a", "([a-z]):", 1).alias("c")))


#!/usr/bin/env python
# coding: utf-8

from preprocess import Preprocess
from flask import Flask, jsonify, request
import config
# import json

app = Flask(__name__)
p = Preprocess()

@app.route('/')
def index():
    data = ['Usage: ',\
            'http://localhost:10080/generate?text=Any%20sentence',\
            'http://localhost:10080/pipeline?sentence=Any%20sentence',\
            'http://localhost:10080/preprocess?sentence=Any%20sentence',\
            '/preprocess_learning',\
            '/srl',\
            'pos',\
            'ner',\
            'ctree',\
            'gapqg'
            ]
    return jsonify(data)

@app.route('/srl')
def srl():
    sentence = request.args.get('sentence')
    data = p.srl(sentence)
    return jsonify(data)

@app.route('/pos')
def pos():
    sentence = request.args.get('sentence')
    data = p.pos(sentence)
    return jsonify(data)

@app.route('/ner')
def ner():
    sentence = request.args.get('sentence')
    data = p.ner(sentence)
    return jsonify(data)

@app.route('/ctree')
def ctree():
    sentence = request.args.get('sentence')
    data = p.ctree(sentence)
    return jsonify(data)

@app.route('/dtree')
def dtree():
    sentence = request.args.get('sentence')
    data = p.dtree(sentence)
    return jsonify(data)

@app.route('/preprocess')
def preprocess():
    sentence = request.args.get('sentence')
    data = p.preprocess(sentence)
    return jsonify(data)

@app.route('/preprocess_learning')
def preprocess_learning():
    sentence = request.args.get('sentence')
    data = p.preprocess_learning(sentence)
    return jsonify(data)

# @app.route('/pipeline')
# def pipeline():
#     sentence = request.args.get('sentence')
#     data = qg.pipeline(sentence)
#     return jsonify(data)

# @app.route('/generate')
# def generate():
#     text = request.args.get('text')
#     data = qg.generate(text)
#     return jsonify(data)

# @app.route('/gapqg')
# def gapqg():
#     return jsonify([])

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=config.port, debug=config.debug)


"""
Tests to ensure that NSDictionary and NSMutableDictionary conform
to the python dict interface

These tests are basicly a port of the Python 3.2 tests for dictionaries
and dictionary views.

TODO:
- Port all tests
- Do the same for python 2.x
- Do the same for sets (NSSet) and lists (NSArray)
"""
import collections.abc

# Import some of the stdlib tests
from test import mapping_tests

import objc
from PyObjCTools.TestSupport import TestCase

NSDictionary = objc.lookUpClass("NSDictionary")
NSMutableDictionary = objc.lookUpClass("NSMutableDictionary")

class TestNSDictionaryInterface(TestCase):
    def dictClass(self):
        return NSDictionary

    def createDictionary(self, **kwds):
        return NSDictionary.dictionaryWithDictionary_(kwds)

    def testConstructor(self):
        value = self.dictClass()()
        self.assertEqual(value, {})

    def testBool(self):
        self.assertIs(not self.createDictionary(), True)
        self.assertTrue(self.createDictionary(foo="1"))
        self.assertIs(bool(self.createDictionary()), False)
        self.assertIs(bool(self.createDictionary(foo="1")), True)

    def testKeys(self):
        d = self.createDictionary()
        self.assertEqual(set(d.keys()), set())

        d = self.createDictionary(a=1, b=2)
        k = d.keys()

        self.assertIn("a", d)
        self.assertIn("b", d)
        self.assertIn("a", k)
        self.assertIn("b", k)

        self.assertNotIsInstance(k, list)

        s = d.keys() | {"c"}
        self.assertEqual(s, {"a", "b", "c"})

        s = {"c"} | d.keys()
        self.assertEqual(s, {"a", "b", "c"})

        s = d.keys() & {"b"}
        self.assertEqual(s, {"b"})

        s = {"b", "c"} & d.keys()
        self.assertEqual(s, {"b"})

        s = d.keys() - {"b"}
        self.assertEqual(s, {"a"})

        s = {"c", "b"} - d.keys()
        self.assertEqual(s, {"c"})

        s = d.keys() ^ {"b", "c"}
        self.assertEqual(s, {"a", "c"})

        s = {"c", "b"} ^ d.keys()
        self.assertEqual(s, {"a", "c"})

        self.assertEqual(
            repr(self.createDictionary(a=1).keys()), "<nsdict_keys(['a'])>"
        )

    def testValues(self):
        d = self.createDictionary()
        self.assertEqual(set(d.values()), set())

        d = self.createDictionary(a=1)

        self.assertEqual(set(d.values()), {1})
        with self.assertRaisesRegex(
            TypeError, r".*\(\) takes 1 positional argument but 2 were given"
        ):
            d.values(None)
        self.assertNotIsInstance(d.values(), list)

        self.assertEqual(
            repr(self.createDictionary(a=1).values()), "<nsdict_values([1])>"
        )

    def testItems(self):
        d = self.createDictionary()
        self.assertEqual(set(d.items()), set())

        d = self.createDictionary(a=1)
        self.assertEqual(set(d.items()), {("a", 1)})
        with self.assertRaisesRegex(
            TypeError, r".*\(\) takes 1 positional argument but 2 were given"
        ):
            d.items(None)
        self.assertEqual(
            repr(self.createDictionary(a=1).items()), "<nsdict_items([('a', 1)])>"
        )

    def testContains(self):
        d = self.createDictionary()
        self.assertNotIn("a", d)
        self.assertFalse("a" in d)
        self.assertTrue("a" not in d)

        d = self.createDictionary(a=1, b=2)
        self.assertIn("a", d)
        self.assertIn("b", d)
        self.assertNotIn("c", d)

        with self.assertRaisesRegex(
            TypeError, r".*\(\) missing 1 required positional argument: 'key'"
        ):
            d.__contains__()

    def testLen(self):
        d = self.createDictionary()
        self.assertEqual(len(d), 0)

        d = self.createDictionary(a=1, b=2)
        self.assertEqual(len(d), 2)

    def testGetItem(self):
        d = self.createDictionary(a=1, b=2)
        self.assertEqual(d["a"], 1)
        self.assertEqual(d["b"], 2)

        with self.assertRaisesRegex(
            TypeError, r".*\(\) missing 1 required positional argument: 'key'"
        ):
            d.__getitem__()

    def testFromKeys(self):
        d = self.dictClass().fromkeys("abc")
        self.assertEqual(d, {"a": None, "b": None, "c": None})

        d = self.createDictionary()
        self.assertIsNot(d.fromkeys("abc"), d)
        self.assertEqual(d.fromkeys("abc"), {"a": None, "b": None, "c": None})
        self.assertEqual(d.fromkeys((4, 5), 0), {4: 0, 5: 0})
        self.assertEqual(d.fromkeys([]), {})

        def g():
            yield 1

        self.assertEqual(d.fromkeys(g()), {1: None})
        with self.assertRaisesRegex(TypeError, "'int' object is not iterable"):
            {}.fromkeys(3)

        d = self.dictClass()(zip(range(6), range(6)))
        self.assertEqual(dict.fromkeys(d, 0), dict(zip(range(6), [0] * 6)))

    def _testCopy(self):
        self.fail("Decide what to do w.r.t. -copy and -mutableCopy")

    def testGet(self):
        d = self.createDictionary()
        self.assertIs(d.get("c"), None)
        self.assertEqual(d.get("c", 3), 3)

        d = self.createDictionary(a=1, b=2)
        self.assertIs(d.get("c"), None)
        self.assertEqual(d.get("c", 3), 3)
        self.assertEqual(d.get("a"), 1)
        self.assertEqual(d.get("a", 3), 1)
        with self.assertRaisesRegex(
            TypeError, r".*\(\) missing 1 required positional argument: 'key'"
        ):
            d.get()
        with self.assertRaisesRegex(
            TypeError, r".*\(\) takes from 2 to 3 positional arguments but 4 were given"
        ):
            d.get(None, None, None)

    def testEq(self):
        self.assertEqual(self.createDictionary(), self.createDictionary())
        self.assertEqual(self.createDictionary(a=1), self.createDictionary(a=1))

        class Exc(Exception):
            pass

        class BadCmp:
            def __eq__(self, other):
                raise Exc()

            def __hash__(self):
                return 1

        d1 = {BadCmp(): 1}
        d2 = {BadCmp(): 1}

        with self.assertRaises(Exc):
            d1 == d2  # noqa: B015

    def testKeysContained(self):
        self.helper_keys_contained(lambda x: x.keys())
        self.helper_keys_contained(lambda x: x.items())

    def helper_keys_contained(self, fn):
        # Test rich comparisons against dict key views, which should behave the
        # same as sets.

        empty = fn(self.createDictionary())
        empty2 = fn(self.createDictionary())
        smaller = fn(self.createDictionary(a=1, b=2))
        larger = fn(self.createDictionary(a=1, b=2, c=3))
        larger2 = fn(self.createDictionary(a=1, b=2, c=3))
        larger3 = fn(self.createDictionary(d=1, b=2, c=3))

        self.assertTrue(smaller < larger)
        self.assertTrue(smaller <= larger)
        self.assertTrue(larger > smaller)
        self.assertTrue(larger >= smaller)

        self.assertFalse(smaller >= larger)
        self.assertFalse(smaller > larger)
        self.assertFalse(larger <= smaller)
        self.assertFalse(larger < smaller)

        self.assertFalse(smaller < larger3)
        self.assertFalse(smaller <= larger3)
        self.assertFalse(larger3 > smaller)
        self.assertFalse(larger3 >= smaller)

        # Inequality strictness
        self.assertTrue(larger2 >= larger)
        self.assertTrue(larger2 <= larger)
        self.assertFalse(larger2 > larger)
        self.assertFalse(larger2 < larger)

        self.assertTrue(larger == larger2)
        self.assertTrue(smaller != larger)

        # There is an optimization on the zero-element case.
        self.assertTrue(empty == empty2)
        self.assertFalse(empty != empty2)
        self.assertFalse(empty == smaller)
        self.assertTrue(empty != smaller)

        # With the same size, an elementwise compare happens
        self.assertTrue(larger != larger3)
        self.assertFalse(larger == larger3)

    def testErrorsInViewContainmentCheck(self):
        class C:
            def __eq__(self, other):
                raise RuntimeError()

        d1 = self.dictClass().dictionaryWithDictionary_({1: C()})
        d2 = self.dictClass().dictionaryWithDictionary_({1: C()})

        with self.assertRaises(RuntimeError):
            d1.items() == d2.items()  # noqa: B015
        with self.assertRaises(RuntimeError):
            d1.items() != d2.items()  # noqa: B015
        with self.assertRaises(RuntimeError):
            d1.items() <= d2.items()  # noqa: B015
        with self.assertRaises(RuntimeError):
            d1.items() >= d2.items()  # noqa: B015

        d3 = self.dictClass().dictionaryWithDictionary_({1: C(), 2: C()})
        with self.assertRaises(RuntimeError):
            d2.items() < d3.items()  # noqa: B015
        with self.assertRaises(RuntimeError):
            d3.items() > d2.items()  # noqa: B015

        k1 = self.dictClass().dictionaryWithDictionary_({1: 1, 2: 2}).keys()
        k2 = self.dictClass().dictionaryWithDictionary_({1: 1, 2: 2, 3: 3}).keys()
        k3 = self.dictClass().dictionaryWithDictionary_({4: 4}).keys()

        self.assertEqual(k1 - k2, set())
        self.assertEqual(k1 - k3, {1, 2})
        self.assertEqual(k2 - k1, {3})
        self.assertEqual(k3 - k1, {4})
        self.assertEqual(k1 & k2, {1, 2})
        self.assertEqual(k1 & k3, set())
        self.assertEqual(k1 | k2, {1, 2, 3})
        self.assertEqual(k1 ^ k2, {3})
        self.assertEqual(k1 ^ k3, {1, 2, 4})

    def testDictviewSetOperationsOnItems(self):
        k1 = self.dictClass().dictionaryWithDictionary_({1: 1, 2: 2}).items()
        k2 = self.dictClass().dictionaryWithDictionary_({1: 1, 2: 2, 3: 3}).items()
        k3 = self.dictClass().dictionaryWithDictionary_({4: 4}).items()

        self.assertEqual(k1 - k2, set())
        self.assertEqual(k1 - k3, {(1, 1), (2, 2)})
        self.assertEqual(k2 - k1, {(3, 3)})
        self.assertEqual(k3 - k1, {(4, 4)})
        self.assertEqual(k1 & k2, {(1, 1), (2, 2)})
        self.assertEqual(k1 & k3, set())
        self.assertEqual(k1 | k2, {(1, 1), (2, 2), (3, 3)})
        self.assertEqual(k1 ^ k2, {(3, 3)})
        self.assertEqual(k1 ^ k3, {(1, 1), (2, 2), (4, 4)})

    def testDictviewMixedSetOperations(self):
        # Just a few for .keys()
        self.assertTrue(
            self.dictClass().dictionaryWithDictionary_({1: 1}).keys() == {1}
        )
        self.assertTrue(
            {1} == self.dictClass().dictionaryWithDictionary_({1: 1}).keys()
        )
        self.assertEqual(
            self.dictClass().dictionaryWithDictionary_({1: 1}).keys() | {2}, {1, 2}
        )
        self.assertEqual(
            {2} | self.dictClass().dictionaryWithDictionary_({1: 1}).keys(), {1, 2}
        )
        self.assertFalse(
            self.dictClass().dictionaryWithDictionary_({1: 1}).keys() == [1]
        )

        # And a few for .items()
        self.assertTrue(
            self.dictClass().dictionaryWithDictionary_({1: 1}).items() == {(1, 1)}
        )
        self.assertTrue(
            {(1, 1)} == self.dictClass().dictionaryWithDictionary_({1: 1}).items()
        )
        self.assertEqual(
            self.dictClass().dictionaryWithDictionary_({1: 1}).items() | {2},
            {(1, 1), 2},
        )
        self.assertEqual(
            {2} | self.dictClass().dictionaryWithDictionary_({1: 1}).items(),
            {(1, 1), 2},
        )
        self.assertFalse(
            self.dictClass().dictionaryWithDictionary_({1: 1}).items() == [(1, 1)]
        )

class TestNSMutableDictionaryInterface(TestNSDictionaryInterface):
    def dictClass(self):
        return NSMutableDictionary

    def createDictionary(self, **kwds):
        return NSMutableDictionary.dictionaryWithDictionary_(kwds)

    def testKeysMutable(self):
        d = self.createDictionary()
        self.assertEqual(set(d.keys()), set())

        d = self.createDictionary(a=1, b=2)
        k = d.keys()

        self.assertIn("a", d)
        self.assertIn("b", d)
        self.assertIn("a", k)
        self.assertIn("b", k)

        self.assertNotIsInstance(k, list)

        self.assertNotIn("c", d)
        self.assertNotIn("c", k)

        d["c"] = 3
        self.assertIn("c", d)
        self.assertIn("c", k)

    def testValuesMutable(self):
        d = self.createDictionary()
        self.assertEqual(set(d.values()), set())

        d = self.createDictionary(a=1)
        v = d.values()

        self.assertEqual(set(v), {1})

        d["b"] = 2
        self.assertEqual(set(v), {1, 2})

    def testItemsMutable(self):
        d = self.createDictionary()
        self.assertEqual(set(d.items()), set())

        d = self.createDictionary(a=1)
        i = d.items()

        self.assertEqual(set(i), {("a", 1)})

        d["b"] = 2

        self.assertEqual(set(i), {("a", 1), ("b", 2)})

    def testGetItem(self):
        d = self.createDictionary(a=1, b=2)
        self.assertEqual(d["a"], 1)
        self.assertEqual(d["b"], 2)

        d["c"] = 3
        d["a"] = 4
        self.assertEqual(d["c"], 3)
        self.assertEqual(d["a"], 4)
        del d["b"]
        self.assertEqual(d, {"a": 4, "c": 3})

        with self.assertRaisesRegex(
            TypeError, r".*\(\) missing 1 required positional argument: 'key'"
        ):
            d.__getitem__()

        class Exc(Exception):
            pass

        class BadEq:
            def __eq__(self, other):
                raise Exc()

            def __hash__(self):
                return hash(23)

        d = self.createDictionary()
        d[BadEq()] = 42
        with self.assertRaisesRegex(KeyError, "23"):
            d.__getitem__(23)

        class BadHash:
            fail = False

            def __hash__(self):
                if self.fail:
                    raise Exc()
                else:
                    return 42

        d = self.createDictionary()
        x = BadHash()
        d[x] = 42
        x.fail = True

        # FIXME
        # with self.assertRaises(Exc):
        #    d[x]

    def testClear(self):
        d = self.createDictionary(a=1, b=2, c=3)
        d.clear()

        self.assertEqual(d, {})

        with self.assertRaisesRegex(
            TypeError, r".*\(\) takes 1 positional argument but 2 were given"
        ):
            d.clear(None)

    def testUpdate(self):
        d = self.createDictionary()
        d.update({1: 100})
        d.update({2: 20})
        d.update({1: 1, 2: 2, 3: 3})
        self.assertEqual(d, {1: 1, 2: 2, 3: 3})

        d.update()
        self.assertEqual(d, {1: 1, 2: 2, 3: 3})

        with self.assertRaisesRegex(
            (TypeError, AttributeError), "'NoneType' object is not iterable"
        ):
            d.update(None)

        class SimpleUserDict:
            def __init__(self):
                self.d = {1: 1, 2: 2, 3: 3}

            def keys(self):
                return self.d.keys()

            def __getitem__(self, i):
                return self.d[i]

        d = self.createDictionary()
        d.update(SimpleUserDict())
        self.assertEqual(d, {1: 1, 2: 2, 3: 3})

        class Exc(Exception):
            pass

        d.clear()

        class FailingUserDict:
            def keys(self):
                raise Exc()

        with self.assertRaises(Exc):
            d.update(FailingUserDict())

        class FailingUserDict:
            def keys(self):
                class BogonIter:
                    def __init__(self):
                        self.i = 1

                    def __iter__(self):
                        return self

                    def __next__(self):
                        if self.i:
                            self.i = 0
                            return "a"
                        raise Exc

                return BogonIter()

            def __getitem__(self, key):
                return key

        with self.assertRaises(Exc):
            d.update(FailingUserDict())

        class FailingUserDict:
            def keys(self):
                class BogonIter:
                    def __init__(self):
                        self.i = ord("a")

                    def __iter__(self):
                        return self

                    def __next__(self):
                        if self.i <= ord("z"):
                            rtn = chr(self.i)
                            self.i += 1
                            return rtn
                        raise StopIteration

                return BogonIter()

            def __getitem__(self, key):
                raise Exc

        with self.assertRaises(Exc):
            d.update(FailingUserDict())

        class badseq:
            def __iter__(self):
                return self

            def __next__(self):
                raise Exc()

        with self.assertRaises(Exc):
            {}.update(badseq())
            with self.assertRaisesRegex(ValueError, "foo"):
                {}.update([(1, 2, 3)])

    def setDefault(self):
        d = self.createDictionary()
        self.assertIs(d.setdefault("key0"), None)
        d.setdefault("key0", [])
        self.assertIs(d.setdefault("key0"), None)
        d.setdefault("key", []).append(3)
        self.assertEqual(d["key"][0], 3)
        d.setdefault("key", []).append(4)
        self.assertEqual(len(d["key"]), 2)
        with self.assertRaisesRegex(TypeError, "foo"):
            d.setdefault()

        class Exc(Exception):
            pass

        class BadHash:
            fail = False

            def __hash__(self):
                if self.fail:
                    raise Exc()
                else:
                    return 42

        x = BadHash()
        d[x] = 42
        x.fail = True
        with self.assertRaises(Exc):
            d.setdefault(x, [])

    def testPopitem(self):
        for copymode in -1, +1:
            # -1: b has same structure as a
            # +1: b is a.copy()
            for log2size in range(12):
                size = 2**log2size
                a = self.createDictionary()
                b = self.createDictionary()
                for i in range(size):
                    a[repr(i)] = i
                    if copymode < 0:
                        b[repr(i)] = i
                if copymode > 0:
                    b = a.mutableCopy()
                for _ in range(size):
                    ka, va = ta = a.popitem()
                    self.assertEqual(va, int(ka))
                    kb, vb = tb = b.popitem()
                    self.assertEqual(vb, int(kb))
                    self.assertFalse(copymode < 0 and ta != tb)
                self.assertFalse(a)
                self.assertFalse(b)

        d = self.createDictionary()
        with self.assertRaisesRegex(
            KeyError, "'popitem on an empty [A-Z0-9_]*NSDictionary[A-Z0-9_]*"
        ):
            d.popitem()

    def testPop(self):
        d = self.createDictionary()
        k, v = "abc", "def"
        d[k] = v
        with self.assertRaisesRegex(KeyError, "^'ghi'$"):
            d.pop("ghi")

        self.assertEqual(d.pop(k), v)
        self.assertEqual(len(d), 0)

        with self.assertRaisesRegex(KeyError, k):
            d.pop(k)

        self.assertEqual(d.pop(k, v), v)
        d[k] = v
        self.assertEqual(d.pop(k, 1), v)

        with self.assertRaisesRegex(
            TypeError, r".*\(\) missing 1 required positional argument: 'key'"
        ):
            d.pop()

        class Exc(Exception):
            pass

        class BadHash:
            fail = False

            def __hash__(self):
                if self.fail:
                    raise Exc()
                else:
                    return 42

        # XXX
        # x = BadHash()
        # d[x] = 42
        # x.fail = True
        # with self.assertRaises(Exc): d.pop( x)

class DictSetTest(TestCase):
    dictclass = NSDictionary

    def testDictKeys(self):
        d = self.dictclass.dictionaryWithDictionary_({1: 10, "a": "ABC"})

        keys = d.keys()

        self.assertEqual(len(keys), 2)
        self.assertEqual(set(keys), {1, "a"})
        self.assertEqual(keys, {1, "a"})
        self.assertNotEqual(keys, {1, "a", "b"})
        self.assertNotEqual(keys, {1, "b"})
        self.assertNotEqual(keys, {1})
        self.assertNotEqual(keys, 42)
        self.assertIn(1, keys)
        self.assertIn("a", keys)
        self.assertNotIn(10, keys)
        self.assertNotIn("Z", keys)
        self.assertEqual(d.keys(), d.keys())

        e = self.dictclass.dictionaryWithDictionary_({1: 11, "a": "def"})
        self.assertEqual(d.keys(), e.keys())

    def testDictItems(self):
        d = self.dictclass.dictionaryWithDictionary_({1: 10, "a": "ABC"})
        items = d.items()

        self.assertEqual(len(items), 2)
        self.assertEqual(set(items), {(1, 10), ("a", "ABC")})
        self.assertEqual(items, {(1, 10), ("a", "ABC")})
        self.assertNotEqual(items, {(1, 10), ("a", "ABC"), "junk"})
        self.assertNotEqual(items, {(1, 10), ("a", "def")})
        self.assertNotEqual(items, {(1, 10)})
        self.assertNotEqual(items, 42)
        self.assertIn((1, 10), items)
        self.assertIn(("a", "ABC"), items)
        self.assertNotIn((1, 11), items)
        self.assertNotIn(1, items)
        self.assertNotIn((), items)
        self.assertNotIn((1,), items)
        self.assertNotIn((1, 2, 3), items)
        self.assertEqual(d.items(), d.items())

        e = d.copy()
        self.assertEqual(d.items(), e.items())

class DictSetTest(DictSetTest):
    dictclass = NSMutableDictionary

    def testDictKeysMutable(self):
        d = self.dictclass.dictionaryWithDictionary_({1: 10, "a": "ABC"})
        e = self.dictclass.dictionaryWithDictionary_({1: 11, "a": "def"})
        self.assertEqual(d.keys(), e.keys())

        del e["a"]
        self.assertNotEqual(d.keys(), e.keys())

    def testDictItemsMutable(self):
        d = self.dictclass.dictionaryWithDictionary_({1: 10, "a": "ABC"})

        e = d.copy()
        self.assertEqual(d.items(), e.items())
        d["a"] = "def"
        self.assertNotEqual(d.items(), e.items())

    def testDictMixedKeysItems(self):
        d = self.dictclass.dictionaryWithDictionary_({(1, 1): 11, (2, 2): 22})
        e = self.dictclass.dictionaryWithDictionary_({1: 1, 2: 2})
        self.assertEqual(d.keys(), e.items())
        self.assertNotEqual(d.items(), e.keys())

    def testDictValues(self):
        d = self.dictclass.dictionaryWithDictionary_({1: 10, "a": "ABC"})
        values = d.values()
        self.assertEqual(set(values), {10, "ABC"})
        self.assertEqual(len(values), 2)

class GeneralMappingTestsNSMutableDictionary(mapping_tests.BasicTestMappingProtocol):
    type2test = NSMutableDictionary

class TestDictUpdates(TestCase):
    def do_test(self, dictType):
        d = dictType()
        d["a"] = 42
        with self.assertRaisesRegex(
            TypeError, "update expected at most 1 argument(s)?, got 2"
        ):
            d.update({}, {})

        d.update({"b": 9})
        self.assertEqual(dict(d), {"a": 42, "b": 9})

        d.update({"a": 2})
        self.assertEqual(dict(d), {"a": 2, "b": 9})

        d.update([("a", 1), ("c", 3)])
        self.assertEqual(dict(d), {"a": 1, "b": 9, "c": 3})

        d.update(d=4, a=9, e=3)
        self.assertEqual(dict(d), {"a": 9, "b": 9, "c": 3, "d": 4, "e": 3})

        d.update()
        self.assertEqual(dict(d), {"a": 9, "b": 9, "c": 3, "d": 4, "e": 3})

    def test_native(self):
        self.do_test(dict)

    def test_objc(self):
        self.do_test(NSMutableDictionary)

class TestABC(TestCase):
    def testDictABC(self):
        self.assertTrue(issubclass(NSDictionary, collections.abc.Mapping))
        self.assertTrue(issubclass(NSMutableDictionary, collections.abc.Mapping))
        self.assertTrue(issubclass(NSMutableDictionary, collections.abc.MutableMapping))

    def testViewABC(self):
        d = NSDictionary.dictionary()
        self.assertTrue(isinstance(d.keys(), collections.abc.KeysView))
        self.assertTrue(isinstance(d.values(), collections.abc.ValuesView))
        self.assertTrue(isinstance(d.items(), collections.abc.ItemsView))

        d = NSMutableDictionary.dictionary()
        self.assertTrue(isinstance(d.keys(), collections.abc.KeysView))
        self.assertTrue(isinstance(d.values(), collections.abc.ValuesView))
        self.assertTrue(isinstance(d.items(), collections.abc.ItemsView))

class TestPyObjCDict(TestCase):
    def test_comparison(self):
        with self.assertRaisesRegex(
            TypeError, "'<' not supported between instances of 'dict' and 'dict'"
        ):
            {} < {}  # noqa: B015
        with self.assertRaisesRegex(
            TypeError, "'<=' not supported between instances of 'dict' and 'dict'"
        ):
            {} <= {}  # noqa: B015
        with self.assertRaisesRegex(
            TypeError, "'>' not supported between instances of 'dict' and 'dict'"
        ):
            {} > {}  # noqa: B015
        with self.assertRaisesRegex(
            TypeError, "'>=' not supported between instances of 'dict' and 'dict'"
        ):
            {} >= {}  # noqa: B015

        with self.assertRaisesRegex(
            TypeError,
            "'<' not supported between instances of '[^']*NSDictionary[^']*' and 'dict'",
        ):
            NSDictionary() < {}  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            "'<=' not supported between instances of '[^']*NSDictionary[^']*' and 'dict'",
        ):
            NSDictionary() <= {}  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            "'>' not supported between instances of '[^']*NSDictionary[^']*' and 'dict'",
        ):
            NSDictionary() > {}  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            "'>=' not supported between instances of '[^']*NSDictionary[^']*' and 'dict'",
        ):
            NSDictionary() >= {}  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            "'<' not supported between instances of 'dict' and '[^']*NSDictionary[^']*'",
        ):
            {} < NSDictionary()  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            "'<=' not supported between instances of 'dict' and '[^']*NSDictionary[^']*'",
        ):
            {} <= NSDictionary()  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            "'>' not supported between instances of 'dict' and '[^']*NSDictionary[^']*'",
        ):
            {} > NSDictionary()  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            "'>=' not supported between instances of 'dict' and '[^']*NSDictionary[^']*'",
        ):
            {} >= NSDictionary()  # noqa: B015

        self.assertFalse(NSMutableDictionary() == [])
        self.assertFalse(NSDictionary() == [])
        self.assertFalse(NSMutableDictionary() == 42)
        self.assertFalse(NSDictionary() == 42)
        self.assertFalse(NSMutableDictionary() == object())
        self.assertFalse(NSDictionary() == object())

    def test_creation(self):
        for dict_type in (NSDictionary, NSMutableDictionary):
            v = dict_type()
            self.assertIsInstance(v, dict_type)
            self.assertEqual(len(v), 0)

            v = dict_type({1: 2, 2: 3})
            self.assertIsInstance(v, dict_type)
            self.assertEqual(len(v), 2)
            self.assertEqual(v[1], 2)
            self.assertEqual(v[2], 3)

            v = dict_type([(1, -1), (2, 9)])
            self.assertIsInstance(v, dict_type)
            self.assertEqual(len(v), 2)
            self.assertEqual(v[1], -1)
            self.assertEqual(v[2], 9)

            v = dict_type(v for v in [(1, -1), (2, 9)])
            self.assertIsInstance(v, dict_type)
            self.assertEqual(len(v), 2)
            self.assertEqual(v[1], -1)
            self.assertEqual(v[2], 9)

            with self.assertRaisesRegex(
                TypeError, "dict expected at most 1 arguments, got 2"
            ):
                dict_type((1, 2), (3, 4))

            v = dict_type(a=3, b=4)
            self.assertEqual(len(v), 2)
            self.assertEqual(v["a"], 3)
            self.assertEqual(v["b"], 4)

            v = dict_type((v for v in [(1, -1), (2, 9)]), a="hello", b="world")
            self.assertIsInstance(v, dict_type)
            self.assertEqual(len(v), 4)
            self.assertEqual(v[1], -1)
            self.assertEqual(v[2], 9)
            self.assertEqual(v["a"], "hello")
            self.assertEqual(v["b"], "world")

    def test_values(self):
        py = {"a": 4, "b": 3}
        oc = NSDictionary(a=4, b=3)

        self.assertIn(4, py.values())
        self.assertIn(4, oc.values())
        self.assertNotIn(9, py.values())
        self.assertNotIn(9, oc.values())

    def test_view_set(self):
        oc = NSDictionary(a=1, b=2, c=3, d=4, e=5)

        v = oc.keys() | {"a", "f"}
        self.assertEqual(v, {"a", "b", "c", "d", "e", "f"})
        self.assertIsInstance(v, set)
        with self.assertRaisesRegex(
            TypeError, r"unsupported operand type\(s\) for |: 'nsdict_keys' and 'tuple'"
        ):
            oc.keys() | ("a", "f")
        with self.assertRaisesRegex(
            TypeError, r"unsupported operand type\(s\) for |: 'tuple' and 'nsdict_keys'"
        ):
            ("a", "f") | oc.keys()

        v = oc.keys() & {"a", "f"}
        self.assertEqual(v, {"a"})
        self.assertIsInstance(v, set)
        with self.assertRaisesRegex(
            TypeError, r"unsupported operand type\(s\) for &: 'nsdict_keys' and 'tuple'"
        ):
            oc.keys() & ("a", "f")
        with self.assertRaisesRegex(
            TypeError, r"unsupported operand type\(s\) for &: 'tuple' and 'nsdict_keys'"
        ):
            ("a", "f") & oc.keys()

        v = oc.keys() ^ {"a", "f"}
        self.assertEqual(v, {"b", "c", "d", "e", "f"})
        self.assertIsInstance(v, set)
        with self.assertRaisesRegex(
            TypeError,
            r"unsupported operand type\(s\) for \^: 'nsdict_keys' and 'tuple'",
        ):
            oc.keys() ^ ("a", "f")
        with self.assertRaisesRegex(
            TypeError,
            r"unsupported operand type\(s\) for \^: 'tuple' and 'nsdict_keys'",
        ):
            ("a", "f") ^ oc.keys()

        v = oc.keys() - {"a", "f"}
        self.assertEqual(v, {"b", "c", "d", "e"})
        self.assertIsInstance(v, set)
        with self.assertRaisesRegex(
            TypeError, r"unsupported operand type\(s\) for -: 'nsdict_keys' and 'tuple'"
        ):
            oc.keys() - ("a", "f")
        with self.assertRaisesRegex(
            TypeError, r"unsupported operand type\(s\) for -: 'tuple' and 'nsdict_keys'"
        ):
            ("a", "f") - oc.keys()

        with self.assertRaisesRegex(
            TypeError,
            r"'<' not supported between instances of 'nsdict_keys' and 'tuple'",
        ):
            oc.keys() < ("a", "f")  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            r"'<=' not supported between instances of 'nsdict_keys' and 'tuple'",
        ):
            oc.keys() <= ("a", "f")  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            r"'>' not supported between instances of 'nsdict_keys' and 'tuple'",
        ):
            oc.keys() > ("a", "f")  # noqa: B015
        with self.assertRaisesRegex(
            TypeError,
            r"'>=' not supported between instances of 'nsdict_keys' and 'tuple'",
        ):
            oc.keys() >= ("a", "f")  # noqa: B015


import torch, math
from torch import nn
from torch.nn import functional as F
from op import FusedLeakyReLU, fused_leaky_relu, upfirdn2d, conv2d_gradfix

class PixelNorm(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, input):
        return input * torch.rsqrt(torch.mean(input ** 2, dim=1, keepdim=True) + 1e-8)

def make_kernel(k):
    k = torch.tensor(k, dtype=torch.float32)
    if k.ndim == 1:
        k = k[None, :] * k[:, None]
    k /= k.sum()
    return k

class Upsample(nn.Module):
    def __init__(self, kernel, factor=2):
        super().__init__()

        self.factor = factor
        kernel = make_kernel(kernel) * (factor ** 2)
        self.register_buffer("kernel", kernel)
        p = kernel.shape[0] - factor
        pad0 = (p + 1) // 2 + factor - 1
        pad1 = p // 2
        self.pad = (pad0, pad1)

    def forward(self, input):
        out = upfirdn2d(input, self.kernel, up=self.factor, down=1, pad=self.pad)
        return out

class Downsample(nn.Module):
    def __init__(self, kernel, factor=2):
        super().__init__()

        self.factor = factor
        kernel = make_kernel(kernel)
        self.register_buffer("kernel", kernel)
        p = kernel.shape[0] - factor
        pad0 = (p + 1) // 2
        pad1 = p // 2
        self.pad = (pad0, pad1)

    def forward(self, input):
        out = upfirdn2d(input, self.kernel, up=1, down=self.factor, pad=self.pad)
        return out

class Blur(nn.Module):
    def __init__(self, kernel, pad, upsample_factor=1):
        super().__init__()

        kernel = make_kernel(kernel)
        if upsample_factor > 1:
            kernel = kernel * (upsample_factor ** 2)
        self.register_buffer("kernel", kernel)
        self.pad = pad

    def forward(self, input):
        out = upfirdn2d(input, self.kernel, pad=self.pad)
        return out

# https://github.com/facebookresearch/semi-discrete-flow/blob/main/model_itemsets.py#L306
class L2MultiheadAttention(nn.Module):
    """ Kim et al. "The Lipschitz Constant of Self-Attention" https://arxiv.org/abs/2006.04710 """
    def __init__(self, embed_dim, num_heads):
        super(L2MultiheadAttention, self).__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.head_dim = embed_dim // num_heads
        assert (
            self.head_dim * num_heads == self.embed_dim
        ), "embed_dim must be divisible by num_heads"

        self.q_weight = nn.Parameter(torch.empty(embed_dim, num_heads, self.head_dim))
        self.v_weight = nn.Parameter(torch.empty(embed_dim, num_heads, self.head_dim))
        self.out_proj = nn.Linear(embed_dim, embed_dim)

        self._reset_parameters()

    def _reset_parameters(self):
        nn.init.zeros_(self.q_weight)
        nn.init.zeros_(self.v_weight)
        nn.init.zeros_(self.out_proj.weight)
        nn.init.zeros_(self.out_proj.bias)

    def forward(self, x):
        """
        Args:
            x: (T, N, D)
            attn_mask: (T, T) added to pre-softmax logits.
        """

        T, N, _ = x.shape

        q = torch.einsum("tbm,mhd->tbhd", x, self.q_weight)
        k = torch.einsum("tbm,mhd->tbhd", x, self.q_weight)
        squared_dist = (
            torch.einsum("tbhd,tbhd->tbh", q, q).unsqueeze(1)
            + torch.einsum("sbhd,sbhd->sbh", k, k).unsqueeze(0)
            - 2 * torch.einsum("tbhd,sbhd->tsbh", q, k)
        )
        attn_logits = -squared_dist / math.sqrt(self.head_dim)
        attn_weights = F.softmax(attn_logits, dim=1)  # (T, S, N, H)
        A = torch.einsum("mhd,nhd->hmn", self.q_weight, self.q_weight) / math.sqrt(
            self.head_dim
        )
        XA = torch.einsum("tbm,hmn->tbhn", x, A)
        PXA = torch.einsum("tsbh,sbhm->tbhm", attn_weights, XA)
        PXAV = torch.einsum("tbhm,mhd->tbhd", PXA, self.v_weight).reshape(
            T, N, self.embed_dim
        )
        return self.out_proj(PXAV)

class SelfAttention(nn.Module):
    def __init__(self, in_channels, embed_dim=None, num_heads=1):
        super().__init__()
        embed_dim = in_channels if embed_dim is None else embed_dim
        self.use_conv = embed_dim != in_channels
        if self.use_conv:
            self.to_input = nn.Conv2d(in_channels, embed_dim, 1, bias=True)
            self.to_output = nn.Conv2d(embed_dim, in_channels, 1, bias=True)
        self.l2attn = L2MultiheadAttention(embed_dim, num_heads)
        self.ff = nn.Sequential(
            nn.Linear(embed_dim, embed_dim),
            nn.GELU(),
            nn.Linear(embed_dim, embed_dim),
        )
        self.ln1 = nn.LayerNorm(embed_dim)
        self.ln2 = nn.LayerNorm(embed_dim)

    def forward(self, input):
        attn_input = self.to_input(input) if self.use_conv else input
        batch, c, h, w = attn_input.shape
        # input: [N, C, H, W] --> [N, H, W, C] --> [N, HWC]
        attn_input = attn_input.permute(0, 2, 3, 1).reshape(batch, h*w*c)
        attn_input = attn_input.reshape(batch, h*w, c)
        # [N, HW+1, C]
        norm_is = self.ln1(attn_input)
        out1 = self.l2attn(norm_is) + attn_input
        norm_out1 = self.ln2(out1)
        out2 = self.ff(norm_out1.view(-1, c)).view(batch, -1, c)
        output = out2 + out1
        # [N, HW, C]
        output = output[:, :h*w, :]
        output = output.reshape(batch, h, w, c).permute(0, 3, 1, 2)
        output = self.to_output(output) if self.use_conv else output
        return output

class TextEncoder(nn.Module):
    def __init__(self, in_dim, out_dim, num_heads=8):
        super().__init__()

        self.embedding = nn.Linear(in_dim, out_dim)
        self.l2attn = L2MultiheadAttention(out_dim, num_heads)
        self.ff = nn.Sequential(
            nn.GELU(),
            nn.Linear(out_dim, out_dim),
            nn.GELU(),
            nn.Linear(out_dim, out_dim),
        )
        self.ln1 = nn.LayerNorm(out_dim)
        self.ln2 = nn.LayerNorm(out_dim)

    def forward(self, text_embeds):
        text_embeds = self.embedding(text_embeds)
        out1 = self.l2attn(text_embeds)
        out1 = self.ln1(out1 + text_embeds)
        out2 = self.ff(out1)
        output = self.ln2(out2 + out1)
        return output

class CrossAttention(nn.Module):
    def __init__(self, in_channels, text_dim, embed_dim=None, num_heads=1, bias=False):
        super().__init__()

        self.in_channels = in_channels
        embed_dim = in_channels if embed_dim is None else embed_dim
        self.embed_dim = embed_dim
        self.use_conv = in_channels != embed_dim
        if self.use_conv:
            self.to_input = nn.Conv2d(in_channels, embed_dim, 1, bias=True)
            self.to_output = nn.Conv2d(embed_dim, in_channels, 1, bias=True)
        self.to_q = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.to_k = nn.Linear(text_dim, embed_dim, bias=bias)
        self.to_v = nn.Linear(text_dim, embed_dim, bias=bias)
        self.mha = nn.MultiheadAttention(embed_dim, num_heads, batch_first=True)
        self.ff = nn.Sequential(
            nn.Linear(embed_dim, embed_dim),
            nn.GELU(),
            nn.Linear(embed_dim, embed_dim),
        )
        self.ln1 = nn.LayerNorm(embed_dim)
        self.ln2 = nn.LayerNorm(embed_dim)

    def forward(self, image_embeds, text_embeds):
        ie = self.to_input(image_embeds) if self.use_conv else image_embeds
        batch, c, h, w = ie.shape
        # image_embeds: [N, C, H, W] --> [N, H, W, C] --> [N, HW, C]
        ie = ie.permute(0, 2, 3, 1).reshape(batch, h*w, c)
        norm_ie = self.ln1(ie)
        query = self.to_q(norm_ie)
        key = self.to_k(text_embeds)
        value = self.to_v(text_embeds)
        attn_output, attn_output_weights = self.mha(query, key, value)
        out1 = attn_output + ie
        out2 = self.ff(self.ln2(out1))
        # [N, HW, C]
        output = out2 + out1
        output = output.reshape(batch, h, w, c).permute(0, 3, 1, 2)
        output = self.to_output(output)if self.use_conv else output
        return output

class EqualConv2d(nn.Module):
    def __init__(
        self, in_channel, out_channel, kernel_size, stride=1, padding=0, bias=True,
    ):
        super().__init__()

        self.weight = nn.Parameter(
            torch.randn(out_channel, in_channel, kernel_size, kernel_size)
        )
        self.scale = 1 / math.sqrt(in_channel * kernel_size ** 2)

        self.stride = stride
        self.padding = padding

        if bias:
            self.bias = nn.Parameter(torch.zeros(out_channel))

        else:
            self.bias = None

    def forward(self, input):
        out = conv2d_gradfix.conv2d(
            input,
            self.weight * self.scale,
            bias=self.bias,
            stride=self.stride,
            padding=self.padding,
        )

        return out

    def __repr__(self):
        return (
            f"{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},"
            f" {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})"
        )

class EqualLinear(nn.Module):
    def __init__(
        self, in_dim, out_dim, bias=True, bias_init=0, lr_mul=1, activation=None,
    ):
        super().__init__()

        self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul))

        if bias:
            self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init))

        else:
            self.bias = None

        self.activation = activation

        self.scale = (1 / math.sqrt(in_dim)) * lr_mul
        self.lr_mul = lr_mul

    def forward(self, input):
        if self.activation:
            out = F.linear(input, self.weight * self.scale)
            out = fused_leaky_relu(out, self.bias * self.lr_mul)

        else:
            out = F.linear(
                input, self.weight * self.scale, bias=self.bias * self.lr_mul
            )

        return out

    def __repr__(self):
        return (
            f"{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})"
        )

class ModulatedConv2d(nn.Module):
    def __init__(
        self, in_channel, out_channel, kernel_size, style_dim, n_kernel=1, demodulate=True,
        upsample=False, downsample=False, blur_kernel=[1, 3, 3, 1],
    ):
        super().__init__()

        self.eps = 1e-8
        self.kernel_size = kernel_size
        self.in_channel = in_channel
        self.out_channel = out_channel
        self.upsample = upsample
        self.downsample = downsample
        self.n_kernel = n_kernel

        if upsample:
            factor = 2
            p = (len(blur_kernel) - factor) - (kernel_size - 1)
            pad0 = (p + 1) // 2 + factor - 1
            pad1 = p // 2 + 1

            self.blur = Blur(blur_kernel, pad=(pad0, pad1), upsample_factor=factor)

        if downsample:
            factor = 2
            p = (len(blur_kernel) - factor) + (kernel_size - 1)
            pad0 = (p + 1) // 2
            pad1 = p // 2

            self.blur = Blur(blur_kernel, pad=(pad0, pad1))

        fan_in = in_channel * kernel_size ** 2
        self.scale = 1 / math.sqrt(fan_in)
        self.padding = kernel_size // 2
        # conv weight shape: out_ch, in_ch, k_h, k_w
        self.weight = nn.Parameter(torch.randn((n_kernel, out_channel, in_channel, kernel_size, kernel_size)))
        if self.n_kernel != 1:
            self.affine = nn.Linear(style_dim, n_kernel)
        self.modulation = EqualLinear(style_dim, in_channel, bias_init=1)
        self.demodulate = demodulate

    def __repr__(self):
        return (
            f"{self.__class__.__name__}({self.in_channel}, {self.out_channel}, {self.kernel_size}, "
            f"upsample={self.upsample}, downsample={self.downsample})"
        )

    def forward(self, input, style):
        batch, in_channel, height, width = input.shape
        if self.n_kernel != 1:
            selection = self.affine(style).softmax(dim=-1)
            selection = selection.view(-1, self.n_kernel, 1, 1, 1, 1)
            ada_weight = (selection * self.weight).sum(dim=1)
        else:
            ada_weight = self.weight

        style = self.modulation(style).view(batch, 1, in_channel, 1, 1)
        weight = self.scale * ada_weight * style

        if self.demodulate:
            demod = torch.rsqrt(weight.pow(2).sum([2, 3, 4]) + 1e-8)
            weight = weight * demod.view(batch, self.out_channel, 1, 1, 1)

        weight = weight.view(
            batch * self.out_channel, in_channel, self.kernel_size, self.kernel_size
        )

        if self.upsample:
            input = input.reshape(1, batch * in_channel, height, width)
            weight = weight.view(
                batch, self.out_channel, in_channel, self.kernel_size, self.kernel_size
            )
            weight = weight.transpose(1, 2).reshape(
                batch * in_channel, self.out_channel, self.kernel_size, self.kernel_size
            )
            out = conv2d_gradfix.conv_transpose2d(
                input, weight, padding=0, stride=2, groups=batch
            )
            _, _, height, width = out.shape
            out = out.view(batch, self.out_channel, height, width)
            out = self.blur(out)

        elif self.downsample:
            input = self.blur(input)
            _, _, height, width = input.shape
            input = input.view(1, batch * in_channel, height, width)
            out = conv2d_gradfix.conv2d(
                input, weight, padding=0, stride=2, groups=batch
            )
            _, _, height, width = out.shape
            out = out.view(batch, self.out_channel, height, width)

        else:
            input = input.reshape(1, batch * in_channel, height, width)
            out = conv2d_gradfix.conv2d(
                input, weight, padding=self.padding, groups=batch
            )
            _, _, height, width = out.shape
            out = out.view(batch, self.out_channel, height, width)

        return out

class NoiseInjection(nn.Module):
    def __init__(self):
        super().__init__()

        self.weight = nn.Parameter(torch.zeros(1))

    def forward(self, image, noise=None):
        if noise is None:
            batch, _, height, width = image.shape
            noise = image.new_empty(batch, 1, height, width).normal_()

        return image + self.weight * noise

class ConstantInput(nn.Module):
    def __init__(self, channel, size=4):
        super().__init__()

        self.input = nn.Parameter(torch.randn(1, channel, size, size))

    def forward(self, input):
        batch = input.shape[0]
        out = self.input.repeat(batch, 1, 1, 1)

        return out

class StyledConv(nn.Module):
    def __init__(
        self, in_channel, out_channel, kernel_size, style_dim,
        n_kernel=1, upsample=False, blur_kernel=[1, 3, 3, 1], demodulate=True,
    ):
        super().__init__()

        self.conv = ModulatedConv2d(
            in_channel, out_channel, kernel_size, style_dim, n_kernel=n_kernel,
            upsample=upsample, blur_kernel=blur_kernel, demodulate=demodulate,
        )

        self.noise = NoiseInjection()
        self.activate = FusedLeakyReLU(out_channel)

    def forward(self, input, style, noise=None):
        out = self.conv(input, style)
        out = self.noise(out, noise=noise)
        out = self.activate(out)

        return out

class ToRGB(nn.Module):
    def __init__(self, in_channel, style_dim, upsample=True, blur_kernel=[1, 3, 3, 1]):
        super().__init__()

        if upsample:
            self.upsample = Upsample(blur_kernel)

        self.conv = ModulatedConv2d(in_channel, 3, 1, style_dim, demodulate=False)
        self.bias = nn.Parameter(torch.zeros(1, 3, 1, 1))

    def forward(self, input, style, skip=None):
        out = self.conv(input, style)
        out = out + self.bias

        if skip is not None:
            skip = self.upsample(skip)

            out = out + skip

        return out

class FromRGB(nn.Module):
    def __init__(self, in_channel, out_channel):
        super().__init__()
        self.convs = nn.Sequential(
            nn.Conv2d(in_channel, 256, 3, padding='same'),
            nn.LeakyReLU(),
            nn.Conv2d(256, out_channel, 1),
        )

    def forward(self, input):
        return self.convs(input)


import psutil

from tendrl.commons import objects
from tendrl.commons.utils import cmd_utils

class Cpu(objects.BaseObject):
    def __init__(self, architecture=None, cores_per_socket=None,
                 cpu_family=None, cpu_op_mode=None,
                 model=None, model_name=None, vendor_id=None,
                 cpu_util_percent=None,
                 *args, **kwargs):
        super(Cpu, self).__init__(*args, **kwargs)
        cpu = self._getNodeCpu()
        self.architecture = architecture or cpu['Architecture']
        self.cores_per_socket = cores_per_socket or cpu["CoresPerSocket"]
        self.cpu_family = cpu_family or cpu["CPUFamily"]
        self.cpu_op_mode = cpu_op_mode or cpu["CpuOpMode"]
        self.model = model or cpu["Model"]
        self.model_name = model_name or cpu["ModelName"]
        self.vendor_id = vendor_id or cpu["VendorId"]
        self.cpu_util_percent = cpu_util_percent or str(psutil.cpu_percent())
        self.value = 'nodes/{0}/Cpu'

    def _getNodeCpu(self):
        '''returns structure

        {"nodename": [{"Architecture":   "architecture",

                   "CpuOpMode":      "cpuopmode",

                   "CPUs":           "cpus",

                   "VendorId":       "vendorid",

                   "ModelName":      "modelname",

                   "CPUFamily":      "cpufamily",

                   "Model":          "Model",

                   "CoresPerSocket": "corespersocket"}, ...], ...}

        '''
        cmd = cmd_utils.Command("lscpu")
        out, err, rc = cmd.run()
        out = str(out)
        if out:
            info_list = out.split('\n')
            cpuinfo = {
                'Architecture': info_list[0].split(':')[1].strip(),
                'CpuOpMode': info_list[1].split(':')[1].strip(),
                'CPUs': info_list[3].split(':')[1].strip(),
                'VendorId': info_list[9].split(':')[1].strip(),
                'ModelName': info_list[12].split(':')[1].strip(),
                'CPUFamily': info_list[10].split(':')[1].strip(),
                'Model': info_list[11].split(':')[1].strip(),
                'CoresPerSocket': info_list[6].split(':')[1].strip()
            }
        else:
            cpuinfo = {
                'Architecture': '', 'CpuOpMode': '',
                'CPUs': '', 'VendorId': '',
                'ModelName': '', 'CPUFamily': '',
                'Model': '', 'CoresPerSocket': ''
            }

        return cpuinfo

    def render(self):
        self.value = self.value.format(NS.node_context.node_id)
        return super(Cpu, self).render()