# Copyright (C) 2022 yui-mhcp project's author. All rights reserved.
# Licenced under the Affero GPL v3 Licence (the "Licence").
# you may not use this file except in compliance with the License.
# See the "LICENCE" file at the root of the directory for the licence information.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" TF 2.0 BERT model, compatible with the `transformers`' implementation """

import tensorflow as tf

from custom_layers import get_activation
from custom_architectures.current_blocks import _get_layer
from custom_architectures.transformers_arch.embedding_head import EmbeddingHead, HParamsEmbeddingHead
from custom_architectures.transformers_arch.text_transformer_arch import TextTransformerEncoder, HParamsTextTransformerEncoder

HParamsBaseBERT = HParamsTextTransformerEncoder(
    use_pooling = True,
    epsilon     = 1e-6
)

HParamsBertMLM  = HParamsBaseBERT(
    transform_activation    = None
)

HParamsBertClassifier   = HParamsBaseBERT(
    num_classes = -1,
    process_tokens  = False,
    process_first_token = False,
    final_drop_rate = 0.1,
    final_activation    = None,
    classifier_type     = 'dense',
    classifier_kwargs   = {}
)

HParamsBertEmbedding   = HParamsBaseBERT(
    ** HParamsEmbeddingHead, process_tokens = True
)

class BertPooler(tf.keras.layers.Layer):
    def __init__(self, embedding_dim, name = None, ** kwargs):
        super().__init__(name = name)
        self.embedding_dim = embedding_dim

        self.dense = tf.keras.layers.Dense(embedding_dim, activation = "tanh", name = "dense")

    def call(self, output):
        return self.dense(output[:, 0])

    def get_config(self):
        config = super().get_config()
        config['embedding_dim'] = self.embedding_dim
        return config

class BaseBERT(TextTransformerEncoder):
    default_params = HParamsBaseBERT

    def __init__(self, vocab_size, embedding_dim, ** kwargs):
        super().__init__(vocab_size = vocab_size, embedding_dim = embedding_dim, ** kwargs)

        self.pooler = BertPooler(** self.hparams, name = "pooler") if self.hparams.use_pooling else None

    def compute_output(self, output, training = False, mask = None, ** kwargs):
        output = super().compute_output(
            output, training = training, mask = mask, ** kwargs
        )

        pooled_output = self.pooler(output) if self.pooler is not None else None

        return (output, pooled_output)

    @classmethod
    def from_pretrained(cls,
                        pretrained_name,
                        pretrained_task = 'base',
                        pretrained      = None,
                        ** kwargs
                       ):
        if pretrained is None:
            with tf.device('cpu') as d:
                pretrained = transformers_bert(pretrained_name, pretrained_task)

        config = cls.default_params(
            vocab_size      = pretrained.config.vocab_size,
            max_token_types = pretrained.config.type_vocab_size,
            max_input_length    = pretrained.config.max_position_embeddings,

            num_layers      = pretrained.config.num_hidden_layers,
            embedding_dim   = pretrained.config.hidden_size,
            drop_rate       = pretrained.config.hidden_dropout_prob,
            epsilon         = pretrained.config.layer_norm_eps,

            mha_num_heads   = pretrained.config.num_attention_heads,
            mha_mask_factor = -10000,
            mha_drop_rate   = pretrained.config.hidden_dropout_prob,
            mha_epsilon     = pretrained.config.layer_norm_eps,

            ffn_dim                 = pretrained.config.intermediate_size,
            ffn_activation          = pretrained.config.hidden_act,
            transform_activation    = pretrained.config.hidden_act
        )

        instance = cls(** config(** kwargs))
        instance._build()

        instance.transfer_weights(pretrained, pretrained_task = pretrained_task)

        return instance

class BertMLM(BaseBERT):
    default_params = HParamsBertMLM

    def __init__(self, vocab_size, embedding_dim, ** kwargs):
        kwargs['use_pooling'] = False
        super().__init__(
            vocab_size = vocab_size, embedding_dim = embedding_dim, ** kwargs
        )

        self.dense  = tf.keras.layers.Dense(embedding_dim, name = "dense_transform")
        self.act    = get_activation(self.hparams.transform_activation)
        self.norm   = tf.keras.layers.LayerNormalization(epsilon = self.hparams.epsilon)

    def build(self, input_shape):
        self.bias = self.add_weight(
            shape = [self.hparams.vocab_size], initializer = "zeros", trainable = True, name = "bias"
        )

    def compute_output(self, output, mask = None, training = False, ** kwargs):
        output, _ = super().compute_output(output, mask = mask, training = training, ** kwargs)

        output = self.dense(output)
        if self.act is not None: output = self.act(output)
        output = self.norm(output, training = training and self.norm_training)

        return self.embeddings.linear(output) + self.bias

class BertClassifier(BaseBERT):
    default_params  = HParamsBertClassifier
    _attr_to_set    = BaseBERT._attr_to_set + ['process_tokens', 'process_first_token']

    def __init__(self, num_classes, vocab_size, embedding_dim, ** kwargs):
        super().__init__(
            vocab_size = vocab_size, embedding_dim = embedding_dim, num_classes = num_classes, ** kwargs
        )

        self.classifier = _get_layer(
            self.hparams.classifier_type, num_classes, name = 'classifier', ** self.hparams.classifier_kwargs
        )

        self.act        = get_activation(self.hparams.final_activation)
        self.dropout    = tf.keras.layers.Dropout(self.hparams.final_drop_rate)

    def compute_output(self, output, mask = None, training = False, ** kwargs):
        output, pooled_out = super(BertClassifier, self).compute_output(
            output, mask = mask, training = training, ** kwargs
        )

        if not self.process_tokens:
            output = pooled_out
        elif self.process_first_token:
            output = output[:, 0]

        if self.dropout is not None: output = self.dropout(output, training = training)
        output = self.classifier(output, training = training)
        if self.act is not None: output = self.act(output)

        return output

class BertNSP(BertClassifier):
    def __init__(self, vocab_size, embedding_dim, num_classes = 2, ** kwargs):
        kwargs.update({'use_pooling' : True, 'process_tokens' : False})
        super().__init__(
            num_classes = 2, vocab_size = vocab_size, embedding_dim = embedding_dim, ** kwargs
        )

class BertEmbedding(BaseBERT):
    default_params  = HParamsBertEmbedding
    _attr_to_set    = BaseBERT._attr_to_set + ['process_tokens']

    def __init__(self, output_dim, vocab_size, embedding_dim, ** kwargs):
        super().__init__(
            vocab_size = vocab_size, embedding_dim = embedding_dim, output_dim = output_dim, ** kwargs
        )
        if 'process_first_token' in kwargs:
            kwargs['token_selector'] = 'first' if kwargs['process_first_token'] else None
        self.embedding_head = EmbeddingHead(** self.hparams)

    def compute_output(self, output, mask = None, training = False, ** kwargs):
        output, pooled_out = super(BertEmbedding, self).compute_output(
            output, mask = mask, training = training, ** kwargs
        )
        if not self.process_tokens: output = pooled_out

        return self.embedding_head(output, mask = mask, training = training)

class BertQA(BertClassifier):
    def __init__(self, * args, ** kwargs):
        kwargs.update({'num_classes' : 2, 'process_tokens' : True, 'process_first_token' : False})
        super().__init__(* args, ** kwargs)

    def compute_output(self, output, mask = None, training = False, ** kwargs):
        output = super().compute_output(
            output, mask = mask, training = training, ** kwargs
        )

        probs   = tf.nn.softmax(output, axis = 1)

        return probs[:, :, 0], probs[:, :, 1]

class DPR(BertEmbedding):
    @classmethod
    def from_pretrained(cls,
                        pretrained_name,
                        pretrained_task = 'question_encoder',
                        pretrained      = None,
                        ** kwargs
                       ):
        if pretrained is None:
            with tf.device('cpu') as d:
                pretrained = transformers_bert(pretrained_name, pretrained_task)

        kwargs.setdefault('output_dim', pretrained.config.projection_dim)
        kwargs.update({
            'process_tokens'    : True,
            'token_selector'    : 'first',
            'hidden_layer_type' : 'dense',
            'final_pooling'     : None,
            'use_final_dense'   : False
        })

        return BertEmbedding.from_pretrained(
            pretrained_name = pretrained_name,
            pretrained_task = pretrained_task,
            pretrained  = pretrained,
            ** kwargs
        )

def transformers_bert(name, task = 'base'):
    #tf.config.set_visible_devices([], 'GPU')
    import transformers
    if task == 'base':
        return transformers.TFBertModel.from_pretrained(name)
    elif task == 'question_encoder':
        return transformers.TFDPRQuestionEncoder.from_pretrained(name)
    elif task in ('lm', 'mlm'):
        return transformers.TFBertForMaskedLM.from_pretrained(name)
    elif task == 'nsp':
        return transformers.TFBertForNextSentencePrediction.from_pretrained(name)
    else:
        raise ValueError("Unknown task !\n  Accepted : {}\n  Got : {}".format(
            tuple(_transformers_pretrained_task.keys()), task
        ))

custom_functions    = {
    'transformers_bert' : transformers_bert,

    'BaseBERT'  : BaseBERT,
    'base_bert' : BaseBERT,
    'BertMLM'   : BertMLM,
    'bert_mlm'  : BertMLM,
    'BertNSP'   : BertNSP,
    'bert_nsp'  : BertNSP,
    'BertEmbedding' : BertEmbedding,
    'BertQA'    : BertQA,
    'bert_qa'   : BertQA,
    'DPR'       : DPR
}

custom_objects  = {
    'TextTransformerEncoder'  : TextTransformerEncoder,

    'BertPooler'        : BertPooler,
    'BaseBERT'          : BaseBERT,
    'BertMLM'           : BertMLM,
    'BertNSP'           : BertNSP,
    'BertEmbedding'     : BertEmbedding,
    'BertQA'            : BertQA,
    'DPR'               : DPR
}

_encoders   = {'Bert' : BertEmbedding, 'DPR' : DPR}
_transformers   = _encoders


import requests

from scrapy import Spider, Request, log
from scrapy.selector import Selector

from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common.exceptions import NoSuchElementException

from utils.sql import get_session
from utils.sql.models.course_unit import CourseUnit
from utils.sql.models.course_video import CourseVideo

from scraper.edx_bot.spiders import EdXLoggerIn
from scraper.edx_bot.spiders.general_course_content import GeneralCourseContent

from scraper.edx_bot.items import CourseVideoItem

class VideoTranscripts(Spider):
    name = 'video_transcripts'
    allowed_domains = ['edx.org']
    session = None

    def start_requests(self):
        self.session = get_session()
        self.edx_logger = EdXLoggerIn()

        driver = self.edx_logger.driver
        driver.maximize_window()

        for video_id in self.session.query(CourseVideo.id).filter(\
            CourseVideo.transcript == None):

            # Get a unit corresponding to that video. Though there may
            # be multiple units that host that video, the transcripts
            # should be similar across all... therefore the first unit
            # should allow us to get the transcript.
            unit_href = self.session.query(CourseUnit.href).filter(\
                CourseUnit.videos.any(CourseVideo.id == video_id)).first()

            yield Request(
                url = unit_href[0],
                meta = {'video_id':video_id[0]},
                cookies = driver.get_cookies(),
                callback = self.fetch_transcript
            )

    def fetch_transcript(self, response):
        driver = self.edx_logger.driver
        driver.get(response.url)

        sequence_el = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located(
                (By.XPATH, '//*[@id="seq_content"]')))

        for module in sequence_el.find_elements_by_xpath('.//div/div/div'):
            data_type = module.get_attribute('data-block-type')

            if data_type == 'video':
                try:
                    for download_button in module.find_elements_by_class_name('video-download-button'):
                        sub_element = download_button.find_element_by_xpath('.//a')
                        if sub_element.text == 'Download transcript':
                            transcript_href = sub_element.get_attribute('href')

                            msg = "Got transcript url=%s" % (transcript_href)
                            log.msg(msg, level=log.INFO)

                            return self.parse_transcript(transcript_href, \
                                driver.get_cookies(), meta = response.meta)

                except NoSuchElementException:
                    msg = "No txt transcript found for video in unit with url=%s" \
                        % (response.url)
                    log.msg(msg, level=log.INFO)

        return None

    def parse_transcript(self, transcript_href, driver_cookies, meta):
        cookies = {d['name']:d['value'] for d in driver_cookies}
        r = requests.get(transcript_href, cookies=cookies)

        return CourseVideoItem(
            _id = meta['video_id'],
            transcript = r.text
        )

    def closed(self, reason):
        self.session.close()
        self.edx_logger.close()


"""Defines event listeners for:

    - creating table objects
    - generating trigger/procedure DDL for audit tables.

"""
from sqlalchemy import Column, DateTime, String, Table

from sqlalchemy_postgresql_audit.ddl import (
    get_audit_spec,
    get_create_trigger_ddl,
    get_drop_trigger_ddl,
)
from sqlalchemy_postgresql_audit.dialect import (
    DEFAULT_AUDIT_TABLE_FUNCTION_NAMING_CONVENTION,
    DEFAULT_AUDIT_TABLE_NAMING_CONVENTION,
    DEFAULT_AUDIT_TABLE_TRIGGER_CONVENTION,
)

def create_audit_table(target, parent):
    """Create an audit table and generate procedure/trigger DDL.

    Naming conventions can be defined for a few of the named elements:

        - audit.table: Controls the name of the table
        - audit.function: Controls the name of the function
        - audit.trigger: Controls the name of the trigger on the table

    This function creates a new companion table to store row versions.

    Any :class:`sqlalchemy.sql.schema.Column`s specified in `table.info['session_settings']` will
    be copied and included in the audit table.

    This function will leave a key in the audited table:

        .. code-block:: python

            table.info['audit.is_audited']

    And a key in the audit table:

        .. code-block:: python

            table.info['audit.is_audit_table']

    Additionally you can find the relevant create/drop ddl at the followng keys:

        .. code-block:: python

            table.info['audit.create_ddl']
            table.info['audit.drop_ddl']

    :param target: The :class:`sqlalchemy.sql.schema.Table` to make an audit table for
    :param parent: The :class:`sqlalchemy.sql.schema.MetaData` to associate the audit table with.
    :return: None
    """
    audit_spec = get_audit_spec(target)

    if not audit_spec.get("enabled"):
        return

    audit_table_naming_convention = parent.naming_convention.get(
        "audit.table", DEFAULT_AUDIT_TABLE_NAMING_CONVENTION
    )
    audit_function_naming_convention = parent.naming_convention.get(
        "audit.function", DEFAULT_AUDIT_TABLE_FUNCTION_NAMING_CONVENTION
    )
    audit_trigger_naming_convention = parent.naming_convention.get(
        "audit.trigger", DEFAULT_AUDIT_TABLE_TRIGGER_CONVENTION
    )

    audit_table_name = audit_table_naming_convention % {
        "table_name": target.name,
        "schema": audit_spec["schema"] or "public",
    }
    audit_function_name = audit_function_naming_convention % {
        "table_name": target.name,
        "schema": audit_spec["schema"] or "public",
    }
    audit_trigger_name = audit_trigger_naming_convention % {
        "table_name": target.name,
        "schema": audit_spec["schema"] or "public",
    }

    columns = [
        Column(col.name, col.type, nullable=True) for col in target.columns.values()
    ]
    session_setting_columns = [col.copy() for col in audit_spec["session_settings"]]
    for col in session_setting_columns:
        col.name = "audit_{}".format(col.name)

    column_elements = session_setting_columns + columns

    audit_table = Table(
        audit_table_name,
        target.metadata,
        Column("audit_operation", String(1), nullable=False),
        Column("audit_operation_timestamp", DateTime, nullable=False),
        Column("audit_current_user", String(64), nullable=False),
        *column_elements,
        schema=audit_spec["schema"]
    )

    target.info["audit.create_ddl"] = get_create_trigger_ddl(
        target.columns,
        audit_table.columns,
        audit_function_name,
        audit_trigger_name,
        target.fullname,
        audit_table.fullname,
        session_setting_columns,
    )

    target.info["audit.drop_ddl"] = get_drop_trigger_ddl(
        audit_function_name, audit_trigger_name, target.fullname
    )
    audit_table.info["audit.target_table"] = target

    audit_table.info["audit.is_audit_table"] = True
    target.info["audit.is_audited"] = True


import tensorflow as tf
from tensorflow import keras

import numpy as np

print(tf.__version__)

# Load in the built-in housing dataset
boston_housing = keras.datasets.boston_housing

(train_data, train_labels), (test_data, test_labels) = boston_housing.load_data()

# Shuffle the training set
order = np.argsort(np.random.random(train_labels.shape))
train_data = train_data[order]
train_labels = train_labels[order]

# Use pandas library to display the first several rows of the dataset in a table
import pandas as pd

column_names = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD',
                'TAX', 'PTRATIO', 'B', 'LSTAT']

df = pd.DataFrame(train_data, columns=column_names)
print(df.head())

# Test data is *not* used when calculating the mean and std.
# Normalizing data - (subtract the mean of the feature and divide by standard deviation)
# Makes training far easier.

mean = train_data.mean(axis=0)
std = train_data.std(axis=0)
# Saving copy of original test_data to display in table at the end of run.
original_test_data = test_data
train_data = (train_data - mean) / std
test_data = (test_data - mean) / std

# Seuquential model with two densely connected hidden layers.
# Output layer returns single, continuous value.

def build_model():
  model = keras.Sequential([
    keras.layers.Dense(64, activation=tf.nn.relu,
                       input_shape=(train_data.shape[1],)),
    keras.layers.Dense(64, activation=tf.nn.relu),
    keras.layers.Dense(1)
  ])

  optimizer = tf.train.RMSPropOptimizer(0.001)

  model.compile(loss='mse',
                optimizer=optimizer,
                metrics=['mae'])
  return model

# model = build_model()
# model.summary()

# Display training progress by printing a single dot for each completed epoch.
class PrintDot(keras.callbacks.Callback):
  def on_epoch_end(self,epoch,logs):
    if epoch % 100 == 0: print('')
    print('.', end='')

EPOCHS = 500

# Store training stats
# history = model.fit(train_data, train_labels, epochs=EPOCHS,
#                     validation_split=0.2, verbose=0,
#                     callbacks=[PrintDot()])

import matplotlib as mpl
mpl.use('TkAgg')
import matplotlib.pyplot as plt

# Graph to display model's improvement.
def plot_history(history):
  plt.figure()
  plt.xlabel('Epoch')
  plt.ylabel('Mean Abs Error [1000$]')
  plt.plot(history.epoch, np.array(history.history['mean_absolute_error']),
           label='Train Loss')
  plt.plot(history.epoch, np.array(history.history['val_mean_absolute_error']),
           label = 'Val loss')
  plt.legend()
  plt.ylim([0,5])
  plt.show()

# plot_history(history)

model = build_model()

# The patience parameter is the amount of epochs to check for improvement.
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)

history = model.fit(train_data, train_labels, epochs=EPOCHS,
                    validation_split=0.2, verbose=0,
                    callbacks=[early_stop, PrintDot()])

plot_history(history)

# Evaluate the model using the testing data and get the total loss as well as
# Mean Absolute Error - which is a common regression metric. In this case,
# it represents the average difference in prediction price to actual price.
[loss, mae] = model.evaluate(test_data, test_labels, verbose=0)

print("\nTesting set Mean Abs Error: ${:7.2f}".format(mae * 1000))

test_predictions = model.predict(test_data).flatten()

# Live coding done below:


import numpy as np

def createVoc(dataset):
    voc_set = set([])
    for i in dataset:
        voc_set = voc_set | set(i)
    return list(voc_set)

def docVec(voc_list, input_doc):
    return_voc = [0]*len(voc_list)
    print_list = []
    for i in input_doc:
        if i in voc_list:
            return_voc[voc_list.index(i)] += 1
        else:
            print_list.append(i)
    # if len(print_list)>0:
    #     print("The words \"",print_list,"\" is not in my vocabulary !")
    return return_voc

def trainNB(train_mat, train_label):
    num_docs = len(train_mat)
    num_words = len(train_mat[0])
    pc = sum(train_label)/float(num_docs)
    p0 = np.ones(num_words)
    p1 = np.ones(num_words)
    p0num = 2.0
    p1num = 2.0
    for i in range(num_docs):
        if train_label[i] == 1:
            p1 += train_mat[i]
            p1num += sum(train_mat[i])
        else:
            p0 += train_mat[i]
            p0num += sum(train_mat[i])
    p0vec = np.log(p0/p0num)
    p1vec = np.log(p1/p1num)
    return p0vec, p1vec, pc

def classify(vec,p0,p1,pc):
    p0 = sum(vec*p0)+np.log(1-pc)
    p1 = sum(vec*p1)+np.log(pc)
    if(p1>p0):
        return 1
    else:
        return 0

def textParse(doc_string):
    import re
    token_list = re.split(r'\W*', doc_string)
    return [token.lower() for token in token_list if(len(token)>2)]

def spamTest():
    doc_list = []; class_list = []; full_list = []
    for i in range(1,26):
        word_list = textParse(open("email/spam/%d.txt"%i).read())
        doc_list.append(word_list)
        full_list.extend(word_list)
        class_list.append(1)
        word_list = textParse(open("email/ham/%d.txt"%i).read())
        doc_list.append(word_list)
        full_list.extend(word_list)
        class_list.append(0)
    voc_list = createVoc(full_list)
    traing_set = list(range(50))
    test_set = []
    for i in range(10):
        rand_index = int(np.random.uniform(0, len(traing_set)))
        test_set.append(traing_set[rand_index])
        del(traing_set[rand_index])
    train_mat = []
    train_class = []
    for i in traing_set:
        train_mat.append(docVec(voc_list, doc_list[i]))
        train_class.append(class_list[i])
    p0,p1,pc = trainNB(np.array(train_mat),np.array(train_class))
    error_count = 0
    for i in test_set:
        word_vec = docVec(voc_list, doc_list[i])
        if classify(np.array(word_vec),p0,p1,pc)!=class_list[i]:
            error_count+=1
    print("The error rate is : ",float(error_count)/len(test_set))


import numpy as np

"""
Solves the linear system Ux=b by backsubstitution,
where U is a Upper Triangular matrix (There is no check for U to be Upper Triangular Matrix implemented)
This is an implementation of Algorithm 3.3 from Scientific Computing Book.
"""

def BackSubstitution(U,b):
    n = b.shape[0]
    x = np.zeros(n)
    for k in range(n-1,-1,-1):
        s = b[k]
        for j in range(k+1,n):
            s -= U[k,j] * x[j]
        x[k] = s/U[k,k]

    return x

"""
Solves the linear system Ux = b by a variant of Backsubstitution called SAXPY (scalar a.x plus y)
where U is an Upper Triangular Matrix.
This is an implementation of Algorithm 3.4 from Scientific Computing Book.
"""

def BackSubstitutionSAXPY(U,b):
    n = b.shape[0]
    x = np.zeros(n)
    for i in range(n-1, -1, -1):
        x[i] = b[i]/U[i,i]
        b[0:i-1] -= x[i] * U[0:i-1, i]
    return x

print(BackSubstitution(np.array(((1,0,0),(0,1,0), (0,0,1))), np.array(((2),(3),(4)))))
print(BackSubstitutionSAXPY(np.array(((1,0,0),(0,1,0), (0,0,1))), np.array(((2.),(3.),(5.)))))


from django import template

from categories.models import Category

register = template.Library()

@register.inclusion_tag('categories/categories_menu.html')
def list_categories(category):
    """
    Create a dictionary with all the categories
    :param category: this is a Category object
    :return: a dictionary with the categories
    """
    categories = Category.objects.all()
    return {'categories': categories}


#%% Pulling Data from GHCN Daily Summary Files
# load modules
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import datetime
import time

#%% Loading Individual Station Data
def load_station(fname):
    # this loads in station data for an individual station
    df = pd.read_csv(fname, dtype={'DATE': object, 'PRCP': float, 'TMAX': float, 'TMIN': float, 'PRCP_ATTRIBUTES': str,
                                   'TMAX_ATTRIBUTES': str, 'TMIN_ATTRIBUTES': str}, low_memory=False)
    data = df.filter(['DATE', 'PRCP', 'TMAX', 'TMIN', 'PRCP_ATTRIBUTES', 'TMAX_ATTRIBUTES', 'TMIN_ATTRIBUTES'])
    data['DATE'] = pd.to_datetime(data['DATE'])
    del df
    return data

#%% Exploding Flags and NaN-ing Flagged Values
def explode_flags(compact_data):
    # this expands flagged data and automatically reassigns flagged values as nan
    min_ind = compact_data.index[compact_data['PRCP_ATTRIBUTES'].notnull()].min()
    if compact_data['PRCP_ATTRIBUTES'][min_ind].count(',') == 3:
        compact_data[['PRCP_MEAS_FLAG', 'PRCP_QUAL_FLAG', 'PRCP_SOURCE_CODE', 'PRCP_TIME']] = compact_data['PRCP_ATTRIBUTES'].str.split(',', expand=True)
    elif compact_data['PRCP_ATTRIBUTES'][min_ind].count(',') == 2:
        compact_data[['PRCP_MEAS_FLAG', 'PRCP_QUAL_FLAG', 'PRCP_SOURCE_CODE']] = compact_data['PRCP_ATTRIBUTES'].str.split(',', expand=True)
    flag_mask = (compact_data['PRCP_QUAL_FLAG'].isna()) | (compact_data['PRCP_QUAL_FLAG'].isin(qual_flags))
    compact_data['PRCP'] = compact_data['PRCP'].where(~flag_mask, np.nan)
    return compact_data

#%% Creating Year Arrays
def get_days_in_year2(years_array, first_year, last_year):
    # gets the number of days in a year
    days_in_year_array = np.zeros(np.shape(years_array))
    i = 0
    for yr in years_array:
        if yr % 4 == 0:
            days_in_year_array[i] = 366
        else:
            days_in_year_array[i] = 365
        i += 1
    total_years_array = np.arange(first_year, last_year+1)
    total_days_array = np.zeros(np.shape(total_years_array))
    i = 0
    for yr in total_years_array:
        if yr % 4 == 0:
            total_days_array[i] = 366
        else:
            total_days_array[i] = 365
        i += 1
    total_days = sum(total_days_array)
    total_days_series = pd.Series(total_days_array, index=total_years_array)
    return days_in_year_array, total_days, total_days_series, total_years_array

#%% Checking for missing data
def get_pct_missing(station_data, days_in_year, total_days, years_array, total_days_series):
    # calculates the amount of missing data in a given station-year and returns relevant summary statistics
    date_mask = (station_data['DATE'].dt.year >= start_year) & (station_data['DATE'].dt.year <= last_year)
    num_days_in_data_year = station_data['PRCP'].groupby(station_data['DATE'][date_mask].dt.year).count()
    num_days_in_data_year_series = pd.Series(num_days_in_data_year, years_array)
    yearly_pct_missing  = 100 - (num_days_in_data_year_series.divide(total_days_series, fill_value=0)*100)
    total_pct_missing = 100 - sum(num_days_in_data_year)/total_days*100
    return yearly_pct_missing, total_pct_missing

#%% Loading Station Metadata File
width = [
    11, #station_id
    9, #latitude
    10, #longitude
    7, #elevation
    3, #state
    31, #name
    4, #gsn_id
    4, #hcn_crn_id
    6] #wmo_id

df = pd.read_fwf('/projects/b1045/GHCN/ghcnd-stations.txt',
                 widths=width, names=['station_id', 'lat', 'lon', 'elev', 'state', 'name', 'gsn_id', 'hcn_crn_id', 'wmo_id'])
stations_meta = df
del df

# df = pd.read_csv('/Users/ryanharp/Documents/great_lakes_precip_variability/supplemental_files/ghcnd-stations_file_compare.csv')
df = pd.read_csv('/projects/b1045/GHCN/ghcnd-stations_file_compare.csv')
unavailable_files = df['missing_from_available']
del df

#%% Preallocating variables
station_id = np.empty(np.shape(stations_meta)[0], dtype=object); station_id[:] = np.nan
latitude = np.zeros(np.shape(stations_meta)[0]); latitude[:] = np.nan
longitude = np.zeros(np.shape(stations_meta)[0]); longitude[:] = np.nan
elevation = np.zeros(np.shape(stations_meta)[0]); elevation[:] = np.nan
name = np.zeros(np.shape(stations_meta)[0], dtype=object); name[:] = np.nan

start_year_array = np.zeros(np.shape(stations_meta)[0]); start_year_array[:] = np.nan
last_year_array = np.zeros(np.shape(stations_meta)[0]); last_year_array[:] = np.nan
num_years_qual_array = np.zeros(np.shape(stations_meta)[0]); num_years_qual_array[:] = np.nan
pct_years_qual_array = np.zeros(np.shape(stations_meta)[0]); pct_years_qual_array[:] = np.nan
total_pct_missing_array = np.zeros(np.shape(stations_meta)[0]); pct_years_qual_array[:] = np.nan
test_array = np.zeros([np.shape(stations_meta)[0], 2]); test_array[:] = np.nan
qual_years_array = list([])
annual_pct_missing_array = list([])
ind_array = list([])

#%% Looping over all stations
qual_flags = ['D', 'G', 'I', 'K', 'L', 'M', 'N', 'O', 'R', 'S', 'T', 'W', 'X', 'Z']
yearly_pct_threshold = 10  # percentage of missing days allowed in a qualifying year
start_time = time.time()

for ind, row in stations_meta[~stations_meta['station_id'].isin(unavailable_files)].iterrows():
    print(ind)
    fname = '/projects/b1045/GHCN/' + row.station_id + '.csv'
    temp_data = load_station(fname)
    if ("PRCP" in temp_data) is False:
        print('skipping: no precip data')
        continue
    elif ("PRCP_ATTRIBUTES" in temp_data) is False:
        print('skipping: no precip data')
        continue
    station_data = explode_flags(temp_data)

    # getting first/last date
    start_date = station_data['DATE'].min()
    start_year = start_date.date().year
    last_date = station_data['DATE'].max()
    last_year = last_date.date().year
    start_year_array[ind] = start_year
    last_year_array[ind] = last_year

    # getting year arrays
    years_array = station_data['DATE'].dt.year.unique()
    days_in_year, total_days, total_days_series, total_years_array = get_days_in_year2(years_array, start_year, last_year)
    yearly_pct_missing, total_pct_missing = get_pct_missing(station_data, days_in_year, total_days, years_array, total_days_series)
    qual_years = total_years_array[yearly_pct_missing <= yearly_pct_threshold]
    num_qual_years = len(qual_years)
    pct_years_qual = num_qual_years/len(total_years_array)*100
    num_years_qual_array[ind] = num_qual_years
    pct_years_qual_array[ind] = pct_years_qual
    total_pct_missing_array[ind] = total_pct_missing
    qual_years_array.append(qual_years)
    annual_pct_missing_array.append(yearly_pct_missing)
    station_id[ind] = row.station_id
    latitude[ind] = row.lat
    longitude[ind] = row.lon
    elevation[ind] = row.elev
    name[ind] = row['name']
    ind_array.append(ind)

end_time = time.time()
print(end_time-start_time)

len_years = last_year_array - start_year_array

#%% Creating metadata file
# creating pandas dataframe
ghcn_summary = pd.DataFrame({'station_id': [], 'latitude': [], 'longitude': [], 'elevation': [], 'name': [],
                              'start_year': [], 'last_year': [], 'len_years': [], 'num_qual_years': [],
                              'pct_qual_years': [], 'pct_days_missing': []})
# populating dataframe
ghcn_summary['station_id'] = station_id
ghcn_summary['latitude'] = latitude
ghcn_summary['longitude'] = longitude
ghcn_summary['elevation'] = elevation
ghcn_summary['name'] = name
ghcn_summary['start_year'] = start_year_array
ghcn_summary['last_year'] = last_year_array
ghcn_summary['len_years'] = len_years
ghcn_summary['num_qual_years'] = num_years_qual_array
ghcn_summary['pct_qual_years'] = pct_years_qual_array
ghcn_summary['pct_days_missing'] = total_pct_missing_array

ghcn_summary = ghcn_summary[~ghcn_summary['station_id'].isnull()]
ghcn_summary.to_csv('/home/rdh0715/ghcn_summary.csv')  # saving metadata/summary file


import re
import pandas as pd
import pdb
import numpy as np
#######################################
# Jan 2021 - Jared - preprocess jobs
#         `
#######################################
glm_all_f = pd.read_csv("../../results/glm_transfer/RMSE_transfer_glm_pball.csv")
train_lakes = [re.search('nhdhr_(.*)', x).group(1) for x in np.unique(glm_all_f['target_id'].values)]
ids = pd.read_csv('../../metadata/pball_site_ids.csv', header=None)
ids = ids[0].values
test_lakes = ids[~np.isin(ids, train_lakes)]

sbatch = ""
ct = 0
# start = [0,1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000]
# end = [1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,12416]

for i in test_lakes:
    ct += 1
    #for each unique lake
    print(i)

    # if not os.path.exists("../../../models/single_lake_models/"+name+"/PGRNN_basic_normAll_pball"):
    header = "#!/bin/bash -l\n#SBATCH --time=00:05:00\n#SBATCH --ntasks=8\n#SBATCH --mem=20g\n#SBATCH --mail-type=ALL\n#SBATCH --mail-user=willa099@umn.edu\n#SBATCH --gres=gpu:k40:2\n#SBATCH --output=test_ea145_%s.out\n#SBATCH --error=test_ea145_%s.err\n\n#SBATCH -p k40"%(i,i)
    script = "source /home/kumarv/willa099/takeme_evaluate_MTL_tst.sh\n" #cd to directory with training script
    # script2 = "python write_NLDAS_xy_pairs.py %s %s"%(l,l2)
    script2 = "python testEALSTM.py %s"%(i)
    # script2 = "python predict_lakes_EALSTM_COLD_DEBUG.py %s %s"%(l,l2)

    # script3 = "python singleModel_customSparse.py %s"%(l)
    all= "\n".join([header,script,script2])
    sbatch = "\n".join(["sbatch ea145_test_%s.sh"%(i),sbatch])
    with open('../../hpc/ea145_test_{}.sh'.format(i), 'w') as output:
        output.write(all)

compile_job_path= '../../hpc/ea145test_jobs.sh'
with open(compile_job_path, 'w') as output2:
    output2.write(sbatch)

print(ct, " jobs created, run this to submit: ", compile_job_path)


import json
from django.http import JsonResponse
from rest_framework.decorators import api_view,authentication_classes,permission_classes
from allauth.account.utils import perform_login
from allauth.account import app_settings as allauth_settings
from app_api.functions.masterdata import auth_user
from .functions.services import addUserService, authentication_service, saveProfileDetails, getModuleLessonService, saveCourseRating, saveAskQuestion, getAskQuestion, \
        getlessonVideoService, saveAssessmentService, updateAssessmentService, saveVideoActivityService,courseModuleNameService
from .models import User_data
from django.views.decorators.csrf import csrf_exempt

@api_view(['POST'])
@authentication_classes([])
@permission_classes([])
def addUser(request):
    response ={
        'data':None,
        'error':None,
        'statusCode':1
    }
    try:

        if request.method == 'POST':
            dataObjs = json.loads(request.POST.get('data'))
            addUserService(dataObjs)
            response['data'] = "Details Saved Sucessfully"
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in Registration'
        response['error'] = str(e)
        raise

    return JsonResponse(response)

@csrf_exempt
@api_view(['POST'])
@authentication_classes([])
@permission_classes([])
def loginView(request):
    response = {
        'data':None,
        'error':None,
        'statusCode':1
    }
    try:

        if request.method == "POST":

            dataObjs = json.loads(request.POST.get('data'))
            auth_token = authentication_service(dataObjs)
            if auth_token[0] != None:
                usr = User_data.objects.filter(usr_email=dataObjs['email']).first()
                perform_login(request._request, usr, allauth_settings.EMAIL_VERIFICATION, signup=False,
                              redirect_url=None, signal_kwargs=None)
                user = auth_user(usr)

                response['token'] = 'token_generated'
                response['data'] = 'Successfully logged-in'
                response['login_type'] = auth_token[1]
            else:
                response['token'] = 'AnonymousUser'
                response['data'] = 'login failed'

            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in Sign-in'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
def saveProfile(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            fileObjs = request.FILES
            dataObjs = json.loads(request.POST.get('data'))
            saveProfileDetails(dataObjs, fileObjs)
            response['data'] = "Profile Details Saved Sucessfully"
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in save profile'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
def saveRating(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            dataObjs = json.loads(request.POST.get('data'))
            user = request.user
            course_rating = saveCourseRating(dataObjs,user)
            response['data'] = course_rating
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in saving Course rating'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
def getModuleLesson(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            dataObjs = json.loads(request.POST.get('data'))
            getModulesLessons = getModuleLessonService(dataObjs)
            response['data'] = getModulesLessons
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in getModuleLesson'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
def saveQuestion(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            dataObjs = json.loads(request.POST.get('data'))

            dataObjs['userId'] = request.user
            saveAskQuestion(dataObjs)

            response['data'] = 'Question Saved'
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in saveQuestion'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
def getQuestion(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            dataObjs = json.loads(request.POST.get('data'))

            userId = request.user

            getquestionsList = getAskQuestion(dataObjs,userId)

            response['data'] = getquestionsList
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in saveQuestion'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
def getlessonVideoDetails(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            dataObjs = json.loads(request.POST.get('data'))
            user = request.user
            lesson_video = getlessonVideoService(dataObjs,user)
            response['data'] = lesson_video
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in saving Course rating'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
def saveAssessmentDetails(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            dataObjs = json.loads(request.POST.get('data'))
            user = request.user
            saveAssessmentService(dataObjs,user)
            response['data'] = 'Assessment details saved successfully'
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in saving Course rating'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
@authentication_classes([])
@permission_classes([])
def updateAssessmentDetails(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            request_data = dict(request.POST)
            dataObjs = {key: request_data[key][0] if request_data[key] else '' for key in request_data}
            updateAssessmentService(dataObjs)
            response['data'] = 'Assessment details updated successfully'
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in saving assessment details'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
def saveVideoActivity(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        if request.method == "POST":
            dataObjs = json.loads(request.POST.get('data'))
            user = request.user
            saveVideoActivityService(dataObjs,user)
            response['data'] = 'Video activity saved successfully'
            response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in saving video activity'
        response['error'] = str(e)
        raise
    return JsonResponse(response)

@api_view(['POST'])
@authentication_classes([])
@permission_classes([])
def courseModuleName(request):
    response = {
        'data': None,
        'error': None,
        'statusCode': 1
    }
    try:
        dataObjs = request.data
        dataObjs = json.dumps(dataObjs)

        data = json.loads(dataObjs)

        course = data['coursid']
        module = data['modid']

        coursetitels = courseModuleNameService(course, module)

        response['courseTitles'] = coursetitels
        response['statusCode'] = 0

    except Exception as e:
        response['data'] = 'Error in courseModuleName'
        response['error'] = str(e)
        raise
    return JsonResponse(response)


#!/usr/bin/env python3
# coding: utf-8
import numpy as np
import scipy.special
from tqdm import tqdm
from scipy import linalg
import concurrent.futures

def proc(w, f, V, **kwargs):
    return np.block([f(w, vec.T, **kwargs) for vec in V.T])

class SSM:
    def __init__(self, L, K, delta, eps, n):
        self.L = L # block SSMの列数 (計算可能な重複固有値の数の上限) L
        self.K = K # 計算可能な固有値の数の上限 = K*L
        self.delta = delta # 特異値の切り捨て
        self.eps = eps # 固有値の有無の判定基準
        self.n = n # 行列のサイズ

        if self.L > self.n:
            print("# Error: L > n")
            exit(-1)

        #
        # V = column-fullrank random matrix
        #
        fullrank = False
        while(not fullrank):
            self.V = np.matrix(np.random.rand(self.n,self.L) + 1j*np.random.rand(self.n,self.L))
            # V should not be rank deficient
            u, s, vh = np.linalg.svd(self.V, full_matrices=False)
            fullrank = s[-1]/s[0] > 1e-4

        #
        # U = column-fullrank random matrix
        #
        fullrank = False
        while(not fullrank):
            self.U = np.matrix(np.random.rand(self.n,self.L) + 1j*np.random.rand(self.n,self.L))
            # vmat should not be rank deficient
            u, s, vh = np.linalg.svd(self.U, full_matrices=False)
            fullrank = s[-1]/s[0] > 1e-4

    # f = A^{-1}(z)*ベクトルを計算する関数
    def run(self, f, n, rho, z0, parallel=False, max_workers=4, **kwargs):
        #
        # compute S
        #
        S = [np.zeros((self.n,self.L),dtype=complex) for k in range(2*self.K)]
        # for each integration point w
        # Ys = np.ndarray((n,self.n,self.L),dtype=complex)
        i = 0
        ws = [z0+rho*np.exp(1j*theta) for theta in np.linspace(0,2*np.pi-(2*np.pi/n),n)]

        # 並列化
        if parallel:
            #with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as excuter:
            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as excuter:
                # Ys = list(tqdm(excuter.map(proc, ws, [f for _ in range(n)], [self.V for _ in range(n)]), total=n))
                Ys = list(
                    tqdm(
                        excuter.map(lambda a,b,c,d: proc(a,b,c,**d), ws, [f for _ in range(n)], [self.V for _ in range(n)], [kwargs for _ in range(n)]), total=n
                        )
                    )
        else:
            # Ys = list(map(proc, ws, [f for _ in range(n)], [self.V for _ in range(n)]))
            Ys = []
            for w, f, V in tqdm(zip(ws,[f for _ in range(n)], [self.V for _ in range(n)]), total=n):
                Ys.append(proc(w,f,V, **kwargs))

        Ymax = 0.0
        for w, Y in zip(ws,Ys):
            # store max value
            Ymax = max(Ymax,np.max(np.abs(Y)))

            # compute S matrix for each k=0,1,..,2*K-1
            zc = (w-z0)/rho
            for k in range(2*self.K):
                S[k] += zc * Y
                zc = zc * (w-z0)/rho

        print("# ")
        print("# SSM results")
        print("# ")

        # if S[0] is small, then no eigenvalue
        print("# S[0]", np.max(abs(S[0])))
        print("# max(|Y|): ", Ymax/n)
        if(np.max(abs(S[0])) < Ymax/n * self.eps):
            print("# No eigenvalue found")
            return [], []

        # Compute M
        M = [self.U.H @ S[k] for k in range(2*self.K)]
        # print(M[0])
        # exit(0)

        # Hankel matrix
        H = np.zeros((self.L*self.K, self.L*self.K), dtype=complex)
        Hs = np.zeros((self.L*self.K, self.L*self.K), dtype=complex)
        jj = 0
        for j in range(self.K):
            ii = 0
            for i in range(self.K):
                H[ii:ii+self.L, jj:jj+self.L] = M[i+j]
                Hs[ii:ii+self.L, jj:jj+self.L] = M[i+j+1]

                ii += self.L

            jj += self.L

        # SVD of H
        u, s, vh = np.linalg.svd(H, full_matrices=False)
        nev = sum(e/s[0] > self.delta for e in s) # number of eigenvalues
        print("# Singular values: ")
        print(s)
        print("# ")

        H = H[0:nev,0:nev]
        Hs = Hs[0:nev,0:nev]
        ev, vr = linalg.eig(Hs, b=H, right=True)

        if any(abs(e)>1 for e in ev):
            print("# Warning: Eigenvalue found outside the path")

        if 2*self.K < nev:
            print("# Error: too small K, nev = ", nev)
            exit(1)

        print("# Number of eigenvalues: ", nev)
        print("# Eigenvalues: ", [rho*zeta+z0 for zeta in ev])

        # Compute right eigenvector
        Sh = np.block([S[k] for k in range(nev)])
        Sh = Sh[:,0:nev]
        # print(Sh.shape, vr.shape)
        vectors = Sh @ vr

        # Normalize eigenvectors
        for i in range(nev):
            norm = 1 / vectors[np.argmax(np.abs(vectors[:,i])),i]
            vectors[:,i] = vectors[:,i] * norm

        return [rho*zeta+z0 for zeta in ev], vectors.T


from typing import List
from unittest import TestCase, main
from itertools import product

class Solution:
    def letterCombinations(self, digits: str) -> List[str]:
        phoneDict = {
            '2': ["a", "b", "c"],
            '3': ["d", "e", "f"],
            '4': ["g", "h", "i"],
            '5': ["j", "k", "l"],
            '6': ["m", "n", "o"],
            '7': ["p", "q", "r", "s"],
            '8': ["t", "u", "v"],
            '9': ["w", "x", "y", "z"],
            '0': [" "]
        }
        digitsList = list(digits)
        searchList = []
        for dig in digitsList:
            searchList.append(phoneDict.get(dig))

        ans = []
        for com in list(product(*searchList)):
            if len(com) != 0:
                ans.append(''.join(com))

        return ans

class TestSolution(TestCase):
    def setUp(self) -> None:
        self.sol = Solution()

    def test_first_case(self):
        ans = self.sol.letterCombinations(digits="23")
        self.assertEqual(ans, ["ad", "ae", "af", "bd", "be", "bf", "cd", "ce", "cf"])

    def test_second_case(self):
        ans = self.sol.letterCombinations(digits="")
        self.assertEqual(ans, [])

if __name__ == '__main__':
    main()


#Question: Given an array, find the length of the smallest subarray such that if you sort the subarray the entire array becomes sorted
#Solution: Sort array and see differences, (Naive approach)

#Naive approach
def findUnsortedSubarrayN(arr):
        """
        :type nums: List[int]
        :rtype: int
        """
        cmin, cmax, end, beg = arr[-1], arr[0], -2, -1
        for i in range(1, len(arr)):
            cmax = max(cmax, arr[i])
            cmin = min(cmin, arr[len(arr) - 1 - i])
            if arr[i] < cmax: end = i
            if arr[len(arr) - 1 - i] > min: beg = len(arr) - 1 - i
        return end - beg + 1

#Optimal approach
def findUnsortedSubarray(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        l, r = -1, 0
        mmin, mmax = float('inf'), float('-inf')
        for i in range(len(nums)):
            mmax = max(mmax, nums[i])
            if nums[i] != mmax: l = i

            mmin = min(mmin, nums[len(nums) - i - 1])
            if nums[len(nums) - i - 1] != mmin: r = len(nums) - i - 1
        print(l, r)
        return l - r + 1

def main():
    a = [1, 4, 6, 8, 3, 2, 9]
    arr = [2,6,4,8,10,9,15]

    print(smallesUnsortedSubarray(a))
    print(smallesUnsortedSubarray(arr))

main()


from functools import partial
from kivy.properties import (DictProperty, ListProperty, ObjectProperty,
								StringProperty, NumericProperty,
								VariableListProperty)
from kivy.core.text import DEFAULT_FONT
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import Button
from kivy.uix.widget import Widget
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.scrollview import ScrollView
from kivy.clock import Clock
from kivy.lang import Builder
from kivy.logger import Logger

class Box(Widget):
	padding = VariableListProperty(6)
	border = VariableListProperty(4)
	font_size = StringProperty('15sp')
	font_name = StringProperty(DEFAULT_FONT)
	background = StringProperty()
	background_color = VariableListProperty([1, 1, 1, 1])
	foreground_color = VariableListProperty([0, 0, 0, 1])

class ScrollableLabel(ScrollView):
	font_size = StringProperty('15sp')
	font_name = StringProperty(DEFAULT_FONT)
	color = VariableListProperty([0, 0, 0, 1])
	line_spacing = NumericProperty(0)
	text = StringProperty()

class MessageBox(Box):
	"""Looks like a TextInput but doesn't accept any input.

	Does support styled text with BBcode.

	"""
	line_spacing = NumericProperty(0)
	text = StringProperty()

class DialogMenu(Box):
	"""Some buttons that make the game do things.

	Set ``options`` to a list of pairs of ``(text, function)`` and the
	menu will be populated with buttons that say ``text`` that call
	``function`` when pressed.

	"""
	options = ListProperty()
	"""List of pairs of (button_text, callable)"""

	def _set_sv_size(self, *args):
		self._sv.width = self.width - self.padding[0] - self.padding[2]
		self._sv.height = self.height - self.padding[1] - self.padding[3]

	def _set_sv_pos(self, *args):
		self._sv.x = self.x + self.padding[0]
		self._sv.y = self.y + self.padding[3]

	def on_options(self, *args):
		if not hasattr(self, '_sv'):
			self._sv = ScrollView(size=self.size, pos=self.pos)
			self.bind(size=self._set_sv_size, pos=self._set_sv_pos)
			layout = BoxLayout(orientation='vertical')
			self._sv.add_widget(layout)
			self.add_widget(self._sv)
		else:
			layout = self._sv.children[0]
			layout.clear_widgets()
		for txt, part in self.options:
			if not callable(part):
				raise TypeError("Menu options must be callable")
			butn = Button(text=txt,
							on_release=part,
							font_name=self.font_name,
							font_size=self.font_size,
							valign='center',
							halign='center')
			butn.bind(size=butn.setter('text_size'))
			layout.add_widget(butn)

class Dialog(BoxLayout):
	"""MessageBox with a DialogMenu beneath it.

	Set the properties ``message_kwargs`` and ``menu_kwargs``,
	respectively, to control them -- but you probably want
	to do that by returning a pair of dicts from an action
	in LiSE.

	"""
	message_kwargs = DictProperty({})
	menu_kwargs = DictProperty({})

	def __init__(self, **kwargs):
		super().__init__(**kwargs)
		self.bind(message_kwargs=self._propagate_msg_kwargs,
					menu_kwargs=self._propagate_menu_kwargs)
		self._propagate_msg_kwargs()
		self._propagate_menu_kwargs()

	def _propagate_msg_kwargs(self, *args):
		if 'msg' not in self.ids:
			Clock.schedule_once(self._propagate_msg_kwargs, 0)
			return
		kw = dict(self.message_kwargs)
		kw.setdefault('background',
						'atlas://data/images/defaulttheme/textinput')
		for k, v in kw.items():
			setattr(self.ids.msg, k, v)

	def _propagate_menu_kwargs(self, *args):
		if 'menu' not in self.ids:
			Clock.schedule_once(self._propagate_menu_kwargs, 0)
			return
		kw = dict(self.menu_kwargs)
		kw.setdefault(
			'background',
			'atlas://data/images/defaulttheme/vkeyboard_background')
		for k, v in kw.items():
			setattr(self.ids.menu, k, v)

class DialogLayout(FloatLayout):
	"""A layout, normally empty, that can generate dialogs

	To make dialogs, set my ``todo`` property to a list. It may contain:

	* Strings, which will be displayed with an "OK" button to dismiss them
	* Lists of pairs of strings and callables, which generate buttons with the
	  string on them that, when clicked, call the callable
	* Lists of pairs of dictionaries, which are interpreted as keyword
	  arguments to :class:`MessageBox` and :class:`DialogMenu`

	In place of a callable you can use the name of a function in my
	``usermod``, a Python module given by name. I'll import it when I need it.

	Needs to be instantiated with a LiSE ``engine`` -- probably an
	:class:`EngineProxy`.

	"""
	dialog = ObjectProperty()
	engine = ObjectProperty()
	todo = ListProperty()
	usermod = StringProperty('user')
	userpkg = StringProperty(None, allownone=True)

	def __init__(self, **kwargs):
		super().__init__(**kwargs)
		self.dialog = Dialog()

	def on_engine(self, *args):
		todo = self.engine.universal.get('last_result')
		if isinstance(todo, list):
			self.todo = todo
		else:
			self.todo = []
		self.idx = self.engine.universal.get('last_result_idx', 0)
		self.engine.universal.connect(self._pull)
		if self.todo:
			self.advance_dialog()

	def _pull(self, *args, key, value):
		if key == 'last_result':
			self.todo = value if value and isinstance(value, list) else []
		elif key == 'last_result_idx':
			self.idx = value if value and isinstance(value, int) else 0

	def on_idx(self, *args):
		lidx = self.engine.universal.get('last_result_idx')
		if lidx is not None and lidx != self.idx:
			self.engine.universal['last_result_idx'] = self.idx
		Logger.debug(f"DialogLayout.idx = {self.idx}")

	def advance_dialog(self, after_ok=None, *args):
		"""Try to display the next dialog described in my ``todo``.

		:param after_ok: An optional callable. Will be called after the user clicks
		a dialog option--as well as after any game-specific code for that option
		has run.

		"""
		self.clear_widgets()
		try:
			Logger.debug(f"About to update dialog: {self.todo[0]}")
			self._update_dialog(self.todo.pop(0), after_ok)
		except IndexError:
			after_ok()

	def _update_dialog(self, diargs, after_ok, **kwargs):
		if diargs is None:
			Logger.debug("DialogLayout: null dialog")
			return
		if isinstance(diargs, tuple) and diargs[0] == "stop":
			if len(diargs) == 1:
				Logger.debug("DialogLayout: null dialog")
				return
			diargs = diargs[1]
		dia = self.dialog
		# Simple text dialogs just tell the player something and let them click OK
		if isinstance(diargs, str):
			dia.message_kwargs = {'text': diargs}
			dia.menu_kwargs = {
				'options': [('OK', partial(self.ok, cb2=after_ok))]
			}
		# List dialogs are for when you need the player to make a choice and don't care much
		# about presentation
		elif isinstance(diargs, list):
			dia.message_kwargs = {'text': 'Select from the following:'}
			dia.menu_kwargs = {
				'options':
				list(map(partial(self._munge_menu_option, after_ok), diargs))
			}
		# For real control of the dialog, you need a pair of dicts --
		# the 0th describes the message shown to the player, the 1th
		# describes the menu below
		elif isinstance(diargs, tuple):
			if len(diargs) != 2:
				# TODO more informative error
				raise TypeError('Need a tuple of length 2')
			msgkwargs, mnukwargs = diargs
			if isinstance(msgkwargs, dict):
				dia.message_kwargs = msgkwargs
			elif isinstance(msgkwargs, str):
				dia.message_kwargs['text'] = msgkwargs
			else:
				raise TypeError("Message must be dict or str")
			if isinstance(mnukwargs, dict):
				mnukwargs['options'] = list(
					map(partial(self._munge_menu_option, after_ok),
						mnukwargs['options']))
				dia.menu_kwargs = mnukwargs
			elif isinstance(mnukwargs, (list, tuple)):
				dia.menu_kwargs['options'] = list(
					map(partial(self._munge_menu_option, after_ok), mnukwargs))
			else:
				raise TypeError("Menu must be dict or list")
		else:
			raise TypeError("Don't know how to turn {} into a dialog".format(
				type(diargs)))
		self.add_widget(dia)

	def ok(self, *args, cb=None, cb2=None):
		"""Clear dialog widgets, call ``cb``s if provided, and advance the dialog queue"""
		self.clear_widgets()
		if cb:
			cb()
		if cb2:
			cb2()

	def _lookup_func(self, funcname):
		from importlib import import_module
		if not hasattr(self, '_usermod'):
			self._usermod = import_module(self.usermod, self.userpkg)
		return getattr(self.usermod, funcname)

	def _munge_menu_option(self, after_ok, option):
		if not isinstance(option, tuple):
			raise TypeError
		name, func = option
		if func is None:
			return name, partial(self.ok, cb2=after_ok)
		if callable(func):
			return name, partial(self.ok, cb=func, cb2=after_ok)
		if isinstance(func, tuple):
			fun = func[0]
			if isinstance(fun, str):
				fun = self._lookup_func(fun)
			args = func[1]
			if len(func) == 3:
				kwargs = func[2]
				func = partial(fun, *args, **kwargs)
			else:
				func = partial(fun, *args)
		if isinstance(func, str):
			func = self._lookup_func(func)
		return name, partial(self.ok, cb=func, cb2=after_ok)

Builder.load_string("""
<Box>:
	canvas.before:
		Color:
			rgba: self.background_color
		BorderImage:
			border: self.border
			pos: self.pos
			size: self.size
			source: self.background
		Color:
			rgba: 1, 1, 1, 1
<ScrollableLabel>:
	Label:
		size_hint_y: None
		height: self.texture_size[1]
		text_size: self.width, None
		text: root.text
		color: root.color
<MessageBox>:
	ScrollableLabel:
		x: root.x + root.padding[0]
		y: root.y + root.padding[3]
		width: root.width - root.padding[2]
		height: root.height - root.padding[1]
		text: root.text
		color: root.foreground_color
<Dialog>:
	orientation: 'vertical'
	pos_hint: {'x': 0, 'y': 0}
	size_hint: 1, 0.3
	MessageBox:
		id: msg
	DialogMenu:
		id: menu
""")


def sumar(numeroUno, numeroDos):
    return numeroUno + numeroDos

#Funcion Lambda

sumarAlternativo=lambda numeroUno,numeroDos:numeroUno+numeroDos

#Llamando funciones

resultado=sumar(5,10)

resultadolambda=sumarAlternativo(20, 10)
print (resultado)
print (resultadolambda)

def saludar (nombre):
    return 'hola',nombre

saludarAlternativo=lambda nombre:'hola'+nombre

#Emular las operaciones basicas a eleccion del usuario

lsumar=lambda n1, n2:n1+n2

resta=lambda n1,n2:n1-n2

multiplicar= lambda n1,n2:n1*n2

dividir=lambda n1,n2:n1/n2

opcion=1

while (opcion!=0):
        opcion=int(input("Digita la opcion deseada"))
        if opcion==1:
            lsumar(4,10)
        elif opcion==2:
            resta (15,5)
        elif opcion==3:
            multiplicar (12, 4)
        elif opcion==4:
            dividir(10,2)


#! /usr/bin/env python3

import os
import re

print(type(os.walk('/media/james/BlackToshiba/Cached_Cdm_files')))

simples_dict = dict()
compounds_dict = dict()
alias_smpl_cmpds = dict()

compiled_re = re.compile('[0-9]+.xml')

for root, dirs, files in os.walk('/media/james/BlackToshiba/Cached_Cdm_files'):
    root_pos = [i for i in root.split('/')].index('Cached_Cdm_files')
    if len(root.split('/')) == root_pos + 3:
        compounds_dict[root.split('/')[root_pos+1]] = dirs
        alias = root.split('/')[root_pos + 1]
        print(alias)

    elif len(root.split('/')) == root_pos + 2:
        pointer_xmls = [i.split('.')[0] for i in files if compiled_re.match(i)]
        simples_dict[root.split('/')[root_pos+1]] = pointer_xmls
        alias = root.split('/')[root_pos + 1]
        print(alias)

    else:
        continue

all_collections = set(i for i in simples_dict)
for i in compounds_dict:
    all_collections.add(i)

for alias in all_collections:
    if alias in simples_dict:
        simples_pointers = tuple(i for i in simples_dict[alias])
    else:
        simples_pointers = tuple()
    if alias in compounds_dict:
        compounds_pointers = tuple(i for i in compounds_dict[alias])
    else:
        compounds_pointers = tuple()
    alias_smpl_cmpds[alias] = (simples_pointers, compounds_pointers)

text_output = ''
for i in alias_smpl_cmpds:
    new_line = '{}\nSimples: {}\nCompounds: {}\n\n'.format(i, alias_smpl_cmpds[i][0], alias_smpl_cmpds[i][1])
    text_output = text_output + new_line

print(text_output)

with open('compound_simple_pointers.txt', 'w') as f:
    f.write(text_output)


"Paul Yang M06 Prg Assignment 13.2"
"This program will read the text file today.txt into the string today_string"

read = open('today.txt','r')
texts = read.readlines()
read.close()
print(len(texts), 'text read')

for text in texts:
    print(text, end='')


# https://homcloud.dev/python-api/interface.html#homcloud.interface.draw_volumes_on_2d_image

import numpy as np
import matplotlib.pyplot as plt
import imageio
import os
from scipy.signal import argrelmin, argrelmax
import homcloud.interface as hc
from skimage.filters import threshold_multiotsu

# 画像の読み込み
print("画像を読み込み中です。")
input_image = "r1_500x_2_cut.png"
input_image_path = "data/" + input_image
image_path = os.path.splitext(input_image)[0]
output_path = "output/" + image_path

# mode="L" とするとグレースケールで読み込まれる
pict = imageio.v3.imread(input_image_path, mode="L")

# 閾値の設定
print("閾値を設定中です。")
# from scipy.signal import argrelmin, argrelmax

# データの生成
histo, bins = np.histogram(pict.ravel(), range=(0,256), bins=256)
x=bins[1:]
y=histo

# # 極値の取得
# # orderを変えることで調整可能
# order = 14
# arg_r_min,arg_r_max=argrelmin(y, order=order),argrelmax(y, order=order)

# # 極小値のリストから、適切な値のみ抽出
# arg_r_min_picked = [i for i in arg_r_min[0] if (20 < i) & (i < 180)]

# # 極値を自分で設定する場合
# #arg_r_min_picked = [50, 140]

# 大津の方法
arg_r_min_picked = threshold_multiotsu(pict)

# 抽出した極小値を表示

# figureとaxesを同時生成
fig, ax = plt.subplots()
# 第3引数はcolor cycle
ax.plot(x,y,"C2")
# 第2引数"ro"は、赤い点で表示の意味
ax.plot(x[arg_r_min_picked],y[arg_r_min_picked],"ro",label="argrelmin")
# 凡例
plt.legend()
# 軸ラベルを追加
ax.set(ylabel="Frequency")
# 保存
plt.savefig(output_path+"-arg_rel_minmax.png")
print("極小値をプロットしたヒストグラムを保存しました。")

# PH解析
print("PH解析中です。")
# homcloudのインポート
# import homcloud.interface as hc

# 2値化
pict_tic = pict > arg_r_min_picked[0]
pict_t2_mo2c = (arg_r_min_picked[0] >= pict) | (pict >= arg_r_min_picked[1])
pict_moss = arg_r_min_picked[1] > pict

plt.imshow(pict_tic, "gray")
plt.savefig(output_path+"-binary_tic.png")

plt.imshow(pict_t2_mo2c, "gray")
plt.savefig(output_path+"-binary_t2_mo2c.png")

plt.imshow(pict_moss, "gray")
plt.savefig(output_path+"-binary_moss.png")

# PH解析
hc.PDList.from_bitmap_levelset(hc.distance_transform(pict_tic, signed=True), save_to=output_path+"-pd_tic.pdgm")
hc.PDList.from_bitmap_levelset(hc.distance_transform(pict_t2_mo2c, signed=True), save_to=output_path+"-pd_t2_mo2c.pdgm")
hc.PDList.from_bitmap_levelset(hc.distance_transform(pict_moss, signed=True), save_to=output_path+"-pd_moss.pdgm")

# 0次のPH図を取得
dimension = 1
pd_tic = hc.PDList(output_path+"-pd_tic.pdgm").dth_diagram(dimension)
pd_t2_mo2c = hc.PDList(output_path+"-pd_t2_mo2c.pdgm").dth_diagram(dimension)
pd_moss = hc.PDList(output_path+"-pd_moss.pdgm").dth_diagram(dimension)

# 0次のPH図を表示
pd_tic.histogram().plot(colorbar={"type": "log"})
plt.savefig(output_path+"-pdimage_tic.png")
pd_t2_mo2c.histogram().plot(colorbar={"type": "log"})
plt.savefig(output_path+"-pdimage_t2_mo2c.png")
pd_moss.histogram().plot(colorbar={"type": "log"})
plt.savefig(output_path+"-pdimage_moss.png")
print(str(dimension)+"次のPH図を保存しました。")

# 逆解析

print("逆解析中です。")
hc.BitmapPHTrees.for_bitmap_levelset(hc.distance_transform(pict_tic, signed=True), save_to=output_path+"-tree_tic.pdgm")
hc.BitmapPHTrees.for_bitmap_levelset(hc.distance_transform(pict_t2_mo2c, signed=True), save_to=output_path+"-tree_t2_mo2c.pdgm")
hc.BitmapPHTrees.for_bitmap_levelset(hc.distance_transform(pict_moss, signed=True), save_to=output_path+"-tree_moss.pdgm")

# bitmap_phtrees(0)で0次のPHの情報を取得
phtrees_tic = hc.PDList(output_path+"-tree_tic.pdgm").bitmap_phtrees(dimension)
phtrees_t2_mo2c = hc.PDList(output_path+"-tree_t2_mo2c.pdgm").bitmap_phtrees(dimension)
phtrees_moss = hc.PDList(output_path+"-tree_moss.pdgm").bitmap_phtrees(dimension)

# 対角線から離れた点のみを取得
# deathが-∞のノードに対応する領域は画像全体となるので、このノードは除外
nodes_tic = [node for node in phtrees_tic.nodes if node.lifetime() > 5 and node.death_time() != np.inf]
nodes_t2_mo2c = [node for node in phtrees_t2_mo2c.nodes if node.lifetime() > 5 and node.death_time() != np.inf]
nodes_moss = [node for node in phtrees_moss.nodes if node.lifetime() > 5 and node.death_time() != np.inf]

print("逆解析結果を出力中です。")
# draw_volumes_on_2d_image関数で可視化
# 引数
#  ・ nodes_tic : 取り出したノード
#  ・ input_image : 下地の画像
#  ・ color : 領域を色付けるための色。（R, G, B）で指定。今回は赤。
#  ・ alpha : 領域を色付けるためのアルファ値。透過度のこと。
#  ・ birth_position : birth positionの色
#  ・ marker_size : birth positionのマーカーサイズ
color = (255, 0, 0)
alpha = 0.2
birth_position=(255, 0, 0)
marker_size = 3
hc.draw_volumes_on_2d_image(nodes_tic, input_image_path, color=color, alpha=alpha, birth_position=birth_position, marker_size=marker_size).save(output_path+"-reverse_tic.png")
hc.draw_volumes_on_2d_image(nodes_t2_mo2c, input_image_path, color=color, alpha=alpha, birth_position=birth_position, marker_size=marker_size).save(output_path+"-reverse_t2_mo2c.png")
hc.draw_volumes_on_2d_image(nodes_moss, input_image_path, color=color, alpha=alpha, birth_position=birth_position, marker_size=marker_size).save(output_path+"-reverse_moss.png")
print("逆解析結果の画像を保存しました。")


"""Integration tests for IPFilter with Whitelist ruleset."""

import unittest
import flask
from flask_ipfilter import IPFilter, Whitelist

class TestIPFilterWhiteList(unittest.TestCase):
    """Test integration between IPFilter and WhiteList ruleset."""

    def setUp(self):
        """Set up the test case."""
        self.app = flask.Flask(__name__)
        self.app.add_url_rule('/', 'index', lambda: "OK")
        self.client = self.app.test_client()
        self.ruleset = Whitelist()
        self.ipfilter = IPFilter(self.app, ruleset=self.ruleset)
        self.request_env = {"REMOTE_ADDR": "192.168.0.1"}

    def test_request_denied_by_default(self):
        """All requests should be denied by default."""
        response = self.client.get("/", environ_base=self.request_env)
        self.assertEqual(response.status_code, 403)

    def test_request_allowed_after_permit(self):
        """A request from a permitted IP address should be allowed."""
        self.ruleset.permit("192.168.0.1")
        response = self.client.get("/", environ_base=self.request_env)
        self.assertEqual(response.status_code, 200)


from __future__ import annotations
import math
from typing import Dict, Optional
from returnn.tf.util.basic import get_shape
import tensorflow as tf

class _ReturnnAudioFeatureExtractor:
    def __init__(self):
        self.opts = None  # type: Optional[Params]
        self.tf_session = None  # type: Optional[tf.compat.v1.Session]
        self.tf_audio_placeholder = None
        self.tf_log_mel = None

    def __repr__(self):
        return f"{self.__class__.__name__}{self.opts.items() if self.opts else 'uninitialized'}"

    def setup(self, opts: Params):
        """
        Setup
        """
        if self.opts:
            self.opts.assert_same(opts)
            return
        self.opts = opts

        with tf.compat.v1.Graph().as_default() as g:
            self.tf_audio_placeholder = tf.compat.v1.placeholder(name="audio", dtype=tf.float32, shape=(None,))
            log_mel = extract_log_mel_features_from_audio(self.tf_audio_placeholder, **opts.items())
            log_mel = tf.squeeze(log_mel, axis=0)
            log_mel = tf.squeeze(log_mel, axis=-1)
            self.tf_log_mel = log_mel
            self.tf_session = tf.compat.v1.Session(graph=g)

    def extract(self, *, audio, num_feature_filters, sample_rate, **_other):
        """
        Extract
        """
        assert sample_rate == self.opts.sample_rate
        assert num_feature_filters == self.opts.num_bins
        return self.tf_session.run(self.tf_log_mel, feed_dict={self.tf_audio_placeholder: audio})

_global_returnn_audio_feature_extractor = _ReturnnAudioFeatureExtractor()

def make_returnn_audio_features_func(cached=_global_returnn_audio_feature_extractor, **opts):
    """
    This can be used for ExtractAudioFeatures in RETURNN,
    e.g. in OggZipDataset or LibriSpeechCorpus or others.

    This is cached by default, but you can disable this by setting cached=None.
    """
    if cached:
        extractor = cached
    else:
        extractor = _ReturnnAudioFeatureExtractor()
    p = default_asr_frontend_params()
    p.update(opts)
    extractor.setup(p)
    return extractor.extract

def default_asr_frontend_params(num_bins=80, sample_rate=16000):
    """
    Make params for default ASR frontent
    """
    p = MelAsrFrontend.make_params()
    p.sample_rate = float(sample_rate)
    p.frame_size_ms = 25.0
    p.frame_step_ms = 10.0
    p.num_bins = num_bins
    p.lower_edge_hertz = 125.0
    p.upper_edge_hertz = 7600.0
    p.preemph = 0.97
    p.noise_scale = 0.0
    p.pad_end = False
    return p

def extract_log_mel_features_from_audio(audio, **opts):
    """Create Log-Mel Filterbank Features from audio samples.
    Args:
      audio: Tensor representing audio samples (normalized in [-1,1)).
        It is currently assumed that the wav file is encoded at 16KHz.
        Shape [num_frames] or [num_frames,1].
    Returns:
      A Tensor representing three stacked log-Mel filterbank energies, sub-sampled
      every three frames.
    """
    assert isinstance(audio, tf.Tensor)
    audio *= 32768  # that's what it expects...
    if audio.shape.ndims >= 2:
        # Remove channel dimension, since we have a single channel.
        audio = tf.squeeze(audio, axis=1)
    audio = tf.expand_dims(audio, axis=0)  # [B,num_frames]
    p = default_asr_frontend_params()
    p.update(opts)
    mel_frontend = MelAsrFrontend(**p.items())
    log_mel, _ = mel_frontend.fprop(audio)
    return log_mel

def extract_log_mel_features_from_wav(wav_bytes_t, **opts):
    """Create Log-Mel Filterbank Features from raw bytes.
    Args:
      wav_bytes_t: Tensor representing raw wav file as a string of bytes. It is
        currently assumed that the wav file is encoded at 16KHz (see DecodeWav,
        below).
    Returns:
      A Tensor representing three stacked log-Mel filterbank energies, sub-sampled
      every three frames.
    """
    result = tf.audio.decode_wav(wav_bytes_t)
    sample_rate, audio = result.sample_rate, result.audio
    audio *= 32768
    # Remove channel dimension, since we have a single channel.
    audio = tf.squeeze(audio, axis=1)
    # TODO(drpng): make batches.
    audio = tf.expand_dims(audio, axis=0)
    p = default_asr_frontend_params()
    p.update(opts)
    mel_frontend = MelAsrFrontend(**p.items())
    with tf.control_dependencies([tf.assert_equal(sample_rate, int(p.sample_rate))]):
        log_mel, _ = mel_frontend.fprop(audio)
    return log_mel

def _next_power_of_two(i):
    return math.pow(2, math.ceil(math.log(i, 2)))

class MelAsrFrontend:
    """An AsrFrontend that implements mel feature extraction from PCM frames.
    This is expressed in pure TensorFlow and without reference to external
    resources.
    The frontend implements the following stages:
        `Framer -> Window -> FFT -> FilterBank -> MeanStdDev -> SubSample`
    The FProp input to this layer can either have rank 3 or rank 4 shape:
        [batch_size, timestep, packet_size, channel_count]
        [batch_size, timestep * packet_size, channel_count]
    For compatibility with existing code, 2D [batch_size, timestep] mono shapes
    are also supported.
    In the common case, the packet_size is 1. The 4D variant is accepted for
    glueless interface to input generators that frame their input samples in
    some way. The external framing choice does not influence the operation of
    this instance, but it is accepted.
    TODO(laurenzo): Refactor call sites to uniformly use the 4D variant and
    eliminate fallback logic in this class.
    Only 1 channel is currently supported.
    TODO(laurenzo): Refactor this class to operate on multi-channel inputs.
    """

    @classmethod
    def make_params(cls):
        """
        Make params
        """
        p = Params()
        p.name = "frontend"
        p.define("random_seed", 0)
        p.define("sample_rate", 16000.0, "Sample rate in Hz")
        p.define("channel_count", 1, "Number of channels.")
        p.define("frame_size_ms", 25.0, "Amount of data grabbed for each frame during analysis")
        p.define("frame_step_ms", 10.0, "Number of ms to jump between frames")
        p.define("num_bins", 80, "Number of bins in the mel-spectrogram output")
        p.define("lower_edge_hertz", 125.0, "The lowest frequency of the mel-spectrogram analsis")
        p.define("upper_edge_hertz", 7600.0, "The highest frequency of the mel-spectrogram analsis")
        p.define(
            "preemph",
            0.97,
            "The first-order filter coefficient used for preemphasis. When it " "is 0.0, preemphasis is turned off.",
        )
        p.define("noise_scale", 8.0, "The amount of noise (in 16-bit LSB units) to add")
        p.define("window_fn", "HANNING", 'Window function to apply (valid values are "HANNING", and None)')
        p.define(
            "pad_end",
            False,
            "Whether to pad the end of `signals` with zeros when the provided "
            "frame length and step produces a frame that lies partially past "
            "its end.",
        )
        p.define(
            "per_bin_mean", None, "Per-bin (num_bins) means for normalizing the spectrograms. " "Defaults to zeros."
        )
        p.define("per_bin_stddev", None, "Per-bin (num_bins) standard deviations. Defaults to ones.")
        p.define("stack_left_context", 0, "Number of left context frames to stack.")
        p.define("stack_right_context", 0, "Number of right context frames to stack.")
        p.define("frame_stride", 1, "The frame stride for sub-sampling.")

        p.define("fft_overdrive", True, "Whether to use twice the minimum fft resolution.")
        p.define("output_floor", 1.0, "Minimum output of filterbank output prior to taking logarithm.")
        p.define(
            "compute_energy",
            False,
            "Whether to compute filterbank output on the energy of spectrum " "rather than just the magnitude.",
        )
        p.define("use_divide_stream", False, "Whether use a divide stream to the input signal.")
        return p

    def __init__(self, **kwargs):
        """
        For allowed kwargs, see self._make_params.
        """
        self.params = self.make_params()
        self.params.update(kwargs)
        p = self.params
        if p.frame_stride < 1:
            raise ValueError("frame_stride must be positive.")

        assert p.channel_count == 1, "Only 1 channel currently supported."
        # Make sure key params are in floating point.
        p.sample_rate = float(p.sample_rate)
        p.frame_step_ms = float(p.frame_step_ms)
        p.frame_size_ms = float(p.frame_size_ms)
        p.lower_edge_hertz = float(p.lower_edge_hertz)
        p.upper_edge_hertz = float(p.upper_edge_hertz)

        self._frame_step = int(round(p.sample_rate * p.frame_step_ms / 1000.0))
        self._frame_size = int(round(p.sample_rate * p.frame_size_ms / 1000.0)) + 1  # +1 for the preemph

        self._fft_size = int(max(512, _next_power_of_two(self._frame_size)))
        if p.fft_overdrive:
            self._fft_size *= 2

        self._create_window_function()

        # Mean/stddev.
        if p.per_bin_mean is None:
            p.per_bin_mean = [0.0] * p.num_bins
        if p.per_bin_stddev is None:
            p.per_bin_stddev = [1.0] * p.num_bins
        assert len(p.per_bin_mean) == p.num_bins
        assert len(p.per_bin_stddev) == p.num_bins

    def _create_window_function(self):
        p = self.params
        if p.window_fn is None:
            self._window_fn = None
        elif p.window_fn == "HANNING":

            def _hanning_window(frame_size, dtype):
                return tf.signal.hann_window(frame_size, dtype=dtype)

            self._window_fn = _hanning_window
        else:
            raise ValueError("Illegal value %r for window_fn param" % p.window_fn)

    @property
    def window_frame_size(self):
        """
        Window frame size
        """
        return self._frame_size

    @property
    def window_frame_step(self):
        """
        Window frame step
        """
        return self._frame_step

    @staticmethod
    def _remove_channel_dim(pcm_audio_data):
        if pcm_audio_data.shape.rank == 3:
            pcm_audio_data = tf.squeeze(pcm_audio_data, 2)
            assert pcm_audio_data.shape.rank == 2, "MelAsrFrontend only supports one channel"
        return pcm_audio_data

    @staticmethod
    def _reshape_to_mono2d(pcm_audio_data, paddings):
        """Reshapes a 3D or 4D input to 2D.
        Since the input to FProp can be 3D or 4D (see class comments), this will
        collapse it back to a 2D, mono shape for internal processing.
        Args:
          pcm_audio_data: 2D, 3D or 4D audio input. See class comments. Must have a
            rank.
          paddings: Original paddings shaped to the first two dims of
            pcm_audio_data.
        Returns:
          Tuple of 2D [batch_size, timestep] mono audio data, new paddings.
        """
        shape = get_shape(pcm_audio_data)
        rank = len(shape)
        if rank == 2:
            return pcm_audio_data, paddings
        elif rank == 3:
            # [batch, time, channel]
            with tf.control_dependencies([tf.assert_equal(shape[2], 1)]):
                return tf.squeeze(pcm_audio_data, axis=2), paddings
        elif rank == 4:
            # [batch, time, packet, channel]
            batch_size, orig_time, orig_packet_size, channel = shape
            time = orig_time * orig_packet_size
            with tf.control_dependencies([tf.assert_equal(channel, 1)]):
                pcm_audio_data = tf.reshape(pcm_audio_data, (batch_size, time))
                # Transform paddings into the new time base with a padding per time
                # step vs per packet by duplicating each packet.
                paddings = tf.reshape(
                    tf.tile(tf.expand_dims(paddings, axis=2), [1, 1, orig_packet_size]), (batch_size, time)
                )
                return pcm_audio_data, paddings
        else:
            raise ValueError("Illegal pcm_audio_data shape")

    def fprop(self, audio, paddings=None):
        """Perform signal processing on a sequence of PCM data.
        NOTE: This implementation does not currently support paddings, and they
        are accepted for compatibility with the super-class.
        TODO(laurenzo): Rework this to support paddings.
        Args:
          audio: int16 or float32 tensor of PCM audio data, scaled to
            [-32768..32768] (versus [-1..1)!). See class comments for supported
            input shapes.
          paddings: per frame 0/1 paddings. Shaped: [batch, frame].
        Returns:
          encoder inputs which can be passed directly to a
          compatible encoder and contains:
          - 'src_inputs': inputs to the encoder, minimally of shape
          [batch, time, ...].
          - 'paddings': a 0/1 tensor of shape [batch, time].
        """
        if paddings is None:
            paddings = tf.zeros_like(audio)

        pcm_audio_data, pcm_audio_paddings = self._reshape_to_mono2d(audio, paddings)

        mel_spectrogram, mel_spectrogram_paddings = self._fprop_chunk(pcm_audio_data, pcm_audio_paddings)

        mel_spectrogram, mel_spectrogram_paddings = self._pad_and_reshape_spec(
            mel_spectrogram, mel_spectrogram_paddings
        )

        return mel_spectrogram, mel_spectrogram_paddings

    @staticmethod
    def _stack_signal(signal, stack_size, stride):
        signal = tf.signal.frame(
            signal=signal,
            frame_length=stack_size,
            frame_step=stride,
            pad_end=False,
            axis=1,
        )
        signal = tf.reshape(signal, get_shape(signal)[:2] + [-1])
        return signal

    def _pad_and_reshape_spec(self, mel_spectrogram, mel_spectrogram_paddings):
        p = self.params
        # Stack and sub-sample.
        stack_size = 1
        if p.stack_left_context > 0:
            # Since left context is leading, pad the left by duplicating the first
            # frame.
            stack_size += p.stack_left_context
            mel_spectrogram = tf.concat([mel_spectrogram[:, 0:1, :]] * p.stack_left_context + [mel_spectrogram], axis=1)
            mel_spectrogram_paddings = tf.concat(
                [mel_spectrogram_paddings[:, 0:1]] * p.stack_left_context + [mel_spectrogram_paddings], axis=1
            )

        if p.stack_right_context > 0:
            stack_size += p.stack_right_context
            mel_spectrogram = tf.concat(
                [mel_spectrogram] + [mel_spectrogram[:, -1:, :]] * p.stack_right_context, axis=1
            )
            mel_spectrogram_paddings = tf.concat(
                [mel_spectrogram_paddings] + [mel_spectrogram_paddings[:, -1:]] * p.stack_right_context, axis=1
            )

        if p.stack_left_context or p.stack_right_context:
            mel_spectrogram = self._stack_signal(mel_spectrogram, stack_size, p.frame_stride)
            mel_spectrogram_paddings = self._stack_signal(mel_spectrogram_paddings, stack_size, p.frame_stride)
            # After stacking paddings, pad if any source frame was padded.
            # Stacks into [batch_size, stacked_frame_dim, stack_size] like the
            # spectrogram stacking above, and then reduces the stack_size dim
            # to the max (effectively, making padding = 1.0 if any of the pre-stacked
            # frames were 1.0). Final shape is [batch_size, stacked_frame_dim].
            mel_spectrogram_paddings = tf.reduce_max(mel_spectrogram_paddings, axis=2)

        # Add feature dim. Shape = [batch, time, features, 1]
        mel_spectrogram = tf.expand_dims(mel_spectrogram, -1)
        return mel_spectrogram, mel_spectrogram_paddings

    def _apply_preemphasis(self, framed_signal):
        p = self.params
        preemphasized = framed_signal[:, :, 1:] - p.preemph * framed_signal[:, :, 0:-1]
        return preemphasized

    def _get_mel_padding(self, pcm_audio_paddings):
        p = self.params
        # shape: [batch, time, _frame_size]
        framed_paddings = tf.signal.frame(pcm_audio_paddings, self._frame_size, self._frame_step, p.pad_end)
        # Pad spectrograms that have any padded frames.
        mel_spectrogram_paddings = tf.reduce_max(framed_paddings, axis=2)
        return mel_spectrogram_paddings

    def _fprop_chunk(self, pcm_audio_chunk, pcm_audio_paddings):
        p = self.params
        pcm_audio_chunk = tf.cast(pcm_audio_chunk, tf.float32)
        if p.use_divide_stream:
            pcm_audio_chunk = pcm_audio_chunk / 32768.0

        # shape: [batch, time, _frame_size]
        framed_signal = tf.signal.frame(pcm_audio_chunk, self._frame_size, self._frame_step, p.pad_end)

        # Pre-emphasis.
        if p.preemph != 0.0:
            preemphasized = self._apply_preemphasis(framed_signal)
        else:
            preemphasized = framed_signal[..., :-1]

        # Noise.
        if p.noise_scale > 0.0:
            noise_signal = tf.random.normal(tf.shape(preemphasized), stddev=p.noise_scale, mean=0.0, seed=p.random_seed)
        else:
            noise_signal = 0.0

        # Apply window fn.
        windowed_signal = preemphasized + noise_signal
        if self._window_fn is not None:
            window = self._window_fn(self._frame_size - 1, framed_signal.dtype)
            windowed_signal *= window

        mel_spectrogram = self._mel_spectrogram(windowed_signal)

        mel_spectrogram_log = tf.math.log(tf.maximum(float(p.output_floor), mel_spectrogram))

        # Mean and stddev.
        mel_spectrogram_norm = (mel_spectrogram_log - tf.convert_to_tensor(p.per_bin_mean)) / tf.convert_to_tensor(
            p.per_bin_stddev
        )
        return mel_spectrogram_norm, self._get_mel_padding(pcm_audio_paddings)

    def _mel_spectrogram(self, signal):
        """Computes the mel spectrogram from a waveform signal.
        Args:
          signal: f32 Tensor, shaped [batch_size, num_samples]
        Returns:
          f32 features Tensor, shaped [batch_size, num_frames, mel_channels]
        """
        p = self.params
        # FFT.
        real_frequency_spectrogram = tf.signal.rfft(signal, [self._fft_size])
        magnitude_spectrogram = tf.abs(real_frequency_spectrogram)
        if p.compute_energy:
            magnitude_spectrogram = tf.square(magnitude_spectrogram)

        # Shape of magnitude_spectrogram is num_frames x (fft_size/2+1)
        # Mel_weight is [num_spectrogram_bins, num_mel_bins]
        mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
            num_mel_bins=p.num_bins,
            num_spectrogram_bins=self._fft_size // 2 + 1,
            sample_rate=p.sample_rate,
            lower_edge_hertz=p.lower_edge_hertz,
            upper_edge_hertz=p.upper_edge_hertz,
            dtype=tf.float32,
        )
        # Weight matrix implemented in the magnitude domain.
        batch_size, num_frames, fft_channels = get_shape(magnitude_spectrogram)
        mel_spectrogram = tf.matmul(
            tf.reshape(magnitude_spectrogram, [batch_size * num_frames, fft_channels]), mel_weight_matrix
        )
        mel_spectrogram = tf.reshape(mel_spectrogram, [batch_size, num_frames, p.num_bins])

        return mel_spectrogram

class Params:
    """
    Dummy class to emulate Lingvo `Params` as simple as possible.
    """

    def __init__(self):
        self._params = {"name": None}

    def define(self, key, default_value, _comment=None):
        """
        Define param
        """
        self._params[key] = default_value

    def update(self, d: Dict[str]):
        """
        Update params
        """
        for key, value in d.items():
            assert key in self._params
            self._params[key] = value

    def items(self):
        """
        Param items
        """
        return self._params

    def __repr__(self):
        return "Params{%r}" % (self._params,)

    def __setattr__(self, key, value):
        if key == "_params":
            super(Params, self).__setattr__(key, value)
            return
        assert key in self._params
        self._params[key] = value

    def __getattr__(self, item):
        if item == "_params":
            raise AttributeError("__init__ not yet called")
        if item not in self._params:
            raise AttributeError("Params: key %r unknown" % item)
        return self._params[item]

    def assert_same(self, other: Params):
        """
        Assert same
        """
        assert sorted(self._params.keys()) == sorted(other._params.keys()), f"{self} vs {other}"
        for key in self._params.keys():
            assert self._params[key] == other._params[key], f"param {key}: {self._params[key]} vs {other._params[key]}"


from tkinter import *
import DepartmentCourses
import Home
import tkinter.ttk as ttk
import ES
import sqlite3
from tkinter import messagebox

def dropdown_defocus_CoursesNew(event):
    event.widget.selection_clear()
import ScrollableFrame

class CoursesNew:
    def __init__(self, root):

        self.parent = Frame(root)
        self.parent.grid(row=0, column=0, sticky='nsew')

        self.rightpad = Frame(self.parent, bg="white")
        self.leftpad = Frame(self.parent, bg="white")
        self.bottomholder=Frame(self.parent, bg="white")
        self.frame = Frame(self.parent,bg="white")

        self.rightpad.grid(row=0,column=0,sticky='nsew',rowspan=2)
        self.leftpad.grid(row=0,column=2,sticky='nsew',rowspan=2)
        self.bottomholder.grid(row=1,column=1,columnspan=3,sticky='nsew')
        self.frame.grid(row=0, column=1, sticky='nsew')

        self.parent.rowconfigure(0,weight=1)
        self.parent.columnconfigure(0,weight=1)
        self.parent.columnconfigure(2,weight=1)

        root.geometry('400x400')
        root.minsize(400, 400)
        root.maxsize(400, 400)

        self.coursecodeLabel = Label(self.frame, text='Course Code',bg="white",fg="black")
        self.coursecodeEntry = Entry(self.frame, borderwidth=0)

        self.coursenameLabel = Label(self.frame, text='Course Name',bg="white",fg="black")
        self.coursenameEntry = Entry(self.frame, borderwidth=0)

        self.creditLabel=Label(self.frame, text='Number of credits',bg="white",fg="black")
        self.creditEntry=Entry(self.frame, borderwidth=0)

        self.professornameLabel = Label(self.frame, text='Professor name',bg="white",fg="black")
        self.professornameEntry = Entry(self.frame, borderwidth=0)

        self.submitButton = Button(self.frame, text='Submit', command=lambda: self.submit(root))
        self.exitButton = Button(self.frame, text="Exit", command=exit)
        self.backButton = Button(self.frame, text="Back",
                                                  command=lambda: self.back(root))

        self.coursecodeLabel.grid(row=2, column=0,sticky=E+S,padx=5,pady=3,)
        self.coursecodeEntry.grid(row=2, column=1, sticky=W+S+E)
        self.coursenameLabel.grid(row=3, column=0,sticky=E,padx=5,pady=3)
        self.coursenameEntry.grid(row=3, column=1, sticky=W+E)

        self.professornameLabel.grid(row=4,column=0,sticky=E,padx=5,pady=3)
        self.professornameEntry.grid(row=4,column=1,sticky=W+E)
        self.creditLabel.grid(row=5,column=0,sticky=E,padx=5,pady=3)
        self.creditEntry.grid(row=5,column=1,sticky=W+E)

        self.submitButton.grid(row=7, column=0, columnspan=2,pady=20)

        self.exitButton.grid(row=8, column=0, pady=10, sticky=S+W)
        self.backButton.grid(row=8, column=1, pady=10, sticky=S+E)

        self.frame.rowconfigure(2,weight=1)
        self.frame.rowconfigure(8,weight=1)
        self.frame.rowconfigure(4, weight=0)
        self.frame.columnconfigure(0, weight=0)
        self.frame.columnconfigure(1, weight=1)

        root.mainloop()

    def back(self, root):
        self.clear()
        root.maxsize(800, 600)
        DepartmentCourses.DepartmentCourses(root)

    def submit(self, root):
        coursecode_ = self.coursecodeEntry.get()
        coursename_ = self.coursenameEntry.get()
        professorname_ = self.professornameEntry.get()
        credits_ = self.creditEntry.get()
        try:
            CoursesNew.addcourse(coursecode_, coursename_, professorname_, credits_)
            messagebox.showinfo('Course', coursename_ + ' added Successfully')
            self.clear()
            DepartmentCourses.DepartmentCourses(root)
        except Exception as e:
            messagebox.showwarning('Adding Course', e)

    @staticmethod
    def addcourse(coursecode_, coursename_, professorname_, credits_):
        lengths = [len(i) for i in [coursecode_, coursename_, professorname_, credits_]]
        if 0 in lengths:
            raise Exception('One or more fields left blank')

        connect_, cursor_ = ES.get_student_db_ES()

        cursor_.execute('SELECT * from all_courses WHERE sub_code=(:code)', {'code': coursecode_})
        results = cursor_.fetchall()
        if results:
            raise Exception('A Course with the same Code already exists.')
        try:
            credits_ = int(credits_)
        except Exception:
            raise Exception('Credits is not an integer')
        if credits_ < 0 or credits_ > 5:
            raise Exception('Credits can only be between 1 and 5.')

        with connect_:
            cursor_.execute("INSERT INTO all_courses VALUES (:sub_code, :course_name,:prof_name,:credits)",
                            {'sub_code': coursecode_, 'course_name': coursename_, 'prof_name': professorname_,
                             'credits': credits_})

    def clear(self):
        self.parent.destroy()


import os
import copy
import torch
from torch import nn
from addict import Dict

from .DetModel import DetModel
from addict import Dict as AttrDict

__all__ = ['DistillationModel']

def load_pretrained_params(_model, _path):
    if _path is None:
        return False
    if not os.path.exists(_path):
        print(f'The pretrained_model {_path} does not exists')
        return False
    params = torch.load(_path)
    state_dict = params['state_dict']
    state_dict_no_module = {k.replace('module.', ''): v for k, v in state_dict.items()}
    _model.load_state_dict(state_dict_no_module)
    return _model

class DistillationModel(nn.Module):
    def __init__(self, config):
        super(DistillationModel, self).__init__()
        self.model_dict = nn.ModuleDict()
        self.model_name_list = []

        sub_model_cfgs = config['models']
        for key in sub_model_cfgs:
            sub_cfg = copy.deepcopy(sub_model_cfgs[key])
            sub_cfg.pop('type')
            freeze_params = False
            pretrained = None

            if 'freeze_params' in sub_cfg:
                freeze_params = sub_cfg.pop('freeze_params')
            if 'pretrained' in sub_cfg:
                pretrained = sub_cfg.pop('pretrained')
            model = DetModel(Dict(sub_cfg))
            if pretrained is not None:
                model = load_pretrained_params(model, pretrained)
            if freeze_params:
                for para in model.parameters():
                    para.requires_grad = False
                model.training = False

            self.model_dict[key] = model
            self.model_name_list.append(key)

    def forward(self, x):
        result_dict = dict()
        for idx, model_name in enumerate(self.model_name_list):
            result_dict[model_name] = self.model_dict[model_name](x)
        return result_dict


mylist = ["a","b","c","d","e","f","g","h"]

def printCards(bundle):
    print(bundle)
    a=0
    for i in bundle:
        print(f'{a}\t{i}')
        a=a+1

class Master:
    def __init__(self):
        self.inventory=mylist

    def givecard(self,target):
        a=self.inventory.pop()
        target.inventory.append(a)
        print(f"Popping self.inventory.pop(): {a}")

class Player:
    def __init__(self):
        self.inventory=[]

    # def addCard(self,card):
    #     self.inventory.append(card)

b={}
for i in range(2):
    b[i]=Player()

deck=Master()
print("deck has")
printCards(deck.inventory)

for i in b:
    print(f"Player {i} has {printCards(b[i].inventory)}")
print(b[0].inventory)
deck.givecard(b[0])
print(b[0].inventory)

print("deck has")
printCards(deck.inventory)

for i in b:
    print(f"Player {i} has {printCards(b[i].inventory)}")


from random import randint

RULE = 'What number is missing in the progression?'

PROGRESSION_LENGTH = 10

def get_progression(start, step):
    stop = start + (PROGRESSION_LENGTH * step)
    progression = list(range(start, stop, step))
    return progression

def get_quiz():
    start = randint(1, 100)
    step = randint(1, 10)
    random_index = randint(1, PROGRESSION_LENGTH - 1)
    progression = get_progression(start, step)
    answer = progression[random_index]
    progression[random_index] = '..'
    question = " ".join([str(i) for i in progression])
    return question, str(answer)