import logging
from datetime import datetime, timedelta
from airflow.models import Variable
from airflow import DAG
from airflow.contrib.operators.dataproc_operator import DataProcSparkOperator, DataprocClusterCreateOperator, DataprocClusterDeleteOperator
from airflow.contrib.operators.gcs_to_bq import GoogleCloudStorageToBigQueryOperator
from airflow.operators.bash_operator import BashOperator
from airflow.operators.dummy_operator import DummyOperator
from airflow.utils.dates import days_ago
from airflow.utils.trigger_rule import TriggerRule

log = logging.getLogger(__name__)

log.info("datetime.today(): " + str(datetime.today()))
log.info("datetime.now(): " + str(datetime.now()))

ENVIRONMENT_GCP = Variable.get('RESERVAS-ENVIRONMENT_GCP')
log.info("LOG DE ENVIRONMENT_GCP: " + ENVIRONMENT_GCP)

# ---------------------- PROJECT ------------------------------
PROJECT_ID = Variable.get('RESERVAS-PROJECT_ID')
log.info("LOG DE PROJECT_ID: " + PROJECT_ID)
NAME_BUSINESS_PROJECT = 'VDG_RESERVAS'
NAME_PRODUCT_TECH = 'google_composer'

# ---------------------- DAG ----------------------------------
NAME_DAG_MAIN = 'DAG_' + NAME_BUSINESS_PROJECT

SCHEDULE_INTERVAL = Variable.get('RESERVAS-SCHEDULE_INTERVAL')
log.info("LOG DE SCHEDULE_INTERVAL: " + SCHEDULE_INTERVAL)

# ---------------------- CLUSTER ------------------------------
dataprocClusterOptions = Variable.get("RESERVAS-DATAPROC_CLUSTER_OPTIONS", deserialize_json=True)

CLUSTER_NAME = dataprocClusterOptions['RESERVAS-CLUSTER_NAME']
log.info("LOG DE CLUSTER_NAME: " + CLUSTER_NAME)

SERVICE_ACCOUNT = dataprocClusterOptions['RESERVAS-SERVICE_ACCOUNT']
log.info("LOG DE SERVICE_ACCOUNT: " + SERVICE_ACCOUNT)

REGION_NAME = dataprocClusterOptions['RESERVAS-REGION_NAME']
log.info("LOG DE REGION_NAME: " + REGION_NAME)

ZONE_NAME = dataprocClusterOptions['RESERVAS-ZONE_NAME']
log.info("LOG DE ZONE_NAME: " + ZONE_NAME)

SUBNET_NAME = dataprocClusterOptions['RESERVAS-SUBNET_NAME']
log.info("LOG DE SUBNET_NAME: " + SUBNET_NAME)

STORAGE_BUCKET_CLUSTER = dataprocClusterOptions['RESERVAS-STORAGE_BUCKET_CLUSTER']
log.info("LOG DE STORAGE_BUCKET_CLUSTER: " + STORAGE_BUCKET_CLUSTER)

# ---------------------- NODE MASTER ----------------------------
NUM_MASTERS = dataprocClusterOptions['RESERVAS-NUM_MASTERS']
log.info("LOG DE NUM_MASTERS: " + NUM_MASTERS)

MASTER_MACHINE_TYPE = dataprocClusterOptions['RESERVAS-MASTER_MACHINE_TYPE']
log.info("LOG DE MASTER_MACHINE_TYPE: " + MASTER_MACHINE_TYPE)

MASTER_DISK_SIZE = dataprocClusterOptions['RESERVAS-MASTER_DISK_SIZE']
log.info("LOG DE MASTER_DISK_SIZE: " + MASTER_DISK_SIZE)

# ---------------------- NODES WORKERS ---------------------------
NUM_WORKERS = dataprocClusterOptions['RESERVAS-NUM_WORKERS']
log.info("LOG DE NUM_WORKERS: " + NUM_WORKERS)

WORKER_MACHINE_TYPE = dataprocClusterOptions['RESERVAS-MASTER_MACHINE_TYPE']
log.info("LOG DE WORKER_MACHINE_TYPE: " + WORKER_MACHINE_TYPE)

WORKER_DISK_SIZE = dataprocClusterOptions['RESERVAS-WORKER_DISK_SIZE']
log.info("LOG DE WORKER_DISK_SIZE: " + WORKER_DISK_SIZE)

# ---------------------- SETTINGS -------------------------------
IMAGE_VERSION = dataprocClusterOptions['RESERVAS-IMAGE_VERSION']
log.info("LOG DE IMAGE_VERSION: " + IMAGE_VERSION)

IDLE_DELETE_TTL = dataprocClusterOptions['RESERVAS-IDLE_DELETE_TTL']
log.info("LOG DE IDLE_DELETE_TTL: " + IDLE_DELETE_TTL)

MAX_TO_STRING_FIELDS = Variable.get('RESERVAS-MAX_TO_STRING_FIELDS')
log.info("LOG DE MAX_TO_STRING_FIELDS: " + MAX_TO_STRING_FIELDS)

LOGGING_ENABLE = Variable.get('RESERVAS-LOGGING_ENABLE')
log.info("LOG DE LOGGING_ENABLE: " + LOGGING_ENABLE)

CLUSTER_PROPERTIES = {"spark:spark.debug.maxToStringFields": MAX_TO_STRING_FIELDS,
                      "spark:spark.sql.debug.maxToStringFields": MAX_TO_STRING_FIELDS,
                      "dataproc:dataproc.logging.stackdriver.job.driver.enable": LOGGING_ENABLE,
                      "dataproc:dataproc.logging.stackdriver.enable": LOGGING_ENABLE,
                      "dataproc:jobs.file-backed-output.enable": LOGGING_ENABLE,
                      "dataproc:dataproc.logging.stackdriver.job.yarn.container.enable": LOGGING_ENABLE}

log.info("LOG DE RESERVAS-CLUSTER_PROPERTIES: " + Variable.get("RESERVAS-CLUSTER_PROPERTIES"))
dataprocClusterProperties = Variable.get("RESERVAS-CLUSTER_PROPERTIES", deserialize_json=True)

for key, value in dataprocClusterProperties.items():
    CLUSTER_PROPERTIES[key] = value

properties = "\n"
for key, value in CLUSTER_PROPERTIES.items():
    properties = properties + key + " : " + value + "\n"
log.info("LOG DE CLUSTER_PROPERTIES: " + properties)

# ---------------------- COMPONENTES ----------------------------
BUCKET_DATAPROC_SPARK_JARS = Variable.get('RESERVAS-BUCKET_DATAPROC_SPARK_JARS')
log.info("LOG DE BUCKET_DATAPROC_SPARK_JARS: " + BUCKET_DATAPROC_SPARK_JARS)

# SPARK_SUBMIT_PROPERTIES_AVRO = {'spark.jars.packages': 'org.apache.spark:spark-avro_2.11:2.4.5'}
SPARK_SUBMIT_PROPERTIES = {'spark.jars.packages': 'org.apache.spark:spark-avro_2.11:2.4.5'}

BIG_QUERY_JAR = "gs://spark-lib/bigquery/spark-bigquery-latest.jar"
NAMESPACE_MAIN_CLASS = 'mx.com.gnp.dlk.vida'

# PREFIX_COMPONENT = 'DLK_HDP_VDI_'
PREFIX_COMPONENT = 'GNP_GDP_VDG_'

PLATFORM = 'GCP'
SIS_ORI_INFO = 'INFO'
SIS_ORI_EVO = 'EVO'
SIS_ORI_PLNK = 'PLNK'

COMPONENT_CATALOGOS_GENERALES = PREFIX_COMPONENT + 'CATALOGOS_GENERALES'
COMPONENT_POLIZAS_CERTIFICADO = PREFIX_COMPONENT + 'POLIZAS_CERTIFICADO'
COMPONENT_SINIESTROS = PREFIX_COMPONENT + 'SINIESTROS'

COMPONENT_RPT_SINIESTROS_PERSONAS_MORALES = PREFIX_COMPONENT + 'RPT_SINIESTROS_PERSONAS_MORALES'

# ---------------------- BIG QUERY ----------------------------
bigQueryOptions = Variable.get("RESERVAS-BIG_QUERY_OPTIONS", deserialize_json=True)

SOURCE_BUCKET = bigQueryOptions['RESERVAS-SOURCE_BUCKET']
log.info("LOG DE SOURCE_BUCKET: " + SOURCE_BUCKET)

SOURCE_OBJECTS = bigQueryOptions['RESERVAS-SOURCE_OBJECTS']
log.info("LOG DE SOURCE_OBJECTS: " + SOURCE_OBJECTS)

PROJECT_ID_BQ = bigQueryOptions['RESERVAS-PROJECT_ID_BQ']
log.info("LOG DE PROJECT_ID_BQ: " + PROJECT_ID_BQ)

DATASET_BQ = bigQueryOptions['RESERVAS-DATASET_BQ']
log.info("LOG DE DATASET_BQ: " + DATASET_BQ)

TABLE_NAME_BQ = bigQueryOptions['RESERVAS-TABLE_NAME_BQ']
log.info("LOG DE TABLENAME_BQ: " + TABLE_NAME_BQ)

default_dag_args = {
    'owner': NAME_BUSINESS_PROJECT,
    'start_date': datetime.strptime("2020-10-01 00:00:00", "%Y-%m-%d %H:%M:%S"),
    # 'start_date': days_ago(0),
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=2)
}

with DAG(
        NAME_DAG_MAIN,
        schedule_interval=SCHEDULE_INTERVAL,
        catchup=False,
        default_args=default_dag_args) as DAG_VDG_RESERVAS:

    dpcco_vdg_create_cluster = DataprocClusterCreateOperator(
        task_id='dpcco_vdg_create_cluster',
        cluster_name=CLUSTER_NAME,
        project_id=PROJECT_ID,
        num_masters=NUM_MASTERS,
        master_machine_type=MASTER_MACHINE_TYPE,
        master_disk_size=MASTER_DISK_SIZE,
        num_workers=NUM_WORKERS,
        worker_machine_type=WORKER_MACHINE_TYPE,
        worker_disk_size=WORKER_DISK_SIZE,
        image_version=IMAGE_VERSION,
        idle_delete_ttl=IDLE_DELETE_TTL,
        zone=ZONE_NAME,
        subnetwork_uri=SUBNET_NAME,
        storage_bucket=STORAGE_BUCKET_CLUSTER,
        properties=CLUSTER_PROPERTIES,
        service_account=SERVICE_ACCOUNT,
        # service_account_scopes = ['https://www.googleapis.com/auth/cloud-platform'],
        tags=['dataproc', PROJECT_ID],
        labels={'dataproc_cluster': CLUSTER_NAME,
                'name_project': NAME_BUSINESS_PROJECT.lower(),
                'project_id': PROJECT_ID}
    )

    # dpso_catalogos_generales = DataProcSparkOperator(
    #     task_id = 'dpso_catalogos_generales',
    #     cluster_name = CLUSTER_NAME,
    #     dataproc_spark_properties = SPARK_SUBMIT_PROPERTIES,
    #     dataproc_spark_jars = [BUCKET_DATAPROC_SPARK_JARS + COMPONENT_CATALOGOS_GENERALES + '.jar'],
    #     main_class = NAMESPACE_MAIN_CLASS + '.' + COMPONENT_CATALOGOS_GENERALES,
    #     arguments = [PLATFORM, ENVIRONMENT_GCP, SIS_ORI_INFO]
    # )

    dpso_vdg_polizas_certificado = DataProcSparkOperator(
        task_id = 'dpso_vdg_polizas_certificado',
        cluster_name = CLUSTER_NAME,
        dataproc_spark_properties = SPARK_SUBMIT_PROPERTIES,
        dataproc_spark_jars = [BUCKET_DATAPROC_SPARK_JARS + COMPONENT_POLIZAS_CERTIFICADO + '.jar'],
        main_class = NAMESPACE_MAIN_CLASS + '.' + COMPONENT_POLIZAS_CERTIFICADO,
        arguments = [PLATFORM, ENVIRONMENT_GCP, SIS_ORI_INFO]
    )

    dpso_vdg_siniestros = DataProcSparkOperator(
        task_id = 'dpso_vdg_siniestros',
        cluster_name = CLUSTER_NAME,
        dataproc_spark_properties = SPARK_SUBMIT_PROPERTIES,
        dataproc_spark_jars = [BUCKET_DATAPROC_SPARK_JARS + COMPONENT_SINIESTROS + '.jar'],
        main_class = NAMESPACE_MAIN_CLASS + '.' + COMPONENT_SINIESTROS,
        arguments = [PLATFORM, ENVIRONMENT_GCP, SIS_ORI_INFO]
    )

    dpso_vdg_rpt_siniestros_personas_morales = DataProcSparkOperator(
        task_id = 'dpso_vdg_rpt_siniestros_personas_morales',
        cluster_name = CLUSTER_NAME,
        dataproc_spark_properties = SPARK_SUBMIT_PROPERTIES,
        dataproc_spark_jars = [BUCKET_DATAPROC_SPARK_JARS + COMPONENT_RPT_SINIESTROS_PERSONAS_MORALES + '.jar'],
        main_class = NAMESPACE_MAIN_CLASS + '.' + COMPONENT_RPT_SINIESTROS_PERSONAS_MORALES,
        arguments = [PLATFORM, ENVIRONMENT_GCP, SIS_ORI_INFO]
    )

    bo_borrar_bq_vdg_rpt_siniestros_personas_morales = BashOperator(
        task_id="bo_borrar_bq_vdg_rpt_siniestros_personas_morales",
        bash_command="bq rm -f -t " + PROJECT_ID_BQ + ":" + DATASET_BQ + "." + TABLE_NAME_BQ
    )

    gcs2bqo_crear_bq_vdg_rpt_siniestros_personas_morales = GoogleCloudStorageToBigQueryOperator(
        task_id='gcs2bqo_crear_bq_vdg_rpt_siniestros_personas_morales',
        bucket = SOURCE_BUCKET,
        source_objects = [SOURCE_OBJECTS],
        destination_project_dataset_table = PROJECT_ID_BQ + "." + DATASET_BQ + "." + TABLE_NAME_BQ,
        source_format = 'PARQUET',
        write_disposition = 'WRITE_TRUNCATE'
    )

    dpcdo_vdg_delete_cluster = DataprocClusterDeleteOperator(
        task_id = 'dpcdo_vdg_delete_cluster',
        cluster_name = CLUSTER_NAME,
        project_id = PROJECT_ID,
        depends_on_past = False,
        trigger_rule = TriggerRule.ALL_DONE
    )

    END_DAG = DummyOperator(
        task_id='FIN',
        depends_on_past = False
    )

dpcco_vdg_create_cluster.dag = DAG_VDG_RESERVAS

dpcco_vdg_create_cluster \
    >> [dpso_vdg_polizas_certificado, dpso_vdg_siniestros] \
    >> dpso_vdg_rpt_siniestros_personas_morales \
    >> bo_borrar_bq_vdg_rpt_siniestros_personas_morales >> gcs2bqo_crear_bq_vdg_rpt_siniestros_personas_morales \
    >> dpcdo_vdg_delete_cluster \
    >> END_DAG


import random
import csv
import numpy as np

def read_file(path):
    with open(path, newline='') as file:
        data = []
        labels = []
        for line in file:
            if line.isspace():
               None
            else:
                tokens = line.split(',')
                # print(tokens)
                data.append([float(tokens[0]), float(tokens[1]), float(tokens[2].split('\r')[0])])
                # labels.append()
        # shuffle_tmp = list(zip(data, labels))
        np.random.seed(2)
        np.random.shuffle(data)
        # print(data)
        # data, labels = zip(*shuffle_tmp)
        return data


#Remove duplicates from sorted array
def rm_dups(arr):
    i = 0
    while i < len(arr)-1:
        if arr[i] == arr[i+1]:
            del arr[i]
        else:
            i += 1
    return len(arr)

import unittest
class Test(unittest.TestCase):
    def test_rm_dups(self):
        arr = [1, 3, 3, 4, 5]
        rm_dups(arr)
        self.assertEqual([1, 3, 4 , 5], arr)

if __name__ == '__main__':
    unittest.main()


import sqlite3
import numpy as np
import matplotlib.pyplot as plt

def displayData(passedDiction):
    """used to display graphs I have it set to display top 10 results based on frequency"""
    NamesXAxis = []
    FrequencyYAxis = []

    for i in range(0,10):#Where top ten results are gotten increase 10 to increase results
        curMax = max(passedDiction, key = passedDiction.get)
        NamesXAxis.append(curMax)
        FrequencyYAxis.append(passedDiction[curMax])
        passedDiction.pop(curMax)

    y_pos = np.arange(len(FrequencyYAxis))
    plt.bar(y_pos, FrequencyYAxis,width = 0.8)
    plt.xticks(y_pos, NamesXAxis,rotation = 80)
    plt.tight_layout()
    for index, value in enumerate(FrequencyYAxis):
        plt.text(index, value, str(value))
    plt.show()

def parse():
    """Used to see how many request each internal page makes"""

    conn = sqlite3.connect("crawl-data.sqlite")
    cur = conn.cursor()
    cur.execute("SELECT extension_session_uuid,url FROM http_requests")
    rows = cur.fetchall()
    infodict = {}
    links = open("TextFiles/revisedList.txt", 'r')
    curPage = links.readline()
    curPage = curPage.strip('\n')
    count = 1
    sumOfVisits = 0
    curId = ""
    curSum = 0

    #Have a count that goes to 5 since we only visited 5 internal pages
    for row in rows:
        #Once count reaches 5 we assume we are done with a specific websites internal pages
        #And add it to the list and reset vars
        if count == 5:
            #Dividing result by 5 to find average amount of third party requests
            infodict[curPage] = sumOfVisits/5
            curPage = links.readline()
            sumOfVisits = 0
            count = 0
        #Else Just continue counting number of third party requests.
        if curId != row[0]:
            curId = row[0]
            count += 1
            sumOfVisits += curSum
            curSum = 0
        else:
            curSum += 1

    displayData(infodict)

def httpWebsites():
    """Gotten from my Project 2
    Used to see which type of third parties we are requesting from
    No changes made"""

    conn = sqlite3.connect("crawl-data.sqlite")
    cur = conn.cursor()
    cur.execute("SELECT extension_session_uuid,url FROM http_requests")
    rows = cur.fetchall()
    infodict = {}
    curID = ""
    curDomain = ""

    for row in rows:
        websitetemp = row[1].split("/")
        website = websitetemp[2]
        if(curID != row[0]):
            curID = row[0]
            curDomain = website
        elif(curDomain in website):
            continue
        else:
            if(website in infodict):
                infodict[website] = infodict[website] + 1
            else:
                infodict[website] = 1
    displayData(infodict)

def CompareToHisList():
    """Compare my results to the HisPar List and sees how many of my internal pages
    are in HisPar List"""

    myList = open("TextFiles/AllSubPages.txt",'r')
    hisList = open("TextFiles/hisList.txt", 'r')

    #HisPar List opened and put into list for easier lookup
    hisListList = []
    for line in hisList:
        line = line.strip('\n')
        hisListList.append(line)

    similar = 0
    different = 0
    #Look for websites from my list in HisPar List
    for line in myList:
        line = line.strip('\n')
        if line in hisListList:
            similar += 1
        else:
            different += 1

    print(str(similar) + " InternalPages found in common")
    print(str(different) + " InternalPages unique to my lookup")

def reviseMyList():
    """Used to take out websites I was unable to visit
    From the list of top 100 Websites to make parsing earsier for me"""

    f = open("100MainPages.txt",'r')
    r = open("TextFiles/NoSubPageGet.txt", 'r')
    p = open("TextFiles/revisedList.txt", 'w')

    badURl = []
    for line in r:
        revisedLine = revisedLine.strip('\n')
        badURl.append(revisedLine)

    for line in f:
        line = line.strip('\n')
        if line in badURl:
            continue
        else:
            p.write(line)
            p.write('\n')

reviseMyList()
CompareToHisList()
parse()
httpWebsites()


# 445. Add Two Numbers II
# Linked List

# https://blog.csdn.net/fuxuemingzhu/article/details/79380457

# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next

# runtime: faster than 99.54%
class Solution1:
    def addTwoNumbers(self, l1: ListNode, l2: ListNode) -> ListNode:
        num1, num2 = '', ''
        while l1:
            num1 += str(l1.val)
            l1 = l1.next
        while l2:
            num2 += str(l2.val)
            l2 = l2.next

        _sum = str(int(num1) + int(num2))
        head = ListNode(_sum[0])
        ans = head
        for i in range(1, len(_sum)):
            node = ListNode(_sum[i])
            head.next = node
            head = head.next

        return ans

# Reverse
# runtime: O(n1+n2)
class Solution:
    def addTwoNumbers(self, l1: ListNode, l2: ListNode) -> ListNode:
        l1 = self.reverseList(l1)
        l2 = self.reverseList(l2)

        head = None
        carry = 0
        while l1 or l2:
            x1 = l1.val if l1 else 0
            x2 = l2.val if l2 else 0
            carry, val = divmod(x1+x2+carry, 10)
            curr = ListNode(val)
            curr.next = head
            head = curr

            l1 = l1.next if l1 else None
            l2 = l2.next if l2 else None

        if carry:
            curr = ListNode(carry)
            curr.next = head
            head = curr
        return head

    def reverseList(self, head):
        prev = None
        while head:
            tmp = head.next
            head.next = prev
            prev = head
            head = tmp
        return prev

# Stack
# runtime: faster than 85.24%
class Solution2:
    def addTwoNumbers(self, l1: ListNode, l2: ListNode) -> ListNode:
        stack1, stack2 = [], []
        while l1:
            stack1.append(l1.val)
            l1 = l1.next
        while l2:
            stack2.append(l2.val)
            l2 = l2.next

        ans = None
        carry = 0

        while stack1 and stack2:
            add = stack1.pop() + stack2.pop() + carry
            carry = 1 if add >= 10 else 0
            temp = ans
            ans = ListNode(add % 10)
            ans.next = temp

        l = stack1 if stack1 else stack2
        while l:
            add = l.pop() + carry
            carry = 1 if add >= 10 else 0
            temp = ans
            ans = ListNode(add % 10)
            ans.next = temp

        if carry:
            temp = ans
            ans = ListNode(1)
            ans.next = temp

        return ans


import os

import networkx as nx
import numpy as np
import scipy.sparse as sp
import tensorflow as tf

import tensorflow_datasets as tfds
from graph_tfds.core.features.sparse import SparseComponents, SparseTensor
from graph_tfds.core.utils.dgl.core import DGL_URL
from graph_tfds.core.utils.file_io import load_pickle, loadtxt

def load_citation_graph(name, data_dir):
    def full_path(attr_name):
        return os.path.join(data_dir, f"ind.{name}.{attr_name}")

    x, y, tx, ty, allx, ally, graph = (
        load_pickle(full_path(n), encoding="latin1")
        for n in ("x", "y", "tx", "ty", "allx", "ally", "graph")
    )

    test_idx_reorder = loadtxt(full_path("test.index"), dtype=np.int64)
    test_idx_range = np.sort(test_idx_reorder)
    a = nx.adjacency_matrix(nx.from_dict_of_lists(graph))  # CSR
    a.setdiag(0)
    a.eliminate_zeros()
    a = a.tocoo()
    links = np.stack((a.row, a.col), axis=1).astype(np.int64)
    if name == "citeseer":
        # Fix citeseer dataset (there are some isolated nodes in the graph)
        # Find isolated nodes, add them as zero-vecs into the right position
        test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder) + 1)
        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
        tx_extended[test_idx_range - min(test_idx_range), :] = tx
        tx = tx_extended
        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
        ty_extended[test_idx_range - min(test_idx_range), :] = ty
        ty = ty_extended

    node_features = sp.vstack((allx, tx)).tolil()
    node_features[test_idx_reorder, :] = node_features[test_idx_range, :]

    onehot_labels = np.vstack((ally, ty))
    onehot_labels[test_idx_reorder, :] = onehot_labels[test_idx_range, :]
    node_labels = np.argmax(onehot_labels, 1)

    test_ids = test_idx_range
    train_ids = np.arange(len(y), dtype=np.int64)
    validation_ids = np.arange(len(y), len(y) + 500, dtype=np.int64)
    node_features = node_features.tocoo()

    return {
        "graph": {
            "links": links,
            "node_features": SparseComponents(
                indices=np.stack(
                    (node_features.row, node_features.col), axis=-1
                ).astype(np.int64),
                values=node_features.data.astype(np.float32),
                dense_shape=node_features.shape,
            ),
        },
        "node_labels": node_labels,
        "train_ids": train_ids,
        "validation_ids": validation_ids,
        "test_ids": test_ids,
    }

_CITATION = """\
@article{sen2008collective,
  title={Collective classification in network data},
  author={Sen, Prithviraj and Namata, Galileo and Bilgic, Mustafa and Getoor, Lise and Galligher, Brian and Eliassi-Rad, Tina},
  journal={AI magazine},
  volume={29},
  number={3},
  pages={93--93},
  year={2008}
}
"""

_DESCRIPTION = "{} citation network, with preprocessing as per DGL."

class CitationGraph(tfds.core.GeneratorBasedBuilder):
    """DatasetBuilder for CitationGraph dataset."""

    VERSION = tfds.core.Version("0.0.1")
    RELEASE_NOTES = {
        "0.0.1": "Initial release.",
    }

    def _info(self) -> tfds.core.DatasetInfo:
        """Returns the dataset metadata."""
        return tfds.core.DatasetInfo(
            builder=self,
            description=_DESCRIPTION.format(self._NAME),
            features=tfds.features.FeaturesDict(
                {
                    "graph": {
                        "links": tfds.core.features.Tensor(
                            shape=(None, 2), dtype=tf.int64
                        ),
                        "node_features": SparseTensor(ndims=2, dtype=tf.float32),
                    },
                    "node_labels": tfds.core.features.Tensor(
                        shape=(None,), dtype=tf.int64,
                    ),
                    "train_ids": tfds.core.features.Tensor(
                        shape=(None,), dtype=tf.int64
                    ),
                    "validation_ids": tfds.core.features.Tensor(
                        shape=(None,), dtype=tf.int64
                    ),
                    "test_ids": tfds.core.features.Tensor(
                        shape=(None,), dtype=tf.int64
                    ),
                }
            ),
            supervised_keys=("graph", "node_labels"),
            homepage="http://www.cs.umd.edu/~sen/lbc-proj/LBC.html",
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager: tfds.download.DownloadManager):
        """Returns SplitGenerators."""
        data_dir = dl_manager.download_and_extract(f"{DGL_URL}{self._URL_NAME}.zip")

        return {
            "full": self._generate_examples(data_dir),
        }

    def _generate_examples(self, data_dir: str):
        """Yields examples."""
        data = load_citation_graph(self._URL_NAME, data_dir)
        yield 0, data

def _vis(builder):
    # pylint: disable=import-outside-toplevel
    import matplotlib.pyplot as plt

    # pylint: enable=import-outside-toplevel

    name = builder.name
    builder.download_and_prepare(
        download_config=tfds.core.download.DownloadConfig(register_checksums=True)
    )
    print(f"Prepared {name}")

    inputs, labels = tf.data.experimental.get_single_element(
        builder.as_dataset(as_supervised=True, split="full")
    )
    row_ids, col_ids = inputs["links"].numpy().T
    labels = labels.numpy()
    G = nx.Graph()
    for i in range(len(labels)):
        G.add_node(i)
    for ri, ci in zip(row_ids, col_ids):
        G.add_edge(ri, ci)
    print(f"{name} has {len(labels)} nodes and {len(row_ids)} edges")
    pos = nx.spring_layout(G)
    nx.draw_networkx_nodes(G, pos, alpha=0.4, node_size=2)
    nx.draw_networkx_edges(G, pos, width=1.0, alpha=0.1)
    plt.show()


from tkinter import *
import requests
import tkinter.font as tkFont
from bs4 import BeautifulSoup
class Main(object):
    def __init__(self):
        self.Setup()
        self.Widgets()
        self.Loop()
    def Setup(self):
        self.Root=Tk()
        self.Root.title("感測器控制平台")
        #self.Root.geometry("250x250")
        self.tempVar=StringVar()
        self.tempVar.set("offline")
        self.humiVar=StringVar()
        self.humiVar.set("offline")
        self.distVar=StringVar()
        self.distVar.set("offline")
        self.Font=tkFont.Font(family="helvetica",size=20)
        self.rVar=IntVar()
        self.rVar.set(0)
        self.gVar=IntVar()
        self.gVar.set(0)
        self.bVar=IntVar()
        self.bVar.set(0)
        self.baseurl="192..168.4.1"
    def Widgets(self):
        self.Labels=[]
        self.Buttons=[]
        self.Scales=[]
        self.Labels.append(Label(self.Root,text="接收資料",font=self.Font,fg="red",relief="solid"))
        self.Labels[-1].grid(row=0,column=0,columnspan=3,sticky=W+E)
        self.Labels.append(Label(self.Root,text="  溫度  ",font=self.Font,relief="solid"))
        self.Labels[-1].grid(row=1,column=0,columnspan=1,sticky=W+E)
        self.Labels.append(Label(self.Root,text="  濕度  ",font=self.Font,relief="solid"))
        self.Labels[-1].grid(row=1,column=1,columnspan=1,sticky=W+E)
        self.Labels.append(Label(self.Root,text="  距離  ",font=self.Font,relief="solid"))
        self.Labels[-1].grid(row=1,column=2,columnspan=1,sticky=W+E)
        self.Labels.append(Label(self.Root,textvariable=self.tempVar,font=self.Font,relief="solid"))
        self.Labels[-1].grid(row=2,column=0,columnspan=1,sticky=W+E)
        self.Labels.append(Label(self.Root,textvariable=self.humiVar,font=self.Font,relief="solid"))
        self.Labels[-1].grid(row=2,column=1,columnspan=1,sticky=W+E)
        self.Labels.append(Label(self.Root,textvariable=self.distVar,font=self.Font,relief="solid"))
        self.Labels[-1].grid(row=2,column=2,columnspan=1,sticky=W+E)
        self.Labels.append(Label(self.Root,text="開關控制",font=self.Font,fg="green",relief="solid"))
        self.Labels[-1].grid(row=3,column=0,columnspan=3,sticky=W+E)
        self.ButtonFrame=Frame(self.Root)
        self.ButtonFrame.grid(row=4,column=0,columnspan=3,sticky=W+E)
        self.Buttons.append(Button(self.ButtonFrame,text="電源ON",command=self.onmain,bg="green",font=self.Font,relief="solid"))
        self.Buttons[-1].grid(row=0,column=0,columnspan=1,sticky=W+E)
        self.Buttons.append(Button(self.ButtonFrame,text="蜂鳴器ON",command=self.onbuzz,bg="green",font=self.Font,relief="solid"))
        self.Buttons[-1].grid(row=0,column=1,columnspan=1,sticky=W+E)
        self.Buttons.append(Button(self.ButtonFrame,text="電源OFF",command=self.offmain,bg="red",font=self.Font,relief="solid"))
        self.Buttons[-1].grid(row=1,column=0,columnspan=1,sticky=W+E)
        self.Buttons.append(Button(self.ButtonFrame,text="蜂鳴器OFF",command=self.offbuzz,bg="red",font=self.Font,relief="solid"))
        self.Buttons[-1].grid(row=1,column=1,columnspan=1,sticky=W+E)
        self.Labels.append(Label(self.Root,text="RGB控制",font=self.Font,fg="blue",relief="solid"))
        self.Labels[-1].grid(row=5,column=0,columnspan=3,sticky=W+E)
        self.Scales.append(Scale(self.Root,variable=self.rVar,command=lambda x : self.updatergb(1,self.rVar.get()),from_=0,to=255,orient=HORIZONTAL,relief="solid",label="紅："))
        self.Scales[-1].grid(row=6,column=0,columnspan=3,sticky=W+E)
        self.Scales.append(Scale(self.Root,variable=self.gVar,command=lambda x : self.updatergb(2,self.gVar.get()),from_=0,to=255,orient=HORIZONTAL,relief="solid",label="綠："))
        self.Scales[-1].grid(row=7,column=0,columnspan=3,sticky=W+E)
        self.Scales.append(Scale(self.Root,variable=self.bVar,command=lambda x : self.updatergb(3,self.bVar.get()),from_=0,to=255,orient=HORIZONTAL,relief="solid",label="藍："))
        self.Scales[-1].grid(row=8,column=0,columnspan=3,sticky=W+E)
        self.Root.resizable(0,0)
    def Loop(self):
        self.Root.after(200,self.updateinfo)
        self.Root.mainloop()
    def updateinfo(self):
        print("updateinfo")
        respond=requests.get("http://192.168.4.1/value.html")
        bs4=BeautifulSoup(respond.text,"lxml")

        self.tempVar.set(bst.find)
        self.Root.after(200,self.updateinfo)
    def onmain(self):
        print("onmain")
        try:
            requests.get("http://192.168.4.1/r1on.html")
        except(requests.exceptions.ConnectionError):
            pass
    def offmain(self):
        print("offmain")
        try:
            requests.get("http://192.168.4.1/r1off.html")
        except(requests.exceptions.ConnectionError):
            pass
    def onbuzz(self):
        print("onbuzz")
        try:
            requests.get("http://192.168.4.1/r2on.html")
        except(requests.exceptions.ConnectionError):
            pass
    def offbuzz(self):
        print("offbuzz")
        try:
            requests.get("http://192.168.4.1/r2off.html")
        except(requests.exceptions.ConnectionError):
            pass
    def updatergb(self,pin,value):
        try:
            requests.get("http://192.168.4.1/color{}={}".format(pin,value))
        except(requests.exceptions.ConnectionError):
            pass
if __name__=="__main__":
    Main()


import threading

class BoundedBuffer:

    def __init__(self, size):
        self.buffer = [None] * size
        self.size = size
        self.count = 0
        self.in_pos = 0
        self.out_pos = 0

        # Synchronization primitives
        self.mutex = threading.Lock()
        self.not_full = threading.Condition(self.mutex)
        self.not_empty = threading.Condition(self.mutex)

    def add(self, item):
        with self.not_full:
            while self.count == self.size:
                self.not_full.wait()

            self.buffer[self.in_pos] = item
            self.in_pos = (self.in_pos + 1) % self.size
            self.count += 1

            self.not_empty.notify()

    def remove(self):
        with self.not_empty:
            while self.count == 0:
                self.not_empty.wait()

            item = self.buffer[self.out_pos]
            self.out_pos = (self.out_pos + 1) % self.size
            self.count -= 1

            self.not_full.notify()
            return item

    def __str__(self):
        rStr = "[ "
        for i in range(self.out_pos, self.in_pos):
            rStr += str(self.buffer[i]) + " "
        rStr += "]"
        return rStr


# -*- coding: utf-8 -*-

from game import fields

class Bear:
    """ powerful but narrow-minded monster """
    def __init__(self, game):
        self.game = game
        self.location = None

    def attack(self, obj):
        obj.get_damage(2)
        obj.get_pushed(self)

    def update(self):
        pass

    def go(self):
        """ bear movement logic """
        field = self.game.fields.at((self.location.coordinates[0] - self.game.player.vector[0],
                                    self.location.coordinates[1] - self.game.player.vector[1]))
        if not field or isinstance(field, fields.Wall):
            return
        elif isinstance(field, fields.Grass) and field.obj in ("tp1", "tp2", "tp3"):
            self.move(field.pair)
            return
        elif isinstance(field, fields.Grass) and field.obj == "mine":
            field.delete_object()
        if self.game.player.location == field:
            self.attack(self.game.player)
            return
        self.move(field)
        # flow down (and left) the river
        if isinstance(field, fields.Water):
            washed_away = False
            while not washed_away:
                next_water_down = self.game.fields.at((field.coordinates[0] + 1, field.coordinates[1]))
                next_water_left = self.game.fields.at((field.coordinates[0], field.coordinates[1] - 1))
                if isinstance(next_water_down, fields.Water):
                    self.move(next_water_down)
                    field = next_water_down
                elif isinstance(next_water_left, fields.Water):
                    self.move(next_water_left)
                    field = next_water_left
                else:
                    washed_away = True

    def move(self, field):
        self.location = field


# coding=utf-8

__title__ = 'test'
__version__ = '0.0.0'
__build__ = 0x000000
__author__ = 'Sam Broderick'
__license__ = 'Apache 2.0 '
__copyright__ = 'Copyright 2016 Sam Broderick'
__docformat__ = 'restructuredtext'

C = [-6, -6, -4, -4, 2, 2, 2]
C1 = [2, 2, -6, -6]
C2 = [-4, -4, 2]

def minkowskiDist(v1, v2, p):
    # type: (array, array, int) -> float
    """
    Assumes vl and v2 are equal -length arrays of numbers.
    Returns Minkowski distance of order p between vl and v2.

    :param v1: array of numbers (list).
    :type v1: array
    :param v2: array of numbers (list), equal length to v1.
    :type v2: array
    :param p: distance order
    :type p: int
    :return: Minkowski distance (p = 2 is the Euclidian distance)
    :rtype: float
    """
    dist = 0.0
    for i in range(len(v1)):
        dist += abs(v1[i] - v2[i]) ** p
    return dist**(1.0/p)

def variance(cluster, mean_pos):
    # type: (list, array) -> float
    total_dist = 0.0
    for c in cluster:
        c_distance = minkowskiDist([mean_pos], [c], 2)
        total_dist += (c_distance)**2
    return total_dist

C1_mean = sum(C1)/float(len(C1))
C2_mean = sum(C2)/float(len(C2))
print('Means for C1 = {}, for C2 = {}'.format(C1_mean, C2_mean))
V1 = variance(C1, C1_mean)
print('Variance C1 = {}'.format(V1))
V2 = variance(C2, C2_mean)
print('Variance C2 = {}'.format(V2))
print('Badness = {}'.format(V1 + V2))

C1 = [2, 2, 2]
C2 = [-6, -6, -4, -4]

C1_mean = sum(C1)/float(len(C1))
C2_mean = sum(C2)/float(len(C2))
print('Means for C1 = {}, for C2 = {}'.format(C1_mean, C2_mean))
V1 = variance(C1, C1_mean)
print('Variance C1 = {}'.format(V1))
V2 = variance(C2, C2_mean)
print('Variance C2 = {}'.format(V2))
print('Badness = {}'.format(V1 + V2))


import os
import pandas as pd
import numpy as np
from pathlib import Path
from utils.get_celeba_info import get_celeba_index
from typing import Tuple
import config

class GlowProvider(object):

    def __init__(self, index, img_path: Path = None, np_path: Path = None):
        # self.index: pd.DataFrame = pd.read_csv(str(index), sep=' ')
        aux = get_celeba_index(index)
        aux.set_index('img_id', inplace=True, drop=False)
        self.index = aux
        self.img_path = img_path
        self.np_path = np_path

    def filter_index(self, features: dict) -> pd.DataFrame:
        df = self.index.copy()
        for key, val in features.items():
            df = df[df[key] == val]
        return df

    def get_np(self, features: dict, sample=0.1, sample_max=np.Inf) -> np.ndarray:
        nps = list()
        nps_generator = self.get_np_yield(features=features, sample=sample, sample_max=sample_max)

        for n in nps_generator:
            nps.append(n)

        return np.concatenate(nps)

    def get_np_yield(self, features: dict, sample=0.1, sample_max=np.Inf) -> np.ndarray:
        df = self.filter_index(features)
        max_elements = min(sample * len(df) // 1, sample_max)
        cur_elements = 0
        indexes = df.index.values
        new_indexes = np.random.permutation(indexes)

        for i in new_indexes:
            file_name = df.loc[i, 'img_id']
            try:
                result = np.load(str((self.np_path / (file_name[:-4] + '.npy'))))
                cur_elements += 1
                if cur_elements % 1000 == 0:
                    print(cur_elements)
                yield result
                if cur_elements >= max_elements:
                    break
            except:
                pass

    def get_features(self) -> list:
        result = self.index.columns.values.tolist()

        try:
            result.remove('img_id')
        except:
            pass
        try:
            result.remove('person_id')
        except:
            pass

        return result

    def get_np_with_features(self, np_name: str) -> Tuple[np.ndarray, dict]:
        indexes = self.index

        result = np.load(str((self.np_path / (np_name[:-4] + '.npy'))))
        return result.copy(), self.index.loc[np_name].to_dict()

    def get_image_names(self):
        image_names = []
        for file_name in self.index.index.tolist():
            if (self.np_path / (file_name[:-4] + '.npy')).exists():
                image_names.append(file_name)

        return image_names

    def get_np_num(self):
        list = os.listdir(self.np_path)
        return len(list)

if __name__ == '__main__':
    fp = GlowProvider(index=config.index_path / 'list_attr_celeba_clean.txt',
                      np_path=config.data_path / 'np')

    images = fp.get_np({'Blond_Hair': 1})

    print('hola')


from aocd.models import Puzzle

DAY = 4
YEAR = 2022

def main():
    puzzle = Puzzle(year=YEAR, day=DAY)
    puzzle.answer_a = solve_a(puzzle.input_data)
    puzzle.answer_b = solve_b(puzzle.input_data)

def read_sections(puzzle_input: str) -> list[tuple[tuple[str, str], ...]]:
    """
    Split "2-4,6-8\n2-3,4-5" into [((2, 4), (6, 8)), ((2, 3), (4, 5))]
    """
    for line in puzzle_input.splitlines():
        yield tuple(tuple(map(int, section.split("-"))) for section in line.split(","))

def section_to_set(section: tuple[int, int]) -> set[int]:
    return set(range(section[0], section[1] + 1))

def solve_a(puzzle_input):
    # how often is one section fully contained in another?
    fully_contained_sections = 0
    for section_a, section_b in read_sections(puzzle_input):
        set_a = section_to_set(section_a)
        set_b = section_to_set(section_b)
        if set_a.issubset(set_b) or set_b.issubset(set_a):
            fully_contained_sections += 1
    return fully_contained_sections

def solve_b(puzzle_input):
    # how often do the sections overlap?
    overlapping_sections = 0
    for section_a, section_b in read_sections(puzzle_input):
        set_a = section_to_set(section_a)
        set_b = section_to_set(section_b)
        if set_a.intersection(set_b):
            overlapping_sections += 1
    return overlapping_sections

if __name__ == "__main__":
    main()


from modules import Image
from modules import numpy as np

# Encoding function
def Encode(source, message, destination, stg):

    # open image in read mode
    img = Image.open(source, 'r')

    # get image dimensions
    width, height = img.size

    # convert img data to numpy array - collection of channels of pixels (decimals)
    image = np.array(list(img.getdata()))

    # number of image channels (4 for PNG - RGBA)
    channel = 4

    # number of pixels = image size / channel
    total_pixels = image.size//channel

    # extend message with delimiter
    message += "$t3g0"

    # convert message to binary messsage (decimal to binary)
    binary_message = ''.join([format(ord(i), "08b") for i in message])

    # number of pixels to be altered
    req_components = len(binary_message)

    # check if required pixels are present in image
    # if not, raise error
    # else proceed encoding
    if req_components > total_pixels:
        stg.GreaterSizeRequired()
    else:
        # reference to message bit to be encoded
        message_index = 0
        # for every pixel in image
        for pixel in range(total_pixels):
            # for every component of pixel
            for colour in range(0, 3):
                # if message still left to be encoded,
                # then encode,
                # else break
                if message_index < req_components:
                    # change the LSB-bit of the pixel's component:
                    # grab the binary form of the component value(R/G/B),
                    # remove the LSB
                    # append the message bit
                    # convert back to int
                    # save the new value calculated
                    image[pixel][colour] = int(
                        bin(image[pixel][colour])[:-1] + binary_message[message_index], 2)
                    message_index += 1

        # reshape matrix
        image = image.reshape(height, width, channel)

        # covert the numpy img data to image file (.PNG)
        enc_img = Image.fromarray(image.astype('uint8'), img.mode)

        # save the image at the destination folder
        enc_img.save(destination)

        # open the success dialog
        stg.OpenSuccessDialog()

# Decoding function
def Decode(src, stg):

    # clear previous decoded message on window
    stg.text_.delete(1.0, "end")

    # open image in read mode
    img = Image.open(src, 'r')

    # convert img data to numpy array - collection of channels of pixels (decimals)
    image = np.array(list(img.getdata()))

    # number of image channels (4 for PNG - RGBA)
    channel = 4

    # number of pixels = image size / channel
    total_pixels = image.size//channel

    # bits to decoded from the encoded image
    hidden_bits = ""

    # for every pixel in image
    for pixel in range(total_pixels):
        # for every component of pixel
        for colour in range(0, 3):
            # grab the LSB of every component of pixel
            hidden_bits += (bin(image[pixel][colour])[-1])

    # split the bits into groups of 8 bits to form it into ASCII character later
    hidden_bytes = [hidden_bits[i:i+8] for i in range(0, len(hidden_bits), 8)]

    # decoded message
    message = ""

    # for every byte in hidden bytes
    for i in range(len(hidden_bytes)):
        # if delimiter is found then break
        if message[-5:] == "$t3g0":
            break
        # convert binary to int to character using ASCII values
        # and append in message
        message += chr(int(hidden_bytes[i], 2))

    # if image was encoded (at least delimiter is present)
    if "$t3g0" in message:
        # remove the delimiter
        stg.decoded_message = message[:-5]
        # clear decoded message window
        stg.text_.delete(1.0, "end")
        # display decoded message
        stg.text_.insert("end", stg.decoded_message)
    # if image was not encoded, delimiter will be absent
    else:
        # clear decoded message window
        stg.text_.delete(1.0, "end")
        # open no message found window
        stg.NoHiddenMessageDialog()


#https://leetcode.com/problems/number-of-ways-to-wear-different-hats-to-each-other/
#There are n people and 40 types of hats labeled from 1 to 40.
#Given a 2D integer array hats, where hats[i] is a list of all hats preferred by the ith person.
#Return the number of ways that the n people wear different hats to each other.
#Since the answer may be too large, return it modulo 10^9 + 7.

class Solution(object):
    def numberWays(self, hats):
        """
        :type hats: List[List[int]]
        :rtype: int
        """
        #Approach: Recursion with Memoization
        #Time Complexity: O(n * (2 ^ n) * h)
        #Space Complexity: O(n * (2 ^ n))
        #where, n is the number of people and h is the maximum number of hats a person can wear
        #Note: 10^9 + 7 is a prime number, so Fermat's Little Theorem can be used.

        def countWays(idx, mask):
            if mask == (1 << len(people)) - 1:
                return 1

            if idx == 41:
                return 0

            if (idx, mask) in self.memo:
                return self.memo[(idx, mask)]

            res = countWays(idx + 1, mask)
            for i in range(len(people)):
                if idx in people[i] and mask & (1 << i) == 0:
                    res += countWays(idx + 1, mask | (1 << i))

            self.memo[(idx, mask)] = res % (10 ** 9 + 7)
            return self.memo[(idx, mask)]

        people = [[] for _ in range(41)]
        for i in range(len(hats)):
            for hat in hats[i]:
                people[hat].append(i)

        self.memo = {}
        return countWays(1, 0)

        #Approach: Recursion with Memoization
        #Time Complexity: O(n * (2 ^ n) * h)
        #Space Complexity: O(n * (2 ^ n))
        #where, n is the number of people and h is the maximum number of hats a person can wear
        #Note: 10^9 + 7 is a prime number, so Fermat's Little Theorem can be used.


from typing import Optional, List
from uuid import uuid4
from common_lib.infra.mysql import DB
from chatapp.users.model.friends import GetFriendsRequest, FriendUser, AddFriendRequest, RemoveFriendRequest

class FriendsDAO:
    def __init__(self):
        self.db = DB()

    def get_friends(self, req: GetFriendsRequest) -> List[FriendUser]:
        query = """
        SELECT CUF.FRND_ID          AS FRND_ID
        , CU.FIRST_NAME             AS FIRST_NAME
        , CU.LAST_NAME              AS LAST_NAME
        FROM CHAT_USER_FRND CUF
        INNER JOIN CHAT_USER CU
        ON CUF.FRND_ID = CU.USER_ID
        WHERE CUF.USER_ID = %(user_id)s;
        """

        with self.db.cursor(dictionary=True) as cursor:
            cursor.execute(query, {
                "user_id": req.user_id
            })

            rows = cursor.fetchall()
            friends = []

            for row in rows:
                friend = FriendUser(
                    user_id=row['FRND_ID'],
                    first_name=row['FIRST_NAME'],
                    last_name=row['LAST_NAME']
                )

                friends.append(friend)

            return friends

    def add_friend(self, req: AddFriendRequest):
        query = """
        INSERT INTO CHAT_USER_FRND
        (USER_ID, FRND_ID)
        VALUES
        (%(user_id)s, %(friend_id)s),
        (%(friend_id)s, %(user_id)s)
        """

        with self.db.cursor(dictionary=True) as cursor:
            cursor.execute(query, {
                "user_id": req.user_id,
                "friend_id": req.friend_id
            })

    def remove_friend(self, req: RemoveFriendRequest):
        # Remove friend from both users
        query1 = """
        DELETE FROM CHAT_USER_FRND
        WHERE USER_ID = %(user_id)s
        AND FRND_ID = %(friend_id)s
        """

        query2 = """
        DELETE FROM CHAT_USER_FRND
        WHERE USER_ID = %(friend_id)s
        AND FRND_ID = %(user_id)s
        """

        with self.db.cursor(dictionary=True) as cursor:
            cursor.execute(query1, {
                "user_id": req.user_id,
                "friend_id": req.friend_id
            })

            cursor.execute(query2, {
                "user_id": req.user_id,
                "friend_id": req.friend_id
            })


import sys

# input
input = sys.stdin.readline

N = int(input())
L = list(map(int, input().split()))

left = [-1 for _ in range(N)]
right = [-1 for _ in range(N)]
count = [0 for _ in range(N)]

stack = []
idx = []
for i in range(N):
    while stack:
        if stack[-1] <= L[i]:
            stack.pop()
            idx.pop()
        else:
            break
    count[i] += len(stack)
    if stack:
        left[i] = idx[-1]
    stack.append(L[i])
    idx.append(i)

stack = []
idx = []
for i in range(N-1, -1, -1):
    while stack:
        if stack[-1] <= L[i]:
            stack.pop()
            idx.pop()
        else:
            break
    count[i] += len(stack)
    if stack:
        right[i] = idx[-1]
    stack.append(L[i])
    idx.append(i)

for i in range(N):
    if count[i] == 0:
        print(0)
    else:
        if left[i] == -1:
            print(count[i], right[i]+1)
        elif right[i] == -1:
            print(count[i], left[i]+1)
        elif i-left[i] <= right[i]-i:
            print(count[i], left[i]+1)
        else:
            print(count[i], right[i]+1)


from DERGroupStatusQueries import DERGroupStatusQueries
from DERGroupStatuses import DERGroupStatuses
from message import HeaderType, OperationSet, OptionType, ReplyType
from spyne import ComplexModel, Unicode, DateTime, AnyDict
import datetime as datetime_

class DERGroupStatusQueriesFaultMessageType(ComplexModel):
    _type_info = [
        ('Reply', ReplyType)
    ]

    def __init__(self, reply=None, **kwargs):
        super().__init__(Reply=reply, **kwargs)
        self.Reply = reply
# end class DERGroupStatusQueriesFaultMessageType

class DERGroupStatusQueriesRequestType(ComplexModel):
    _type_info = [
        ('StartTime', DateTime.customize(min_occurs=0)),
        ('EndTime', DateTime.customize(min_occurs=0)),
        ('Option', OptionType.customize(max_occurs='unbounded', min_occurs=0)),
        ('ID', Unicode.customize(max_occurs='unbounded', min_occurs=0)),
        ('DERGroupStatusQueries', DERGroupStatusQueries),
        ('__ANY__', AnyDict)
    ]

    def __init__(self, startTime=None, endTime=None, option=None, id=None, dERGroupStatusQueries=None, **kwargs):
        super().__init__(StartTime=startTime, EndTime=endTime, Option=option, ID=id, DERGroupStatusQueries=dERGroupStatusQueries, **kwargs)
        if isinstance(startTime, str):
            initvalue_ = datetime_.datetime.strptime(startTime, '%Y-%m-%dT%H:%M:%S')
        else:
            initvalue_ = startTime
        self.startTime = initvalue_
        if isinstance(endTime, str):
            initvalue_ = datetime_.datetime.strptime(endTime, '%Y-%m-%dT%H:%M:%S')
        else:
            initvalue_ = endTime
        self.EndTime = initvalue_
        if option is None:
            self.Option = []
        else:
            self.Option = option
        if id is None:
            self.ID = []
        else:
            self.ID = id
        self.DERGroupStatusQueries = dERGroupStatusQueries
        # if anytypeobjs_ is None:
        #     self.anytypeobjs_ = []
        # else:
        #     self.anytypeobjs_ = anytypeobjs_
# end class DERGroupStatusQueriesRequestType

class DERGroupStatusQueriesRequestMessageType(ComplexModel):
    _type_info = [
        ('Header', HeaderType),
        ('Request', DERGroupStatusQueriesRequestType.customize(min_occurs=0))
    ]

    def __init__(self, header=None, request=None, **kwargs):
        super().__init__(Header=header, Request=request, **kwargs)
        self.Header = header
        self.Request = request
# end class DERGroupStatusQueriesRequestMessageType

class DERGroupStatusQueriesPayloadType(ComplexModel):
    _type_info = [
        ('DERGroupStatuses', DERGroupStatuses.customize(min_occurs=0)),
        ('OperationSet', OperationSet.customize(min_occurs=0)),
        ('Compressed', Unicode.customize(min_occurs=0)),
        ('Format', Unicode.customize(min_occurs=0))
    ]

    def __init__(self, dERGroupStatuses=None, operationSet=None, compressed=None, format=None, **kwargs):
        super().__init__(DERGroupStatuses=dERGroupStatuses, OperationSet=operationSet, Compressed=compressed, Format=format, **kwargs)
        self.DERGroupStatuses = dERGroupStatuses
        self.OperationSet = operationSet
        self.Compressed = compressed
        self.Format = format
# end class DERGroupStatusQueriesPayloadType

class DERGroupStatusQueriesResponseMessageType(ComplexModel):
    _type_info = [
        ('Header', HeaderType),
        ('Reply', ReplyType),
        ('Payload', DERGroupStatusQueriesPayloadType.customize(min_occurs=0))
    ]

    def __init__(self, header=None, reply=None, payload=None, **kwargs):
        super().__init__(Header=header, Reply=reply, Payload=payload, **kwargs)
        self.Header = header
        self.Reply = reply
        self.Payload = payload
# end class DERGroupStatusQueriesResponseMessageType

class DERGroupStatusQueriesResponseType(ComplexModel):
    _type_info = [
        ('Header', HeaderType),
        ('Reply', ReplyType),
        ('Payload', DERGroupStatusQueriesPayloadType.customize(min_occurs=0))
    ]

    def __init__(self, header=None, reply=None, payload=None, **kwargs):
        super().__init__(Header=header, Reply=reply, Payload=payload, **kwargs)
        self.header = header
        self.reply = reply
        self.payload = payload
# end class DERGroupStatusQueriesResponseType


import numpy as np
import random
#% matplotlib inline

class Gridworld:
    '''

    '''

    def __init__(self):
        self.num_rows = 5
        self.num_cols = 5
        self.num_cells = self.num_cols * self.num_rows

        # Choose starting position of the agent randomly among the first 5 cells
        self.agent_position = np.random.randint(0, 5)

        # Choose position of the gold and bomb
        self.bomb_positions = np.array([18])
        self.gold_positions = np.array([23])
        self.terminal_states = np.array([self.bomb_positions, self.gold_positions])

        # Specify rewards
        self.rewards = np.zeros(self.num_cells)
        self.rewards[self.bomb_positions] = -10
        self.rewards[self.gold_positions] = 10

        # Specify available actions
        self.actions = ["UP", "RIGHT", "DOWN", "LEFT"]

    def get_available_actions(self):
        return self.actions

    def make_step(self, action):

        if random.random() < 0.2:
            i = np.random.randint(0, 4)
            action = self.actions[i]

        old_position = self.agent_position
        new_position = self.agent_position

        # Update new_position based on the chosen action and check whether agent hits a wall.
        if action == "UP":
            candidate_position = self.agent_position + self.num_cols
            if candidate_position < self.num_cells:
                new_position = candidate_position
        elif action == "RIGHT":
            candidate_position = self.agent_position + 1
            if candidate_position % self.num_cols > 0:
                new_position = candidate_position
        elif action == "DOWN":
            candidate_position = self.agent_position - self.num_cols
            if candidate_position >= 0:
                new_position = candidate_position
        elif action == "LEFT":
            candidate_position = self.agent_position - 1
            if candidate_position % self.num_cols < self.num_cols - 1:
                new_position = candidate_position
        else:
            raise ValueError('Action was mis-specified!')

        # Update the position of the agent.
        self.agent_position = new_position

        # Get reward
        reward = self.rewards[new_position]

        # Deduct 1 from reward if agent moved
        # if old_position != new_position:
        # reward -= 1

        # Deduct 1 from reward for every attempted move
        reward -= 1

        return reward

    def is_terminal(self, state):
        if state in self.terminal_states:
            return True
        else:
            return False

    def choose_random_action(self, available_actions):
        number_of_actions = len(available_actions)
        random_index = np.random.randint(0, number_of_actions)
        action = available_actions[random_index]
        return action

    def get_bomb_location(self):
        return self.bomb_positions

    def get_gold_location(self):
        return self.gold_positions


"""
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time:2022/4/14 17:30
# @Author:YiShouquan
# @File:show_scope.py
# @Update:
"""
def solution(num: int)-> int:
    # your code here
    st = str(num)
    li = list(st)
    for i in range(len(li)):
        if li[i:i+3]==["7","9","7"]:
            li.pop(i+1)
    return int("".join(li))

assert solution(797) == 77
assert solution(7979797) == 7777
assert solution(165561786121789797) == 16556178612178977


import os
import subprocess
import sys
import requests
import json

from django.core.mail import EmailMultiAlternatives
from django.template import RequestContext, TemplateDoesNotExist
from django.template.loader import render_to_string
from django.conf import settings
from django.core.mail import send_mail

from .models import StockData

import datetime
from datetime import timedelta
from time import sleep
from threading import Thread
from scraper.views import YahooScraper

from ib_api.views import *

TODAY = datetime.datetime.today()
MAX_PAST = TODAY - timedelta(30)

def send_email_alert(subject, email_template_name,
              context, to_email, html_email_template_name=None, request=None, from_email=None):
    """
    Sends a django.core.mail.EmailMultiAlternatives to `to_email`.
    """

    print(f'''
    subject: {subject}
    html email template name: {html_email_template_name}
    context: {context}
    to email: {to_email}
    request: {request}
    from email: {from_email}
    ''')
    ctx_dict = {}
    if request is not None:
        ctx_dict = RequestContext(request, ctx_dict)
    # update ctx_dict after RequestContext is created
    # because template context processors
    # can overwrite some of the values like user
    # if django.contrib.auth.context_processors.auth is used
    if context:
        ctx_dict.update(context)

    # Email subject *must not* contain newlines
    from_email = from_email or getattr(settings, 'DEFAULT_FROM_EMAIL')
    if email_template_name:
        message_txt = render_to_string(email_template_name,
                                       ctx_dict)

        email_message = EmailMultiAlternatives(subject, message_txt,
                                               from_email, to_email)
    else:
        try:
            message_html = render_to_string(
                html_email_template_name, ctx_dict)
            email_message = EmailMultiAlternatives(subject, message_html,
                                                   from_email, to_email)
            email_message.content_subtype = 'html'
        except TemplateDoesNotExist:
            pass

    try:
        email_message.send()
    except Exception as e:
        if settings.DEBUG:
            print(f'ERROR: email not sent (utils/utils.py). Reason: {e}')
            print(sys.exc_info())

def date_obj_to_date(date_obj, date_format):
    if date_format == 'slash':
        str_date_arr = str(date_obj).split(' ')[0].split('-')
        year = str_date_arr[0].strip()
        month = str_date_arr[1].strip()
        day = str_date_arr[2].strip()
        date = f'{day}/{month}/{year}'
    else:
        str_date_arr = str(date_obj).split(' ')[0].split('/')
        year = str_date_arr[0].strip()
        month = str_date_arr[1].strip()
        day = str_date_arr[2].strip()

        date = f'{year}-{month}-{day}'

    return date

def change_check(index_change):
    if index_change >= 0.003:
        return 'green'
    elif index_change < 0.003 and index_change >= -0.003:
        return 'orange'
    else:
        return 'red'

RED_OVER_93 = '#FF1000'

def week_color(week_value, week3=False):
    if week_value <= 33:
        if week3:
            return 'green'
        else:
            return '#6cbf6c'
    elif week_value > 33 and week_value <= 65:
        if week3:
            return 'orange'
        else:
            return '#f1b55f'
    elif week_value > 65 and week_value <= 93:
        if week3:
            return 'red'
        else:
            return '#dc6460'
    else:
        if week3:
            return RED_OVER_93
        else:
            return '#dc6460'

def gap_check(stock_df, api_connected=False, realtime_price=None):
    days_back = 1
    gap_count = 0
    gaps = []
    gaps_colors = []

    while gap_count < 4 and days_back < len(stock_df)/2:
        if api_connected:
            open_value = realtime_price
        else:
            open_value = stock_df.loc[stock_df.index[-days_back]]['Open']

        close_value = stock_df.loc[stock_df.index[-days_back-1]]['Close']
        delta_price = open_value - close_value
        delta_rate = (delta_price / close_value) * 100
        if delta_rate > 1.26:
            gaps.append(float("%0.2f"%delta_rate))

            if gap_count == 0:
                gaps_colors.append('green')
            else:
                gaps_colors.append('#6cbf6c')

            gap_count +=1
            days_back += 1

        elif delta_rate >= 1:
            gaps.append(float("%0.2f"%delta_rate))

            if gap_count == 0:
                gaps_colors.append('orange')
            else:
                gaps_colors.append('#f1b55f')

            gap_count +=1
            days_back += 1

        elif delta_rate > 0.75:
            gaps.append(float("%0.2f"%delta_rate))

            if gap_count == 0:
                gaps_colors.append('red')
            else:
                gaps_colors.append('#dc6460')

            gap_count +=1
            days_back += 1

        else:
            days_back += 1

    print(f'gaps: {gaps} \n gaps_colors: {gaps_colors}')

    return gaps, gaps_colors

def gap_1_check (prev_close, todays_open):

    delta_price = todays_open - prev_close
    gap_1 = (delta_price / prev_close) * 100

    if gap_1 > 1.25:
        gap_1_color = 'green'

    elif (gap_1 <=1.25 and gap_1 >= 0.75) or (gap_1 <= -0.75 and gap_1 >= -1.25):
        gap_1_color = 'orange'

    elif gap_1 < -1.25:
        gap_1_color = 'red'
    else:
        gap_1_color = ''

    return round(gap_1,2), gap_1_color

def week_values(stock_df, week_index):

    last_max = stock_df['High'][-week_index:].values.max()
    last_min = stock_df['Low'][-week_index:].values.min()
    current_price = stock_df['Close'].values[-1]

    relative_value = (100  / ( last_max - last_min)) * (current_price - last_min)

    last_max = float("%0.2f"%last_max)
    last_min = float("%0.2f"%last_min)

    relative_value_tmp = float("%0.2f"%relative_value)
    relative_value = 100 if relative_value_tmp > 100 else relative_value_tmp

    return relative_value, last_min, last_max

def send_mail(subject, email_template_name,
              context, to_email, html_email_template_name=None, request=None, from_email=None):
    """
    Sends a django.core.mail.EmailMultiAlternatives to `to_email`.
    """

    print(f'''
    subject: {subject}
    html email template name: {html_email_template_name}
    context: {context}
    to email: {to_email}
    request: {request}
    from email: {from_email}
    ''')
    ctx_dict = {}
    if request is not None:
        ctx_dict = RequestContext(request, ctx_dict)
    # update ctx_dict after RequestContext is created
    # because template context processors
    # can overwrite some of the values like user
    # if django.contrib.auth.context_processors.auth is used
    if context:
        ctx_dict.update(context)

    # Email subject *must not* contain newlines
    from_email = from_email or getattr(settings, 'DEFAULT_FROM_EMAIL')
    if email_template_name:
        message_txt = render_to_string(email_template_name,
                                       ctx_dict)

        email_message = EmailMultiAlternatives(subject, message_txt,
                                               from_email, to_email)
    else:
        try:
            message_html = render_to_string(
                html_email_template_name, ctx_dict)
            email_message = EmailMultiAlternatives(subject, message_html,
                                                   from_email, to_email)
            email_message.content_subtype = 'html'
        except TemplateDoesNotExist:
            pass

    try:
        email_message.send()
    except Exception as e:
        if settings.DEBUG:
            print(f'ERROR: email not sent (utilities.py). Reason: {e}')
            print(sys.exc_info())

def reset_email_alerts():
    # Resetting all email alerts to False
    stocks = StockData.objects.all()
    for stock in stocks:
        stock.stock_email_alert = False
        stock.save()

def reset_alarms(stock):

    stock.stock_alarm_trigger_set = False
    stock.stock_price_down_alarm = False # reseting the up/down alerts
    stock.stock_price_up_alarm = False
    stock.stock_alarm_sound_on_off = False # Resetting the alarm sound
    stock.stock_alarm_delta = 0.0
    stock.stock_initial_price = None

    stock.stock_alarm_1 = None
    stock.stock_alarm_2 = None
    stock.stock_alarm_3 = None
    stock.stock_alarm_4 = None
    stock.stock_alarm_5 = None
    stock.stock_alarm_6 = None
    stock.stock_alarm_7 = None
    stock.stock_alarm_8 = None
    stock.stock_alarm_9 = None
    stock.stock_alarm_10 = None

    stock.stock_alarm_1_color = None
    stock.stock_alarm_2_color = None
    stock.stock_alarm_3_color = None
    stock.stock_alarm_4_color = None
    stock.stock_alarm_5_color = None
    stock.stock_alarm_6_color = None
    stock.stock_alarm_7_color = None
    stock.stock_alarm_8_color = None
    stock.stock_alarm_9_color = None
    stock.stock_alarm_10_color = None

    # stock.stock_load_price = None

    stock.save()

    return

def indices_scrapers(stop=False):
    pass
    # NAS = 55555
    # DOW = 77777
    # SNP = 88888
    # VIX = 11111
    # R2K = 22222

    # dow = {
    #     'url': 'https://finance.yahoo.com/quote/YM=F?p=YM=F',
    #     'index_api_id': DOW
    # }
    # vix = {
    #     'url':'https://finance.yahoo.com/quote/%5EVIX?p=^VIX&.tsrc=fin-srch',
    #     'index_api_id': VIX
    # }

    # snp = {
    #     'url':'https://finance.yahoo.com/quote/ES=F?p=ES=F',
    #     'index_api_id': SNP
    # }

    # nas = {
    #     'url':'https://finance.yahoo.com/quote/NQ=F?p=NQ=F',
    #     'index_api_id': NAS
    # }
    # r2k = {
    #     'url':'https://finance.yahoo.com/quote/RTY=F?p=RTY=F',
    #     'index_api_id': R2K
    # }

    # dow_scraper = YahooScraper(dow)
    # nas_scraper = YahooScraper(nas)
    # snp_scraper = YahooScraper(snp)
    # vix_scraper = YahooScraper(vix)
    # r2k_scraper = YahooScraper(r2k)

    # t_dow = Thread(target=dow_scraper.run)
    # t_nas = Thread(target=nas_scraper.run)
    # t_snp = Thread(target=snp_scraper.run)
    # t_vix = Thread(target=vix_scraper.run)
    # t_r2k = Thread(target=r2k_scraper.run)

    # if stop:
    #     print('#### Stopping scrapers....')
    #     current_running_threads = threading.active_count()
    #     print(f"ACTIVE THREADS AFTER STOPPING: ***{current_running_threads}**")
    # else:
    #     t_dow.start()
    #     t_nas.start()
    #     t_snp.start()
    #     t_vix.start()
    #     t_r2k.start()
    #     current_running_threads = threading.active_count()
    #     print(f"ACTIVE THREADS WITH SCRAPERS: ***************{current_running_threads}****************")
    #     print('SCRAPERS RUNNING...')

def tws_start():
    tws = subprocess.Popen(settings.SCRIPTS_PATH + '/tws_start.sh')
    print(f'>>>>>START TWS: {tws}')

def tws_stop():
    tws = subprocess.Popen(settings.SCRIPTS_PATH + '/tws_stop.sh')
    print(f'>>>>>STOPPING TWS: {tws}')


import sys

sys.path.insert(0, './models/')
import torch
import itertools
from utils import *

def get_model(config):
	device = torch.device('cuda:' + str(config['device_id']))
	model_name = config['model_name']
	if model_name == 'beta_vae':
		from beta_vae import VAE
		model = VAE().to(device)
		optimizer = (torch.optim.Adagrad(model.parameters(), lr=config['learning_r']),)
	elif model_name == 'factor_vae':
		raise NotImplementedError
	elif model_name == 'betavae_cnn':
		from beta_vae_cnn import VAE
		model = VAE(latent_dim=config['latent_dim']).to(device)
		optimizer = (torch.optim.Adam(model.parameters(), lr=config['learning_r']),)
	elif model_name == 'infogan':
		from infogan import InfoGan
		model = InfoGan(config)
		model.decoder.apply(weights_init_normal)
		model.encoder.apply(weights_init_normal)
		g_optimizer = set_optimizer([model.decoder.parameters(), model.encoder.module_Q.parameters(
            ),model.encoder.latent_cont.parameters()], lr=config['learning_r_G'],config=config)
		d_optimizer = set_optimizer([model.encoder.module_shared.parameters(), model.encoder.module_D.parameters()], lr=config['learning_r_D'],config=config)
		optimizer = (d_optimizer, g_optimizer)
	return model, optimizer

def set_optimizer(param_list, lr, config):
	params_to_optimize = itertools.chain(*param_list)
	optimizer = torch.optim.Adam(params_to_optimize, lr=lr, betas=(config['beta1'],  config['beta2']))
	return optimizer


#https://leetcode.com/problems/largest-rectangle-in-histogram/
#Given an array of integers heights representing the histogram's bar height where the width of each bar is 1, return the area of the largest rectangle in the histogram.

class Solution(object):
    def largestRectangleArea(self, heights):
        """
        :type heights: List[int]
        :rtype: int
        """
        # initialize the stack
        stack = []
        # initialize the maximum area
        max_area = 0
        # iterate over the heights
        for i in range(len(heights)):
            # if the stack is not empty and the current height is less than the height at the top of the stack
            while stack and heights[i] < heights[stack[-1]]:
                # set the top of the stack to the current height
                top = stack.pop()
                # if the stack is not empty
                if stack:
                    # set the width to the current index minus the index at the top of the stack minus 1
                    width = i - stack[-1] - 1
                # if the stack is empty
                else:
                    # set the width to the current index
                    width = i
                # set the maximum area to the maximum of the maximum area and the height at the top of the stack times the width
                max_area = max(max_area, heights[top] * width)
            # push the current index onto the stack
            stack.append(i)
        # iterate over the stack
        while stack:
            # set the top of the stack to the current height
            top = stack.pop()
            # if the stack is not empty
            if stack:
                # set the width to the current index minus the index at the top of the stack minus 1
                width = len(heights) - stack[-1] - 1
            # if the stack is empty
            else:
                # set the width to the current index
                width = len(heights)
            # set the maximum area to the maximum of the maximum area and the height at the top of the stack times the width
            max_area = max(max_area, heights[top] * width)
        # return the maximum area
        return max_area


from bs4 import BeautifulSoup
import pandas as pd
import requests

html_data = requests.get("https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-PY0220EN-SkillsNetwork/labs/project/revenue.htm").text

soup = BeautifulSoup(html_data, "html.parser")

revenue_data = []
divs = soup.find_all("div", {"class":"col-xs-6"})

for div in divs:
    if "Tesla Quarterly Revenue" in div.text:
        for tr in div.find_all("tr"):
            tds = tr.find_all("td")
            if len(tds) == 2:
                date = tds[0].text
                revenue = tds[1].text.replace(',', '').replace('$', '')
                revenue_data.append({"Date": date, "Revenue": revenue})

tesla_revenue = pd.DataFrame(revenue_data)

tesla_revenue.dropna(inplace=True)

tesla_revenue = tesla_revenue[tesla_revenue['Revenue'] != ""]

print(tesla_revenue.tail(5))


import torch
import torch.nn as nn
import torch.nn.functional as F

import numpy as np

class LearnedPositionalEmbedding(nn.Module):
    def __init__(self, d_model, max_len=512):
        super().__init__()

        # Compute the positional encodings once in log space.
        pe = torch.zeros(max_len, d_model).float().to(device='cuda')
        pe.require_grad = True
        pe = pe.unsqueeze(0)
        self.pe=nn.Parameter(pe)
        torch.nn.init.normal_(self.pe,std = d_model ** -0.5)

    def forward(self, x):
        return self.pe[:, :x.size(1)]

class BERTEmbedding3(nn.Module):
    """
    BERT Embedding which is consisted with under features
        1. PositionalEmbedding : adding positional information using sin, cos
        sum of all these features are output of BERTEmbedding
    """

    def __init__(self, input_dim, max_len, dropout=0.1):
        """
        :param vocab_size: total vocab size
        :param embed_size: embedding size of token embedding
        :param dropout: dropout rate
        """
        super().__init__()
        self.learnedPosition = LearnedPositionalEmbedding(d_model=input_dim,max_len=max_len)
        self.dropout = nn.Dropout(p=dropout)

    def forward(self, sequence):
        x = self.learnedPosition(sequence)+sequence
        return self.dropout(x)

class Attention(nn.Module):
    """
    Compute 'Scaled Dot Product Attention
    """

    def forward(self, query, key, value, mask=None, dropout=None):
        scores = torch.matmul(query, key.transpose(-2, -1)) \
                 / math.sqrt(query.size(-1))

        if mask is not None:
            scores = scores.masked_fill(mask == 0, -1e9)

        p_attn = F.softmax(scores, dim=-1)

        if dropout is not None:
            p_attn = dropout(p_attn)

        return torch.matmul(p_attn, value), p_attn

class MultiHeadedAttention(nn.Module):
    """
    Take in model size and number of heads.
    """

    def __init__(self, h, d_model, dropout=0.1):
        super().__init__()
        assert d_model % h == 0

        # We assume d_v always equals d_k
        self.d_k = d_model // h
        self.h = h

        self.linear_layers = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(3)])
        self.output_linear = nn.Linear(d_model, d_model)
        self.attention = Attention()

        self.dropout = nn.Dropout(p=dropout)

    def forward(self, query, key, value, mask=None):
        batch_size = query.size(0)

        # 1) Do all the linear projections in batch from d_model => h x d_k
        query, key, value = [l(x).view(batch_size, -1, self.h, self.d_k).transpose(1, 2)
                             for l, x in zip(self.linear_layers, (query, key, value))]

        # 2) Apply attention on all the projected vectors in batch.
        x, attn = self.attention(query, key, value, mask=mask, dropout=self.dropout)

        # 3) "Concat" using a view and apply a final linear.
        x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k)

        return self.output_linear(x)

class GELU(nn.Module):
    """
    Paper Section 3.4, last paragraph notice that BERT used the GELU instead of RELU
    """

    def forward(self, x):
        return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))

class PositionwiseFeedForward(nn.Module):
    "Implements FFN equation."

    def __init__(self, d_model, d_ff, dropout=0.1):
        super(PositionwiseFeedForward, self).__init__()
        self.w_1 = nn.Linear(d_model, d_ff)
        self.w_2 = nn.Linear(d_ff, d_model)
        self.dropout = nn.Dropout(dropout)
        self.activation = GELU()

    def forward(self, x):
        return self.w_2(self.dropout(self.activation(self.w_1(x))))

class LayerNorm(nn.Module):
    "Construct a layernorm module (See citation for details)."

    def __init__(self, features, eps=1e-6):
        super(LayerNorm, self).__init__()
        self.a_2 = nn.Parameter(torch.ones(features))
        self.b_2 = nn.Parameter(torch.zeros(features))
        self.eps = eps

    def forward(self, x):
        mean = x.mean(-1, keepdim=True)
        std = x.std(-1, keepdim=True)
        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2

class SublayerConnection(nn.Module):
    """
    A residual connection followed by a layer norm.
    Note for code simplicity the norm is first as opposed to last.
    """

    def __init__(self, size, dropout):
        super(SublayerConnection, self).__init__()
        self.norm = LayerNorm(size)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, sublayer):
        "Apply residual connection to any sublayer with the same size."
        return x + self.dropout(sublayer(self.norm(x)))

class TransformerBlock(nn.Module):
    """
    Bidirectional Encoder = Transformer (self-attention)
    Transformer = MultiHead_Attention + Feed_Forward with sublayer connection
    """

    def __init__(self, hidden, attn_heads, feed_forward_hidden, dropout):
        """
        :param hidden: hidden size of transformer
        :param attn_heads: head sizes of multi-head attention
        :param feed_forward_hidden: feed_forward_hidden, usually 4*hidden_size
        :param dropout: dropout rate
        """

        super().__init__()
        self.attention = MultiHeadedAttention(h=attn_heads, d_model=hidden)
        self.feed_forward = PositionwiseFeedForward(d_model=hidden, d_ff=feed_forward_hidden, dropout=dropout)
        self.input_sublayer = SublayerConnection(size=hidden, dropout=dropout)
        self.output_sublayer = SublayerConnection(size=hidden, dropout=dropout)
        self.dropout = nn.Dropout(p=dropout)

    def forward(self, x, mask):
        x = self.input_sublayer(x, lambda _x: self.attention.forward(_x, _x, _x, mask=mask))
        x = self.output_sublayer(x, self.feed_forward)
        return self.dropout(x)

class BERT2(nn.Module):
    """
    BERT model : Bidirectional Encoder Representations from Transformers.
    """

    def __init__(self, input_dim, max_len, hidden=768, n_layers=12, attn_heads=12, dropout=0.1, mask_prob=0.8):
        """
        :param vocab_size: vocab_size of total words
        :param hidden: BERT model hidden size
        :param n_layers: numbers of Transformer blocks(layers)
        :param attn_heads: number of attention heads
        :param dropout: dropout rate
        """

        super().__init__()
        self.hidden = hidden
        self.n_layers = n_layers
        self.attn_heads = attn_heads
        self.max_len=max_len
        self.input_dim=input_dim
        self.mask_prob=mask_prob

        clsToken = torch.zeros(1,1,self.input_dim).float().cuda()
        clsToken.require_grad = True
        self.clsToken= nn.Parameter(clsToken)
        torch.nn.init.normal_(self.clsToken, std = hidden ** -0.5)

        # paper noted they used 4*hidden_size for ff_network_hidden_size
        self.feed_forward_hidden = hidden * 4

        # embedding for BERT, sum of positional, segment, token embeddings
        self.embedding = BERTEmbedding3(input_dim=input_dim, max_len=max_len+1)

        # multi-layers transformer blocks, deep network
        self.transformer_blocks = nn.ModuleList(
            [TransformerBlock(hidden, attn_heads, self.feed_forward_hidden, dropout) for _ in range(n_layers)])

        for module in self.modules():
            if isinstance(module, nn.Embedding):
                #nn.init.normal_(module.weight, mean=0, std=0.02)
                nn.init.uniform_(module.weight, -0.06, 0.06)
            if isinstance(module, nn.Linear):
                nn.init.normal_(module.weight, mean=0, std=0.02)
                if hasattr(module, "bias") and module.bias is not None:
                    nn.init.constant_(module.bias, 0.0)
            if isinstance(module, nn.LayerNorm):
                module.bias.data.zero_()
                module.weight.data.fill_(1.0)

    def forward(self, input_vectors):
        # attention masking for padded token
        # torch.ByteTensor([batch_size, 1, seq_len, seq_len)
        batch_size=input_vectors.shape[0]
        sample=None
        if self.training:
            bernolliMatrix=torch.cat((torch.tensor([1]).float().cuda(), (torch.tensor([self.mask_prob]).float().cuda()).repeat(self.max_len)), 0).unsqueeze(0).repeat([batch_size,1])
            self.bernolliDistributor=torch.distributions.Bernoulli(bernolliMatrix)
            sample=self.bernolliDistributor.sample()
            mask = (sample > 0).unsqueeze(1).repeat(1, sample.size(1), 1).unsqueeze(1)
        else:
            mask=torch.ones(batch_size,1,self.max_len+1,self.max_len+1).cuda()

        # embedding the indexed sequence to sequence of vectors
        x = torch.cat((self.clsToken.repeat(batch_size,1,1),input_vectors),1)
        x = self.embedding(x)

        # running over multiple transformer blocks
        for transformer in self.transformer_blocks:
            x = transformer.forward(x, mask)

        return x, sample

class MaxPool3dSamePadding(nn.MaxPool3d):

    def compute_pad(self, dim, s):
        if s % self.stride[dim] == 0:
            return max(self.kernel_size[dim] - self.stride[dim], 0)
        else:
            return max(self.kernel_size[dim] - (s % self.stride[dim]), 0)

    def forward(self, x):
        # compute 'same' padding
        (batch, channel, t, h, w) = x.size()
        #print t,h,w
        out_t = np.ceil(float(t) / float(self.stride[0]))
        out_h = np.ceil(float(h) / float(self.stride[1]))
        out_w = np.ceil(float(w) / float(self.stride[2]))
        #print out_t, out_h, out_w
        pad_t = self.compute_pad(0, t)
        pad_h = self.compute_pad(1, h)
        pad_w = self.compute_pad(2, w)
        #print pad_t, pad_h, pad_w

        pad_t_f = pad_t // 2
        pad_t_b = pad_t - pad_t_f
        pad_h_f = pad_h // 2
        pad_h_b = pad_h - pad_h_f
        pad_w_f = pad_w // 2
        pad_w_b = pad_w - pad_w_f

        pad = (pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)
        #print x.size()
        #print pad
        x = F.pad(x, pad)
        return super(MaxPool3dSamePadding, self).forward(x)

class Unit3D(nn.Module):

    def __init__(self, in_channels,
                 output_channels,
                 kernel_shape=(1, 1, 1),
                 stride=(1, 1, 1),
                 padding=0,
                 activation_fn=F.relu,
                 use_batch_norm=True,
                 use_bias=False,
                 name='unit_3d'):

        """Initializes Unit3D module."""
        super(Unit3D, self).__init__()

        self._output_channels = output_channels
        self._kernel_shape = kernel_shape
        self._stride = stride
        self._use_batch_norm = use_batch_norm
        self._activation_fn = activation_fn
        self._use_bias = use_bias
        self.name = name
        self.padding = padding

        self.conv3d = nn.Conv3d(in_channels=in_channels,
                                out_channels=self._output_channels,
                                kernel_size=self._kernel_shape,
                                stride=self._stride,
                                padding=0, # we always want padding to be 0 here. We will dynamically pad based on input size in forward function
                                bias=self._use_bias)

        if self._use_batch_norm:
            self.bn = nn.BatchNorm3d(self._output_channels, eps=0.001, momentum=0.01)

    def compute_pad(self, dim, s):
        if s % self._stride[dim] == 0:
            return max(self._kernel_shape[dim] - self._stride[dim], 0)
        else:
            return max(self._kernel_shape[dim] - (s % self._stride[dim]), 0)

    def forward(self, x):
        # compute 'same' padding
        (batch, channel, t, h, w) = x.size()
        #print t,h,w
        out_t = np.ceil(float(t) / float(self._stride[0]))
        out_h = np.ceil(float(h) / float(self._stride[1]))
        out_w = np.ceil(float(w) / float(self._stride[2]))
        #print out_t, out_h, out_w
        pad_t = self.compute_pad(0, t)
        pad_h = self.compute_pad(1, h)
        pad_w = self.compute_pad(2, w)
        #print pad_t, pad_h, pad_w

        pad_t_f = pad_t // 2
        pad_t_b = pad_t - pad_t_f
        pad_h_f = pad_h // 2
        pad_h_b = pad_h - pad_h_f
        pad_w_f = pad_w // 2
        pad_w_b = pad_w - pad_w_f

        pad = (pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)
        #print x.size()
        #print pad
        x = F.pad(x, pad)
        #print x.size()

        x = self.conv3d(x)
        if self._use_batch_norm:
            x = self.bn(x)
        if self._activation_fn is not None:
            x = self._activation_fn(x)
        return x

class InceptionModule(nn.Module):
    def __init__(self, in_channels, out_channels, name):
        super(InceptionModule, self).__init__()

        self.b0 = Unit3D(in_channels=in_channels, output_channels=out_channels[0], kernel_shape=[1, 1, 1], padding=0,
                         name=name+'/Branch_0/Conv3d_0a_1x1')
        self.b1a = Unit3D(in_channels=in_channels, output_channels=out_channels[1], kernel_shape=[1, 1, 1], padding=0,
                          name=name+'/Branch_1/Conv3d_0a_1x1')
        self.b1b = Unit3D(in_channels=out_channels[1], output_channels=out_channels[2], kernel_shape=[3, 3, 3],
                          name=name+'/Branch_1/Conv3d_0b_3x3')
        self.b2a = Unit3D(in_channels=in_channels, output_channels=out_channels[3], kernel_shape=[1, 1, 1], padding=0,
                          name=name+'/Branch_2/Conv3d_0a_1x1')
        self.b2b = Unit3D(in_channels=out_channels[3], output_channels=out_channels[4], kernel_shape=[3, 3, 3],
                          name=name+'/Branch_2/Conv3d_0b_3x3')
        self.b3a = MaxPool3dSamePadding(kernel_size=[3, 3, 3],
                                stride=(1, 1, 1), padding=0)
        self.b3b = Unit3D(in_channels=in_channels, output_channels=out_channels[5], kernel_shape=[1, 1, 1], padding=0,
                          name=name+'/Branch_3/Conv3d_0b_1x1')
        self.name = name

    def forward(self, x):
        b0 = self.b0(x)
        b1 = self.b1b(self.b1a(x))
        b2 = self.b2b(self.b2a(x))
        b3 = self.b3b(self.b3a(x))
        return torch.cat([b0,b1,b2,b3], dim=1)

class InceptionI3d(nn.Module):
    """Inception-v1 I3D architecture.
    The model is introduced in:
        Quo Vadis, Action Recognition? A New Model and the Kinetics Dataset
        Joao Carreira, Andrew Zisserman
        https://arxiv.org/pdf/1705.07750v1.pdf.
    See also the Inception architecture, introduced in:
        Going deeper with convolutions
        Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed,
        Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich.
        http://arxiv.org/pdf/1409.4842v1.pdf.
    """

    # Endpoints of the model in order. During construction, all the endpoints up
    # to a designated `final_endpoint` are returned in a dictionary as the
    # second return value.
    VALID_ENDPOINTS = (
        'Conv3d_1a_7x7',
        'MaxPool3d_2a_3x3',
        'Conv3d_2b_1x1',
        'Conv3d_2c_3x3',
        'MaxPool3d_3a_3x3',
        'Mixed_3b',
        'Mixed_3c',
        'MaxPool3d_4a_3x3',
        'Mixed_4b',
        'Mixed_4c',
        'Mixed_4d',
        'Mixed_4e',
        'Mixed_4f',
        'MaxPool3d_5a_2x2',
        'Mixed_5b',
        'Mixed_5c',
        'Logits',
        'Predictions',
    )

    def __init__(self, num_classes=400, spatial_squeeze=True,
                 final_endpoint='Logits', name='inception_i3d', in_channels=3, dropout_keep_prob=0.5):
        """Initializes I3D model instance.
        Args:
          num_classes: The number of outputs in the logit layer (default 400, which
              matches the Kinetics dataset).
          spatial_squeeze: Whether to squeeze the spatial dimensions for the logits
              before returning (default True).
          final_endpoint: The model contains many possible endpoints.
              `final_endpoint` specifies the last endpoint for the model to be built
              up to. In addition to the output at `final_endpoint`, all the outputs
              at endpoints up to `final_endpoint` will also be returned, in a
              dictionary. `final_endpoint` must be one of
              InceptionI3d.VALID_ENDPOINTS (default 'Logits').
          name: A string (optional). The name of this module.
        Raises:
          ValueError: if `final_endpoint` is not recognized.
        """

        if final_endpoint not in self.VALID_ENDPOINTS:
            raise ValueError('Unknown final endpoint %s' % final_endpoint)

        super(InceptionI3d, self).__init__()
        self._num_classes = num_classes
        self._spatial_squeeze = spatial_squeeze
        self._final_endpoint = final_endpoint
        self.logits = None

        if self._final_endpoint not in self.VALID_ENDPOINTS:
            raise ValueError('Unknown final endpoint %s' % self._final_endpoint)

        self.end_points = {}
        end_point = 'Conv3d_1a_7x7'
        self.end_points[end_point] = Unit3D(in_channels=in_channels, output_channels=64, kernel_shape=[7, 7, 7],
                                            stride=(2, 2, 2), padding=(3,3,3),  name=name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'MaxPool3d_2a_3x3'
        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),
                                                             padding=0)
        if self._final_endpoint == end_point: return

        end_point = 'Conv3d_2b_1x1'
        self.end_points[end_point] = Unit3D(in_channels=64, output_channels=64, kernel_shape=[1, 1, 1], padding=0,
                                       name=name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'Conv3d_2c_3x3'
        self.end_points[end_point] = Unit3D(in_channels=64, output_channels=192, kernel_shape=[3, 3, 3], padding=1,
                                       name=name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'MaxPool3d_3a_3x3'
        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),
                                                             padding=0)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_3b'
        self.end_points[end_point] = InceptionModule(192, [64,96,128,16,32,32], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_3c'
        self.end_points[end_point] = InceptionModule(256, [128,128,192,32,96,64], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'MaxPool3d_4a_3x3'
        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[3, 3, 3], stride=(2, 2, 2),
                                                             padding=0)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_4b'
        self.end_points[end_point] = InceptionModule(128+192+96+64, [192,96,208,16,48,64], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_4c'
        self.end_points[end_point] = InceptionModule(192+208+48+64, [160,112,224,24,64,64], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_4d'
        self.end_points[end_point] = InceptionModule(160+224+64+64, [128,128,256,24,64,64], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_4e'
        self.end_points[end_point] = InceptionModule(128+256+64+64, [112,144,288,32,64,64], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_4f'
        self.end_points[end_point] = InceptionModule(112+288+64+64, [256,160,320,32,128,128], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'MaxPool3d_5a_2x2'
        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[2, 2, 2], stride=(2, 2, 2),
                                                             padding=0)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_5b'
        self.end_points[end_point] = InceptionModule(256+320+128+128, [256,160,320,32,128,128], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'Mixed_5c'
        self.end_points[end_point] = InceptionModule(256+320+128+128, [384,192,384,48,128,128], name+end_point)
        if self._final_endpoint == end_point: return

        end_point = 'Logits'
        self.avg_pool = nn.AvgPool3d(kernel_size=[2, 7, 7],
                                     stride=(1, 1, 1))
        self.dropout = nn.Dropout(dropout_keep_prob)
        self.logits = Unit3D(in_channels=384+384+128+128, output_channels=self._num_classes,
                             kernel_shape=[1, 1, 1],
                             padding=0,
                             activation_fn=None,
                             use_batch_norm=False,
                             use_bias=True,
                             name='logits')

        self.build()

    def replace_logits(self, num_classes):
        self._num_classes = num_classes
        self.logits = Unit3D(in_channels=384+384+128+128, output_channels=self._num_classes,
                             kernel_shape=[1, 1, 1],
                             padding=0,
                             activation_fn=None,
                             use_batch_norm=False,
                             use_bias=True,
                             name='logits')

    def build(self):
        for k in self.end_points.keys():
            self.add_module(k, self.end_points[k])

    def forward(self, x):
        for end_point in self.VALID_ENDPOINTS:
            if end_point in self.end_points:
                x = self._modules[end_point](x) # use _modules to work with dataparallel

        x = self.logits(self.dropout(self.avg_pool(x)))
        if self._spatial_squeeze:
            logits = x.squeeze(3).squeeze(3)
        # logits is batch X time X classes, which is what we want to work with
        return logits

    def extract_features(self, x):
        for end_point in self.VALID_ENDPOINTS:
            if end_point in self.end_points:
                x = self._modules[end_point](x)
        return self.avg_pool(x)

def _inception(model_path, **kwargs):
    """Constructs a ResNet-18 model.

    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = InceptionI3d(400, in_channels=3)
    if model_path=='':
        return model
    params = torch.load(model_path)
    model.load_state_dict(params)
    return model

class rgb_I3D64f(nn.Module):
    def __init__(self, modelPath='./weights/rgb_imagenet.pth'):
        super(rgb_I3D64f, self).__init__()
        # self.dp = nn.Dropout(p=0.8)
        self.avgpool = nn.AvgPool3d((1, 7, 7), stride=1)
        self.features=nn.Sequential(*list(_inception(model_path=modelPath).children())[3:])

        # self.fc_action = nn.Linear(1024, num_classes)
        # for param in self.features.parameters():
        #     param.requires_grad = True

        # torch.nn.init.xavier_uniform_(self.fc_action.weight)
        # self.fc_action.bias.data.zero_()

    def forward(self, x):
        x = self.features(x)
        x = self.avgpool(x)
        # x = x.view(x.size(0), -1)
        # x = self.dp(x)
        # x = self.fc_action(x)
        return x

class rgb_I3D64f_bert2(nn.Module):
    def __init__(self, num_classes, modelPath='./weights/rgb_imagenet.pth', extract=True):
        super(rgb_I3D64f_bert2, self).__init__()
        self.extract = True
        self.eval = True
        self.hidden_size=1024
        self.n_layers=1
        self.attn_heads=8
        self.num_classes=num_classes
        # self.length=length
        self.dp = nn.Dropout(p=0.8)

        self.features=nn.Sequential(*list(_inception(model_path=modelPath).children())[3:])

        for param in self.features.parameters():
            param.requires_grad = True

        self.bert = BERT2(self.hidden_size, 8 , hidden=self.hidden_size, n_layers=self.n_layers, attn_heads=self.attn_heads)
        print(sum(p.numel() for p in self.bert.parameters() if p.requires_grad))
        #
        self.avgpool = nn.AdaptiveAvgPool3d(output_size=(8, 1, 1))

        # used for classification task only
        if self.train:
            self.fc_action = nn.Linear(self.hidden_size, num_classes)
            torch.nn.init.xavier_uniform_(self.fc_action.weight)
            self.fc_action.bias.data.zero_()

    def forward(self, x):
        # print('input\t' + str(np.shape(x)))
        x = self.features(x)
        # print('feature extracted\t' + str(np.shape(x)))
        x = self.avgpool(x)
        # print('average pooled\t' + str(np.shape(x)))
        x = x.view(x.size(0), self.hidden_size,8)
        x = x.transpose(1,2)
        norm = x.norm(p=2, dim = -1, keepdim=True)
        x = x.div(norm)
        input_vectors=x
        # print('transformed\t' + str(np.shape(x)))
        output , maskSample = self.bert(x)
        # print('BERTed\t' + str(np.shape(output)))

        sequenceOut=output[:,1:,:]
        if self.extract:
            return sequenceOut
        else:
            classificationOut = output[:,0,:]
            if eval:
                output=self.dp(classificationOut)
            else:
                output = classificationOut
            # print('BERTed2\t' + str(np.shape(output)))
            x = self.fc_action(output)
            # print('output\t' + str(np.shape(x)))
            # print(np.shape(x))
            return x, input_vectors, sequenceOut, maskSample

    def eval(self):
        self.eval = True

    def train(self):
        self.eval = False