from selenium import webdriver
import time
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import csv
import os
import openpyxl
from tkinter import Tk
import argparse
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager
import re

def scrap_from_url(args, driver,url=None, load_page=True):
    page = 0
    max_page = 55
    extracted_infos = []
    count = 0
    item_num = 1
    user_ids = set()
    while page < max_page:
        links = []
        page += 1
        print('page: ', page)
        names = driver.find_elements_by_tag_name("a")
        for name in names:
            link = name.get_attribute('href')
            if 'https://shopping.yahoo.co.jp/review/item/list?store_id=' in link:
                links.append(link)
        # print('num of link: ', len(links))
        for link in links:
            count += 1
            print(count)
            url=link
            url_splt = re.split('[?.,=_/&]',url)
            user_id = url_splt[url_splt.index('id') + 1]

            if user_id not in user_ids:
                 user_ids.add(user_id)
                 print('Totally', len(user_ids), 'stores have been scanned')
            else:
                continue
            template1 = 'https://store.shopping.yahoo.co.jp/'
            template2 = '/info.html'
            info_page = template1 + user_id + template2

            driver.get(info_page)
            if ('error' in driver.current_url) or ('notfound' in driver.current_url):
                continue
            info = driver.find_element_by_tag_name('body').text.split()
            person_profile = {}
            shop_name = info[info.index('会社名（商号）') + 1]
            address = info[info.index('住所') + 1]
            manager_name = info[info.index('代表者') + 1]
            phone=''
            fax=''
            email=''
            for pos,element in enumerate(info):
                if 'お問い合わせ' == element:
                    phone = info[pos + 1]
                    email = info[pos + 2]
                    break
                if 'お問い合わせ電話番号' == element:
                    phone = info[pos + 1]
                if 'お問い合わせファックス番号' == element:
                    fax = info[pos + 1]
                if 'お問い合わせメールアドレス' == element:
                    email = info[pos + 1]
                    break
            person_profile['SHOP NAME'] = shop_name
            person_profile['MANAGER'] = manager_name
            person_profile['ADDRESS'] = address
            person_profile['PHONE'] = phone
            person_profile['FAX'] = fax
            person_profile['EMAIL'] = email
            extracted_infos.append(person_profile)
        if page % 2 == 0:
            with open('results_yahoo14.csv', 'w', encoding='utf8', newline='') as output_file:
                fc = csv.DictWriter(output_file, fieldnames=extracted_infos[0].keys())
                fc.writeheader()
                fc.writerows(extracted_infos)
        item_num += 30
        next_page = 'https://shopping.yahoo.co.jp/category/13457/2494/37524/list?X=8&view=list&sc_i=shp_pc_catelist_nrwcgt&b='
        next_page = next_page + str(item_num)
        driver.get(next_page)
    with open('results_yahoo14.csv', 'w', encoding='utf8', newline='') as output_file:
        fc = csv.DictWriter(output_file, fieldnames=extracted_infos[0].keys())
        fc.writeheader()
        fc.writerows(extracted_infos)
    print('Done')
####################################################################################
if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--search_words', default=None)
    parser.add_argument('--url', default=None)
    parser.add_argument('--mode', default='url')
    args = parser.parse_args()

    # driver = webdriver.Chrome(ChromeDriverManager(version="91.0.4472.101").install())
    driver = webdriver.Chrome()
    # driver.set_page_load_timeout(1)
    driver.get('https://shopping.yahoo.co.jp/category/13457/2494/37524/list?X=8&view=list&sc_i=shp_pc_catelist_nrwcgt&b=1')
    # driver.execute_script("driver.stop();")
    # driver.maximize_window()
    scrap_from_url(args, driver, args.url, load_page=False)


from unified.core.auth import Auth
from datetime import datetime, timezone
import requests
import json

TYPEFORM_BASE_URL = 'https://api.typeform.com'

def rest(method, url, body=None, token=None):

    headers = {
        "Authorization": f"Bearer {token}",
        "Content-Type": "application/json"
    }

    return requests.request(method, url, headers=headers, json=body)

def epoch_to_format(format, epoch):
    """Convert  epoch to given format"""

    return datetime.fromtimestamp(int(epoch[:10]), tz=timezone.utc).strftime(format)

def get_typeform_answer(typeform_answers, field_id):
    """Get the answer for the field with given"""

    for answer in typeform_answers:
        if field_id == answer['field']['id']:
            if answer['type'] == 'choice':
                return answer[answer['type']]['label']
            else:
                return answer[answer['type']]

    return None

def remove_field(pl, key):
    """Remove items with given key in any dict at any level in the nested dict/list/tuple"""

    if type(pl) is tuple:
        r = (remove_field(v, key) for v in pl)

    elif type(pl) is list:
        r = [remove_field(v, key) for v in pl]

    elif type(pl) is dict:
        r = {k: remove_field(v, key) for (k, v) in pl.items() if k != key}
    else:
        r = pl

    return r


# ----------------------------------------------------------------------------
# Created By  : Tobias Mink, Marvin Kemper
# ---------------------------------------------------------------------------
"""
Creates a legend which holds metadata corresponding to the colors of the mesh.
"""
# ---------------------------------------------------------------------------
from data.dictionarys import original_layers
from data.lists import colors

def do(self, task: str):
    if task == 'remove':
        self.plotter.remove_legend()
    if task == 'add':
        new_list = []
        for mesh_name, color in zip(original_layers.keys(), colors):
            new_list.append([mesh_name, color])
        self.plotter.add_legend(labels=new_list, bcolor='#0c1726', face="r", loc="upper left", size=(0.1, 0.1))


#!/usr/bin/python3
""" Base class module """

import json

class Base:
    """ My Base class """

    __nb_objects = 0

    def __init__(self, id=None):
        ''' Instantiates the new Base object '''
        if id is not None:
            self.id = id
        else:
            Base.__nb_objects += 1
            self.id = Base.__nb_objects

    @staticmethod
    def to_json_string(list_dictionaries):
        ''' that returns the JSON string representation
        of list_dictionaries

        Args:
            list_dictionaries: is a list of dictionaries '''
        if list_dictionaries is None or list_dictionaries == '[]':
            return "[]"
        else:
            return json.dumps(list_dictionaries)

    @classmethod
    def save_to_file(cls, list_objs):
        ''' writes the JSON string representation
        of list_objs to a file '''
        filename = cls.__name__ + '.json'
        dics = []
        #lists = []
        #with open(filename, mode='w') as file:
        #    if list_objs is None:
        #        file.write('[]')
        #    else:
        #        for i in range(len(list_objs)):
        #            list_dic.append(list_objs[i].to_dictionary())

        if list_objs is None:
            return []
        for lists in list_objs:
            dics.append(lists.to_dictionary())

        with open(filename, mode='w') as file:
            file.write(Base.to_json_string(dics))

    def from_json_string(json_string):
        ''' from json to dic '''
        if json_string is None or json_string == '[]':
            return '[]'
        return json.loads(json_string)


from django.db import models
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.core.validators import MaxValueValidator, MinValueValidator

# Create your models here.

CATEGORY = (
        ('Education', 'Education'),
        ('Healthcare', 'Healthcare'),
        ('Hospitality', 'Hospitality'),
        ('Manufacturing', 'Manufacturing'),
        ('Media', 'Media'),
        ('Software', 'Software'),
    )

class Company(models.Model):
	admin = models.OneToOneField(User, on_delete=models.SET_NULL, null=True)
	name = models.CharField(_("Company Name"), max_length=200, unique=True)
	category = models.CharField(choices=CATEGORY, max_length=100)
	email = models.CharField(max_length=100, unique=True)
	mobile = models.PositiveIntegerField(blank=True, null=True, validators=[MaxValueValidator(9999999999)])
	address = models.TextField(max_length=200, blank=True, null=True)

	def __str__(self):
		return self.name

class UserProfile(models.Model):
	user = models.OneToOneField(User, on_delete=models.CASCADE)
	name = models.CharField(max_length=255)
	date_of_birth = models.DateField(_("Date of birth"), blank=True, null=True)
	email = models.EmailField(max_length=255, unique=True, null=True)
	mobile = models.PositiveIntegerField(blank=True, null=True, validators=[MaxValueValidator(9999999999)])
	company = models.ForeignKey(Company, on_delete=models.CASCADE)

	def __str__(self):
		return self.user.username


while True:
    parameters = input().split()

    if parameters == ['0']:
        break

    A, B, C = parameters
    A, B, C = int(A), int(B), float(C)/100

    houseSize = A * B
    sizeLand = (houseSize * 1)/C
    houseRealSize = int(sizeLand**(1/2))

    print(houseRealSize)


import pickle
from gensim.models import Word2Vec

with open("data/word_vector.pkl", "rb") as f:
    word_vector = pickle.load(f)

#train word2vec model
model = Word2Vec(sentences=word_vector, window=3, sg=1, hs=0, negative=20,
                 alpha=0.03, min_alpha=0.0007, vector_size=100, epochs=10,
                 seed=24)

#save the model
with open("model/word2vec_model.pkl", "wb") as f:
    pickle.dump(model, f, protocol=pickle.HIGHEST_PROTOCOL)


__author__ = 'davidnola'

class PassthroughModel:
    def fit(self, features, targets):
        best_score = 999999999999999999

        for i in range(len(features[0])):
            curscore = 0
            for j in range(len(features)):
                curscore += (targets[j]-features[j][i])
            if curscore < best_score:
                best_score = curscore
                self.best = i

    def predict(self, features):
        #print "Best is:", self.best
        return [f[self.best] for f in features]

    def get_params(self):
        return "{}"

    def score(self, features, targets):
        p = self.predict(features)
        p = [round(x, 0) for x in p]
        score = 0
        for i in range(len(p)):
            if p[i] == targets[i]:
                score+=1

        score = score/float(len(targets))
        return score


#!/usr/bin/env python

"""
Common utilities that can be widely used across the package
"""

import numpy as np
import os
import glob
from itertools import groupby
import sys
import time

from icecube.utils.logger import Logger

logger = Logger(os.path.basename(__file__))

class DirUtils:
    @staticmethod
    def get_dir_files(local_dir, fext=None):
        """
        scrape the given directory for all files or files with certain extension.
        fext: str or list of strings.
        """
        local_dir = os.path.join(local_dir, "")
        fpaths = []  # contains names of all images.

        if fext and not (isinstance(fext, list)):
            fext = list(fext)

        if fext:
            for ext in fext:
                ext = ext.replace("*", "")  # if asterik was passed by mistake.
                fpaths.extend(glob.glob(local_dir + "*" + ext))
        else:
            fpaths.extend(glob.glob(local_dir + "*"))

        fnames = [p.replace(local_dir, "") for p in fpaths]

        return fnames, fpaths

def get_dummy_metadata():
    """
    Incase of a temporal gap, the following metadata is used
    """
    return dict({"product_file": "None"})

def get_product_metadata(product_file):
    """
    Metadata appended for labels xr.DataArray
    """
    return dict({"product_file": product_file})

def assert_metadata_exists(metadata_df):
    """
    Make sure that metadata is non-empty
    Returns True if non-empty otherwise raises exception
    """
    for df_col in metadata_df.columns:
        if metadata_df[df_col].isnull().values.all():
            raise ValueError(
                "No valid values found for pd.Series: {} for pd.column:{} against given configuration".format(
                    metadata_df[df_col], df_col
                )
            )

    return True

def all_equal(iterable):
    g = groupby(iterable)
    return next(g, True) and not next(g, False)

def match_dict_key(d, value):
    """
    get key by matching value of a dictionary.
    d: dictionary of type dict.
    value: value to match
    returns: key of matched value.
    """
    for k, v in d.items():
        if v == value:
            return k

    print(f"value: {value} not found in dictionary")

def draw_progress_bar(percent, barLen=50):
    sys.stdout.write("\r")
    progress = ""
    for i in range(barLen):
        if i < int(barLen * percent):
            progress += "="
        else:
            progress += " "
    sys.stdout.write(f"[ {progress} ] {percent * 100:.2f}%")
    sys.stdout.flush()

def measure_time(func):
    """
    :param func:
    :return:
    """

    def time_it(*args, **kwargs):
        time_started = time.time()
        return_value = func(*args, **kwargs)
        time_elapsed = time.time()
        logger.info(
            "{execute} running time is {sec} seconds".format(
                execute=func.__name__,
                sec=round(time_elapsed - time_started, 4),
            )
        )
        return return_value

    return time_it

class NumpyEncoder:
    """Special json encoder for numpy types"""

    @classmethod
    def encode(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return str(obj.tolist())
        else:
            return obj

def get_slc_metadata(h5f):
    """
    Read metadata from SLC header

    Args:
        opened_h5_file: SLC h5 file opened as a h5py File
    """

    meta_dict = {}

    # h5py doesn't read .keys() correctly, but returns a generator. So parsing to list to get all keys.
    key_list = [x for x in h5f.keys()]

    # Check if there already is a list of bands
    if "bands" in key_list:
        non_meta_keys = h5f["bands"]
    else:
        non_meta_keys = ["s_i", "s_q"]

    # Splines are not passed as they are not necessary. We just skip them for now.
    # This doesn't not effect the SLC in any meaningfull way
    additional_bands_to_skip_for_now = [
        "RPC",
        "height_spline",
        "lat_spline",
        "lon_spline",
    ]
    for key in key_list:
        if key not in non_meta_keys + additional_bands_to_skip_for_now:

            h5_meta_val = h5f[key]

            if type(h5_meta_val[()]) == bytes:
                # h5py 3.0+ introduces str stored as bytes. We use .asstr() to read it (but breaks with h5py < 2.10)
                meta_dict[key] = np.array_str(np.array(h5_meta_val.asstr()[()]))

            elif type(h5_meta_val[()]) == np.bytes_:
                # This is most likely a date
                meta_dict[key] = np.array_str(
                    np.squeeze(np.char.decode(h5_meta_val, "utf-8"))
                )

            else:
                # We're handling a numpy array so we should be good to go
                meta_dict[key] = np.array_str(np.array(h5_meta_val[()]))

                # RPCs are nested under "RPC/" in the h5 thus need to be parsed in a specific manner
    RPC_source = h5f["RPC"]

    for key, val in RPC_source.items():
        meta_dict[f"RPC_{key}"] = np.array(val, dtype=np.float32)

    return meta_dict


import code
import readline
import atexit
import os

class HistoryConsole(code.InteractiveConsole):
    def __init__(self, locals=None, filename="<console>",
                 histfile=os.path.expanduser("~/.console-history")):
        code.InteractiveConsole.__init__(self, locals, filename)
        self.init_history(histfile)

    def init_history(self, histfile):
        readline.parse_and_bind("tab: complete")
        if hasattr(readline, "read_history_file"):
            try:
                readline.read_history_file(histfile)
            except IOError:
                pass
            atexit.register(self.save_history, histfile)

    def save_history(self, histfile):
        readline.set_history_length(1000)
        readline.write_history_file(histfile)

if __name__ == '__main__':
    while True:
        c = HistoryConsole().raw_input()
        print(c)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

try:
	from setuptools import setup
except ImportError:
	from distutils.core import setup

PACKAGE = 'qiniu'
NAME = 'qiniu'
DESCRIPTION = 'Qiniu Resource Storage SDK for Python 2.X.'
LONG_DESCRIPTION = 'see:\nhttps://github.com/qiniu/python-sdk\n'
AUTHOR = 'Shanghai Qiniu Information Technologies Co., Ltd.'
AUTHOR_EMAIL = 'support@qiniu.com'
MAINTAINER_EMAIL = 'fengliyuan@qiniu.com'
URL = 'https://github.com/qiniu/python-sdk'
VERSION = __import__(PACKAGE).__version__

setup(
	name=NAME,
	version=VERSION,
	description=DESCRIPTION,
	long_description=LONG_DESCRIPTION,
	author=AUTHOR,
	author_email=AUTHOR_EMAIL,
	maintainer_email=MAINTAINER_EMAIL,
	license='MIT',
	url=URL,
	packages=['qiniu', 'qiniu.test', 'qiniu.auth', 'qiniu.rs', 'qiniu.rs.test'],
	platforms='any',
	classifiers=[
		'Intended Audience :: Developers',
		'License :: OSI Approved :: MIT License',
		'Operating System :: OS Independent',
		'Programming Language :: Python',
		'Programming Language :: Python :: 2',
		'Programming Language :: Python :: 2.6',
		'Programming Language :: Python :: 2.7',
		'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
		'Topic :: Software Development :: Libraries :: Python Modules'
	],
	test_suite = 'nose.collector'
)


import torch
import torch.nn as nn
import numpy as np

class InferenceRunner:

    def __init__(self, model, prediction_mode='simple', scaler_x=None, scaler_y=None):
        self.model = model
        self.prediction_mode = prediction_mode
        self.last_prediction = None
        self.scaler_x = scaler_x
        self.scaler_y = scaler_y

    def predict_rec(self, features):
        pass

    def predict_simple(self, features):
        features = np.stack([self.scaler_x.transform(features.cpu()[i]) for i in range(features.shape[0])], 0).astype(np.float32)
        features = torch.from_numpy(features).to('cuda' if torch.cuda.is_available() else 'cpu')
        preds = self.model(features)
        if self.scaler_y:
            preds = self.scaler_y.inverse_transform(preds.detach().cpu().numpy())
        return preds

    def __call__(self, features):
        if self.prediction_mode == 'recurrent':
            return self.predict_rec(features)
        elif self.prediction_mode == 'simple':
            return self.predict_simple(features)
        else:
            raise ValueError('Invalid prediction mode. Expected recurrent or simple')

class LSTMPredictor(nn.Module):

    def __init__(self, num_features, hidden_dim, tagset_size=6):
        super(LSTMPredictor, self).__init__()
        self.num_layers = 1
        self.hidden_dim = hidden_dim
        self.lstm = nn.LSTM(num_features, hidden_dim, batch_first=True)
        self.sim_prediction = nn.Linear(hidden_dim, tagset_size)

    def forward(self, features):
        h0 = torch.zeros(1, features.size(0), self.hidden_dim).to('cuda' if torch.cuda.is_available() else 'cpu')
        c0 = torch.zeros(1, features.size(0), self.hidden_dim).to('cuda' if torch.cuda.is_available() else 'cpu')
        seq, (hidden_state, cell_state) = self.lstm(features, (h0, c0))
        hidden_state = hidden_state.detach()
        predictions = self.sim_prediction(hidden_state.squeeze())
        predictions = predictions + features[:, -1].squeeze()
        return predictions

class Conv1dPredictor(nn.Module):

    def __init__(self, num_features, target_size=6):
        super(Conv1dPredictor, self).__init__()
        self.conv1 = nn.Conv1d(num_features, 256, 2, stride=1)
        self.conv2 = nn.Conv1d(256, 256, 2, stride=1)
        self.conv3 = nn.Conv1d(256, 256, 2, stride=1)
        self.conv4 = nn.Conv1d(256, target_size, 2, stride=1)
        self.preds = nn.Linear(21 * target_size, target_size)

        self.bn1 = nn.BatchNorm1d(256)
        self.bn2 = nn.BatchNorm1d(256)
        self.bn3 = nn.BatchNorm1d(256)
        self.bn4 = nn.BatchNorm1d(target_size)

        self.pool1 = nn.MaxPool1d(1)
        self.pool2 = nn.MaxPool1d(1)
        self.pool3 = nn.MaxPool1d(1)
        self.pool4 = nn.MaxPool1d(1)

    def forward(self, features):
        x = self.conv1(features.permute(0, 2, 1))
        x = self.bn1(x)
        x = self.pool1(x)
        x = self.conv2(x)
        x = self.bn2(x)
        x = self.pool2(x)
        x = self.conv3(x)
        x = self.bn3(x)
        x = self.pool3(x)
        x = self.conv4(x)
        x = self.bn4(x)
        x = self.pool4(x)
        predictions = self.preds(x.view(x.size(0), -1))
        return predictions

class MLPPredictor(nn.Module):

    def __init__(self, num_features, seq_len, hidden_dim, target_size=6):
        super(MLPPredictor, self).__init__()
        self.num_layers = 1
        self.hidden_dim = hidden_dim
        self.num_features = num_features
        self.seq_len = seq_len
        self.dense1 = nn.Linear(seq_len * num_features, hidden_dim)
        self.bn1 = nn.BatchNorm1d(hidden_dim)
        self.dense2 = nn.Linear(hidden_dim, hidden_dim // 2)
        self.bn2 = nn.BatchNorm1d(hidden_dim // 2)
        self.dense3 = nn.Linear(hidden_dim // 2, target_size)
        self.target_size = target_size
        self.relu = nn.ReLU()

    def forward(self, features):
        x = features.view(-1, self.seq_len * self.num_features)
        x = self.dense1(x)
        # x = self.bn1(x)
        x = self.dense2(x)
        # x = self.bn2(x)
        predictions = self.dense3(x)
        predictions = predictions.view(-1, self.target_size)
        return predictions


"""Friend request model."""

# Django
from django.db import models

# Utilities
from app.utils.models import FbModel

class FriendRequest(FbModel):
    """Friend request."""

    requesting_user = models.ForeignKey('users.User', on_delete=models.CASCADE)

    requested_user = models.ForeignKey(
        'users.User',
        on_delete=models.CASCADE,
        related_name='requested_user')

    accepted = models.BooleanField(default=False)

    def __str__(self):
        """Return requesting_user and requested_user."""
        return 'from @{} to @{}'.format(
            self.requesting_user.username,
            self.requested_user.username)


############################################################################################################################
#                                                                                #
#                                           Fun project                                                                    #
#                                           created by:                                                                    #
#                                                 P.Sri Ranga Sai                                                          #
#                   Thank you for looking at my code. :)                                                                   #
############################################################################################################################

# Imports
import csv

# Constants
rank = 0
# opening our " data to be sorted " file
file = open('sample_data.csv','r')

# Reading as csv
csv_reader = csv.reader(file,delimiter=',')

# converting csv_object to list object
csv_list = list(csv_reader)

# opening new file where the sorted results are stored
file1 = open('sorted_sample_data.csv','w+')

# Inserting 1st row which contains names of columns along with rank ahead of the whole row
rows = csv_list[0]
file1.write("Rank,")
for i in rows:
    if i == rows[-1]:
        file1.write(i)
    else:
        file1.write(i + ',')
file1.write('\n')

# deducing 1st row and reverse sorting the csv_list
csv_list = csv_list[1:]
csv_list.sort(key= lambda x: x[-2], reverse=True)

# inserting each sorted row along with rank into sorted file
for i in csv_list:
    if rank == 0:
        prev_i = csv_list[csv_list.index(i)]    # previous data
        rank += 1
    else:
        prev_i = csv_list[csv_list.index(i) - 1] # previous data  used to check whether two persons got same marks or not
    if prev_i[-2] == i[-2]:           #if previous score is equal to current score rank wont change
        rank = rank
    else:
        rank += 1                     # else rank is incremented by 1
    file1.write(f'{rank},')           # writing rank to file
    for j in i:                       # writing remaining rows to file
        if j == i[-1] :
            file1.write(j)
        else:
            file1.write(j + ',')
    file1.write('\n')

# closing the opened files
file.close()
file1.close()


import ast
import os
import re
import sys

from setuptools import setup, __version__ as setuptools_version

def readme(name='README.rst'):
    try:
        with open(name) as f:
            rst = f.read()
        return re.sub(
            r'(^|\n).. include::\s*([^\n]+)($|\n)',
            lambda m: m.group(1) + (readme(m.group(2)) or '') + m.group(3),
            rst
        )
    except (IOError, OSError):
        return

def get_version():
    module_path = os.path.join(os.path.dirname(__file__), 'nirum_http.py')
    module_file = open(module_path)
    try:
        module_code = module_file.read()
    finally:
        module_file.close()
    tree = ast.parse(module_code, module_path)
    for node in ast.iter_child_nodes(tree):
        if not isinstance(node, ast.Assign) or len(node.targets) != 1:
            continue
        target, = node.targets
        if isinstance(target, ast.Name) and target.id == '__version__':
            value = node.value
            if isinstance(value, ast.Str):
                return value.s
            raise ValueError('__version__ is not defined as a string literal')
    raise ValueError('could not find __version__')

setup_requires = []
install_requires = [
    'nirum >= 0.6.0',
    'requests >= 2.11.1',
    'six',
]
tests_require = [
    'flake8-import-order >= 0.12, < 1.0',
    'flake8-import-order-spoqa >= 1.0.0, < 2.0.0',
    # flake8 does not yet support pycodestyle 2.4.0.
    # Can be remove after this issue is fixed: https://gitlab.com/pycqa/flake8/issues/415
    'pycodestyle >= 2.0, < 2.4.0',
    'pytest >= 3.1.2, < 4.0.0',
    'pytest-flake8 >= 0.9.0, < 1.0.0',
    'requests-mock >= 1.3.0, < 1.4.0',
]
extras_require = {
    'tests': tests_require,
}
below35_requires = [
    'typing',
]

if 'bdist_wheel' not in sys.argv and sys.version_info < (3, 5):
    install_requires.extend(below35_requires)

if tuple(map(int, setuptools_version.split('.'))) < (17, 1):
    setup_requires = ['setuptools >= 17.1']
    extras_require.update({":python_version=='3.4'": below35_requires})
    extras_require.update({":python_version=='2.7'": below35_requires})
else:
    extras_require.update({":python_version<'3.5'": below35_requires})

setup(
    name='nirum-http',
    version=get_version(),
    description='Nirum HTTP transport for Python',
    long_description=readme(),
    url='https://github.com/nirum-lang/nirum-python-http',
    bugtrack_url='https://github.com/nirum-lang/nirum/issues',
    author='Kang Hyojun',
    author_email='iam.kanghyojun' '@' 'gmail.com',
    license='MIT license',
    py_modules=['nirum_http'],
    install_requires=install_requires,
    setup_requires=setup_requires,
    extras_require=extras_require
)


from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import redirect, render
from django.urls import reverse
from django.views.generic import DetailView, ListView
from django.views.generic.edit import CreateView, UpdateView

from dfirtrack_main.forms import ReportitemForm
from dfirtrack_main.logger.default_logger import debug_logger
from dfirtrack_main.models import Notestatus, Reportitem

class ReportitemList(LoginRequiredMixin, ListView):
    login_url = '/login'
    model = Reportitem
    template_name = 'dfirtrack_main/reportitem/reportitem_list.html'
    context_object_name = 'reportitem_list'

    def get_queryset(self):
        debug_logger(str(self.request.user), " REPORTITEM_LIST_ENTERED")
        return Reportitem.objects.order_by('reportitem_id')

class ReportitemDetail(LoginRequiredMixin, DetailView):
    login_url = '/login'
    model = Reportitem
    template_name = 'dfirtrack_main/reportitem/reportitem_detail.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        reportitem = self.object
        reportitem.logger(str(self.request.user), " REPORTITEM_DETAIL_ENTERED")
        return context

class ReportitemCreate(LoginRequiredMixin, CreateView):
    login_url = '/login'
    model = Reportitem
    form_class = ReportitemForm
    template_name = 'dfirtrack_main/reportitem/reportitem_generic_form.html'

    def get(self, request, *args, **kwargs):
        # get id of first status objects sorted by name
        notestatus = Notestatus.objects.order_by('notestatus_name')[0].notestatus_id

        if 'system' in request.GET:
            system = request.GET['system']
            form = self.form_class(
                initial={
                    'notestatus': notestatus,
                    'system': system,
                }
            )
        else:
            form = self.form_class(
                initial={
                    'notestatus': notestatus,
                }
            )
        debug_logger(str(request.user), " REPORTITEM_ADD_ENTERED")
        return render(
            request,
            self.template_name,
            {
                'form': form,
                'title': 'Add',
            },
        )

    def post(self, request, *args, **kwargs):
        form = self.form_class(request.POST)
        if form.is_valid():
            reportitem = form.save(commit=False)
            reportitem.reportitem_created_by_user_id = request.user
            reportitem.reportitem_modified_by_user_id = request.user
            reportitem.save()
            form.save_m2m()
            reportitem.logger(str(request.user), " REPORTITEM_ADD_EXECUTED")
            messages.success(request, 'Reportitem added')
            if 'documentation' in request.GET:
                return redirect(
                    reverse('documentation_list')
                    + f'#reportitem_id_{reportitem.reportitem_id}'
                )
            else:
                return redirect(
                    reverse('system_detail', args=(reportitem.system.system_id,))
                )
        else:
            return render(
                request,
                self.template_name,
                {
                    'form': form,
                    'title': 'Add',
                },
            )

class ReportitemUpdate(LoginRequiredMixin, UpdateView):
    login_url = '/login'
    model = Reportitem
    form_class = ReportitemForm
    template_name = 'dfirtrack_main/reportitem/reportitem_generic_form.html'

    def get(self, request, *args, **kwargs):
        reportitem = self.get_object()
        form = self.form_class(instance=reportitem)
        reportitem.logger(str(request.user), " REPORTITEM_EDIT_ENTERED")
        return render(
            request,
            self.template_name,
            {
                'form': form,
                'title': 'Edit',
            },
        )

    def post(self, request, *args, **kwargs):
        reportitem = self.get_object()
        form = self.form_class(request.POST, instance=reportitem)
        if form.is_valid():
            reportitem = form.save(commit=False)
            reportitem.reportitem_modified_by_user_id = request.user
            reportitem.save()
            form.save_m2m()
            reportitem.logger(str(request.user), " REPORTITEM_EDIT_EXECUTED")
            messages.success(request, 'Reportitem edited')
            if 'documentation' in request.GET:
                return redirect(
                    reverse('documentation_list')
                    + f'#reportitem_id_{reportitem.reportitem_id}'
                )
            else:
                return redirect(
                    reverse('system_detail', args=(reportitem.system.system_id,))
                )
        else:
            return render(
                request,
                self.template_name,
                {
                    'form': form,
                    'title': 'Edit',
                },
            )

class ReportitemSetUser(LoginRequiredMixin, UpdateView):
    login_url = '/login'
    model = Reportitem

    def get(self, request, *args, **kwargs):
        reportitem = self.get_object()
        reportitem.reportitem_assigned_to_user_id = request.user
        reportitem.save()
        reportitem.logger(str(request.user), " REPORTITEM_SET_USER_EXECUTED")
        messages.success(request, 'Reportitem assigned to you')

        # redirect
        if 'documentation' in request.GET:
            return redirect(
                reverse('documentation_list')
                + f'#reportitem_id_{reportitem.reportitem_id}'
            )
        else:
            return redirect(
                reverse('reportitem_detail', args=(reportitem.reportitem_id,))
            )

class ReportitemUnsetUser(LoginRequiredMixin, UpdateView):
    login_url = '/login'
    model = Reportitem

    def get(self, request, *args, **kwargs):
        reportitem = self.get_object()
        reportitem.reportitem_assigned_to_user_id = None
        reportitem.save()
        reportitem.logger(str(request.user), " REPORTITEM_UNSET_USER_EXECUTED")
        messages.warning(request, 'User assignment for reportitem deleted')

        # redirect
        if 'documentation' in request.GET:
            return redirect(
                reverse('documentation_list')
                + f'#reportitem_id_{reportitem.reportitem_id}'
            )
        else:
            return redirect(
                reverse('reportitem_detail', args=(reportitem.reportitem_id,))
            )


import numpy as np
import pandas as pd
import random

class Em():
    """
    2^3 cell prob

    p1s = [p(--1), p(1--|--1), p(1--|--0), p(-1-|1-1), p(-1-|1-0), p(-1-|0-1), p(-1-|0-0)]
    p0s = 1 - p1s

    weights
    Kw1s = [p(-1-|1-1), p(-1-|1-0), p(-1-|0-1), p(-1-|0-0)]
    Kw0s = 1 - Kw1s

    Kw1s.T @ K = pkeK1
    Kw0s.T @ K = pkeK0

    Lw1s = [p(1--|-11), p(1--|-10), p(1--|-01), p(1--|-00)]
    Lw0s = 1 - Lw1s

    Lw1s.T @ L = pkeL1
    Lw0s.T @ L = pkeL0
    """
    def __init__(self):

        #data
        self.H = [293, 4, 176, 3, 23, 2, 197, 17]
        self.K = [100, 5, 82, 6]
        self.L = [90, 5, 150, 10]

        #prime index
        self.pkH = ['111','110','101','100','011','010','001','000']

        self.pkK = ['11','10','01','00']
        self.pkeK1 = ['111','110','011','010']
        self.pkeK0 = ['101','100','001','000']

        self.pkL = ['11','10','01','00']
        self.pkeL1 = ['111','110','101','100']
        self.pkeL0 = ['011','010','001','000']

        #record prob by time
        self.p = []

        #initial weights
        self.Kw1s = np.array([0.5,0.5,0.5,0.5])
        self.Kw0s = 1 - self.Kw1s
        self.Lw1s = np.array([0.5,0.5,0.5,0.5])
        self.Lw0s = 1 - self.Lw1s

        #df for H K L
        self.df_H = pd.DataFrame(self.H, index = self.pkH, columns = ['H'])
        self.df_K = pd.DataFrame(self.K, index = self.pkK)
        self.df_L = pd.DataFrame(self.L, index = self.pkL)

    #methods
    def make_current_estimated_df(self):
        print("system>>> merging data frame")
        self.df_eK = pd.DataFrame((self.df_K.iloc[:,0]*self.Kw1s).tolist(), index = self.pkeK1).append(pd.DataFrame((self.df_K.iloc[:,0]*self.Kw0s).tolist(), index = self.pkeK0))
        self.df_eK.columns = ['eK']
        self.df_eL = pd.DataFrame((self.df_L.iloc[:,0]*self.Lw1s).tolist(), index = self.pkeL1).append(pd.DataFrame((self.df_L.iloc[:,0]*self.Lw0s).tolist(), index = self.pkeL0))
        self.df_eL.columns = ['eL']
        self.df_curr = self.df_H.join(self.df_eK).join(self.df_eL)

    def update_w1s(self):
        print("system>>> updating weight")
        self.Kw1s[0] = sum(self.df_curr.loc['111',:])/(sum(self.df_curr.loc['111',:])+sum(self.df_curr.loc['101',:]))
        self.Kw1s[1] = sum(self.df_curr.loc['110',:])/(sum(self.df_curr.loc['110',:])+sum(self.df_curr.loc['100',:]))
        self.Kw1s[2] = sum(self.df_curr.loc['011',:])/(sum(self.df_curr.loc['011',:])+sum(self.df_curr.loc['001',:]))
        self.Kw1s[3] = sum(self.df_curr.loc['010',:])/(sum(self.df_curr.loc['010',:])+sum(self.df_curr.loc['000',:]))
        self.Kw0s = 1 - self.Kw1s

        self.Lw1s[0] = sum(self.df_curr.loc['111',:])/(sum(self.df_curr.loc['111',:])+sum(self.df_curr.loc['011',:]))
        self.Lw1s[1] = sum(self.df_curr.loc['110',:])/(sum(self.df_curr.loc['110',:])+sum(self.df_curr.loc['010',:]))
        self.Lw1s[2] = sum(self.df_curr.loc['101',:])/(sum(self.df_curr.loc['101',:])+sum(self.df_curr.loc['001',:]))
        self.Lw1s[3] = sum(self.df_curr.loc['100',:])/(sum(self.df_curr.loc['100',:])+sum(self.df_curr.loc['000',:]))
        self.Lw0s = 1 - self.Lw1s

    def get_curr_p(self):
        p_list = []
        p_index = ['p(--1)', 'p(1--|--1)', 'p(1--|--0)', 'p(-1-|1-1)', 'p(-1-|1-0)', 'p(-1-|0-1)', 'p(-1-|0-0)']
        p_list.append(self.df_curr.iloc[[0,2,4,6],:].sum().sum()/self.df_curr.sum().sum())
        p_list.append(self.df_curr.iloc[[0,2],:].sum().sum()/self.df_curr.iloc[[0,2,4,6],:].sum().sum())
        p_list.append(self.df_curr.iloc[[1,3],:].sum().sum()/self.df_curr.iloc[[1,3,5,7],:].sum().sum())
        p_list.append(self.df_curr.iloc[[0],:].sum().sum()/self.df_curr.iloc[[0,2],:].sum().sum())
        p_list.append(self.df_curr.iloc[[1],:].sum().sum()/self.df_curr.iloc[[1,3],:].sum().sum())
        p_list.append(self.df_curr.iloc[[4],:].sum().sum()/self.df_curr.iloc[[4,6],:].sum().sum())
        p_list.append(self.df_curr.iloc[[5],:].sum().sum()/self.df_curr.iloc[[5,7],:].sum().sum())
        return pd.DataFrame(p_list, index = p_index)

    def get_curr_p0(self):
        p_list = []
        p_index = ['p(111)', 'p(110)', 'p(101)', 'p(100)', 'p(011)', 'p(010)', 'p(001)','p(000)']
        p_list.append(self.df_curr.iloc[0,:].sum().sum()/self.df_curr.sum().sum())
        p_list.append(self.df_curr.iloc[1,:].sum().sum()/self.df_curr.sum().sum())
        p_list.append(self.df_curr.iloc[2,:].sum().sum()/self.df_curr.sum().sum())
        p_list.append(self.df_curr.iloc[3,:].sum().sum()/self.df_curr.sum().sum())
        p_list.append(self.df_curr.iloc[4,:].sum().sum()/self.df_curr.sum().sum())
        p_list.append(self.df_curr.iloc[5,:].sum().sum()/self.df_curr.sum().sum())
        p_list.append(self.df_curr.iloc[6,:].sum().sum()/self.df_curr.sum().sum())
        p_list.append(self.df_curr.iloc[7,:].sum().sum()/self.df_curr.sum().sum())
        return pd.DataFrame(p_list, index = p_index)

    def test_HKL(self):
        print("system>>> back checking start...")
        print("system>>> p0 = [0.1,0.4,0.2,0.05,0.02,0.1,0.12,0.01]")
        self.H = np.random.multinomial(2000, [0.1,0.4,0.2,0.05,0.02,0.1,0.12,0.01],size=1)[0]
        self.K = np.random.multinomial(2000, [0.1,0.4,0.2,0.05,0.02,0.1,0.12,0.01],size=1)[0]
        self.L = np.random.multinomial(2000, [0.1,0.4,0.2,0.05,0.02,0.1,0.12,0.01],size=1)[0]

        self.K = [self.K[0]+self.K[2],self.K[1]+self.K[3],self.K[4]+self.K[6],self.K[5]+self.K[7]]
        self.L = [self.L[0]+self.L[4],self.L[1]+self.L[5],self.L[2]+self.L[6],self.L[3]+self.L[7]]
        self.df_H = pd.DataFrame(self.H, index = self.pkH, columns = ['H'])
        self.df_K = pd.DataFrame(self.K, index = self.pkK)
        self.df_L = pd.DataFrame(self.L, index = self.pkL)

    def run(self, iter):
        self.make_current_estimated_df()
        print("system>>> ready...")
        print(self.df_curr)
        print("system>>> run...")

        for i in range(iter):
            # print("system>>> {}th trial".format(i))
            old_Kw1s = self.Kw1s.copy()
            old_Lw1s = self.Lw1s.copy()
            self.update_w1s()
            self.make_current_estimated_df()
            self.p.append([i,self.get_curr_p()])
            # print(self.df_curr.sum().sum())
            # print("system>>> abs change : K:{}, L:{}".format(sum(abs(old_Kw1s-self.Kw1s)),sum(abs(old_Lw1s-self.Lw1s))))

        print("system>>> finished")

if __name__ == "__main__":
    em = Em()
    # em.run(10)
    # print(em.df_curr)
    # print(em.p[4])
    # print(em.get_curr_p0)

#i python code
# import os
# os.chdir("C:\\EM_mid")
# from EM_module import Em
# em = Em()

#simulation result
#set p = [0.1,0.4,0.2,0.05,0.02,0.1,0.12,0.01]
#iter = 100
# p(111)	0.099057
# p(110)	0.390393
# p(101)	0.208457
# p(100)	0.044688
# p(011)	0.019553
# p(010)	0.107853
# p(001)	0.120599
# p(000)	0.009399


import streamlit as st
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
import pandas as pd

st.set_page_config(page_title="Solution Data", page_icon='\U00002705')

st.header(':green[Solution Data] ✅')
st.markdown(
    """
ในแง่ Structure ตัว Solution จะเหมือนกับ Exercises เลย ต่างกันที่
- เพิ่มความสามารถในการ Upload VDO Solution ได้
- ถ้าเป็นไปได้อยากให้มีการ Upload ภาพได้หลายภาพ แล้วกดเล่นทีละภาพ Step-by-step แบบ Mini Animation
"""
)

st.divider()

tab = st.tabs(['Soln 1: Typical Solution', 'Soln 2: VDO Solution'])

with tab[0]:
    st.markdown(
        '''
Exercises 1: ภาพด้านล่างแสดง Surface Plot ของ Function พร้อม Contour Line ของมัน
'''
    )
    st.write('จงหาว่าฟังก์ชันนี้คืออะไร')

    col1 = st.columns(2)
    with col1[0]:
        X = np.linspace(-50,50,200)
        Y = np.linspace(-50,50,200)
        y,x = np.meshgrid(X,Y)
        z = x**2 - y**2
        surface = go.Surface(x=x,y=y,z=z)
        fig = go.Figure(surface)
        fig.update_traces(contours_z=dict(show=True, usecolormap=True,
                                  highlightcolor="limegreen", project_z=True))

        fig.update_layout(template='plotly_dark')
        st.plotly_chart(fig, use_container_width=True)
        st.caption('สามารถ Interact กับ Plot นี้ได้')

    with col1[1]:
        options = [
        "1. $ f (x,y) = x^2 + y^2 $",
        "2. $ f (x,y) = x^2 - y^2 $",
        "3. $ f (x,y) = 2x - 2y $",
        "4. $ f (x,y) = 2x + 2y $"
        ]
        ans1 = st.radio('ตัวเลือก',options)
        st.write(":orange[คำตอบของฉันคือ: ]",ans1)
        st.success('คำตอบที่ถูกต้องคือ ข้อ 8.')
        st.write('เพราะว่า')
        st.write('bla bla bla')
        st.write('.')
        st.write('.')

with tab[1]:
    st.write('Exercises 2: จากข้อมูลที่ให้มา จงเติมตัวเลขลงในตารางให้ถูกต้อง')
    col2 = st.columns(2)
    with col2[0]:
        st.image('table_question.png')
    with col2[1]:
        df = pd.read_csv('cattles.csv')
        st.dataframe(df)
        st.success('พี่ Meth จะมาเฉลยให้ด้านล่างเลยจ้าาา')
        st.video('meth_soln.mp4')

st.divider()


from MNL import *
import numpy as np
import matplotlib.pyplot as plt

#Defining kronecker delta function
def kro(a, b):
    return 1 if a == b else 0

# Defining the matrix in the problem
def matri6(x, y, m=0.2, dim=20):
    i, j = x // dim, x % dim
    a, b = y // dim, y % dim

    mat = 0.5 * (
        (kro(i + 1, a) * kro(j, b))
        + (kro(i - 1, a) * kro(j, b))
        - (4 * kro(i, a) * kro(j, b))
        + (kro(i, a) * kro(j + 1, b))
        + (kro(i, a) * kro(j - 1, b))
    ) + (m**2) * kro(i, a) * kro(j, b)
    return mat

#Multiply function with vector
def func_multi(f, x):
    n = len(x)
    prod = np.zeros(n)
    for i in range(n):
        prod[i] = sum([f(i, j) * x[j] for j in range(n)])

    return prod

# Function of Conjugate Gradient
def con_grad_of(func, b, tol):
    n = len(b)
    count = 0
    x = np.zeros(n)
    r = b - func_multi(func, x)
    d = np.copy(r)
    residue = [np.linalg.norm(r)]
    iterations = [count]
    for i in range(n):
        Ad = func_multi(func, d)
        rprevdot = np.dot(r, r)
        alpha = rprevdot / np.dot(d, Ad)
        x += alpha * d
        r -= alpha * Ad
        rnextdot = np.dot(r, r)
        count += 1
        iterations.append(count)
        residue.append(np.linalg.norm(r))

        if np.linalg.norm(r) < tol:
            return x, iterations, residue

        else:
            beta = rnextdot / rprevdot
            d = r + beta * d
            rprevdot = rnextdot

def Inv_mat(A, tol, N=400):
    inv = []
    B = np.identity(N)
    for i in range(3):
        x, iter, residue = con_grad_of(A, B[:, i], tol)
        inv.append(x)

    return np.array(inv).T, iter, residue

x, iter, res = Inv_mat(matri6, 1e-6)
# Printing matrix
print("Inverse of matrix is : ")
print(x)

# Generating plot of convergence
plt.plot(iter, res)
plt.title('Residue vs Iteration plot')
plt.xlabel("Iterations")
plt.ylabel("Residue")
plt.show()


class Solution:
    def checkInclusion(self, s1: str, s2: str) -> bool:

        if len(s1) > len(s2):
            return False

        my_trie = Trie()
        my_trie.build_trie(s2)
        my_trie.build_reverse(s2)

        def getperms(temp, j, perms):
            if j == len(temp):
                perm = ''.join(temp)
                perms.append(perm)
                return

            for i in range(j, len(temp)):
                temp[i], temp[j] = temp[j], temp[i]
                getperms(temp, j+1, perms)
                temp[i], temp[j] = temp[j], temp[i]

        perms = []
        temp = list(s1)
        getperms(temp, 0, perms)
        for perm in perms:
            if my_trie.contains(perm):
                return True

        return False

class Trie:
    def __init__(self):
        self.root = {}
        self.end = '*'

    def build_trie(self, string2):
        for i in range(len(string2)):
            node = self.root
            for j in range(i, len(string2)):
                char = string2[j]
                if char not in node:
                    node[char] = {}
                node = node[char]
            node[self.end] = True

    def build_reverse(self, string2):
        newstr = string2[::-1]
        for i in range(len(newstr)):
            node = self.root
            for j in range(i, len(newstr)):
                char = newstr[j]
                if char not in node:
                    node[char] = {}
                node = node[char]
            node[self.end] = True

    def contains(self, string1):
        node = self.root
        for char in string1:
            if char in node:
                node = node[char]
            else:
                return False
        return True

s1, s2 = "rvwrk", "lznomzggwrvrkxecjaq" # wrvrk or krvrw

s1, s2 = "abcdxabcde", "abcdeabcdx"

"adc"
"dcda"

print(Solution().checkInclusion(s1,s2))


import cv2
import argparse
from os.path import join
import numpy as np
np.set_printoptions(suppress=True, precision=4)
import itertools
import matplotlib.pyplot as plt
import sys
import json

def read_projector_calibration_images(directory, camera):
    base_im = camera.load_image_grayscale(join(directory, 'base.png'))
    x_ims = [camera.load_image_grayscale(join(directory, 'x_im%d.png' % (i, ))) for i in range(15)]
    y_ims = [camera.load_image_grayscale(join(directory, 'y_im%d.png' % (i, ))) for i in range(8)]

    return base_im, x_ims, y_ims

def find_line(base_im, im, threshhold=0.1, dist=12):
    base_im = base_im.copy().astype(np.float32)
    im = im.copy().astype(np.float32)

    base_im /= np.mean(base_im)
    im /= np.mean(im)

    r, t = find_line_binary_image(im - base_im > threshhold)

    if r < 0:
        r *= -1
        t += np.pi
    wt = np.maximum(im - base_im, 0)

    rs = np.sum(np.indices(wt.shape) * np.array([[[np.sin(t)]], [[np.cos(t)]]]), 0)
    wt[np.abs(rs - r) > dist] = 0
    wt -= 0.05
    wt[wt < 0] = 0

    inds = np.stack(np.where(wt), -1).astype(np.float32)
    wts = np.expand_dims(wt[wt>0], 1)

    mean = np.sum(wts * inds, 0) / np.sum(wts)
    inds -= mean

    vals, vec = (np.linalg.eig(np.matmul((wts*inds).T, (wts*inds))))
    big = np.argmax(np.abs(vals))
    small = np.argmin(np.abs(vals))

    rho = np.abs(np.dot(mean, vec[:, small]))
    theta = np.arctan2(vec[0, small], vec[1, small])

    while theta > t + np.pi / 2:
        theta -= np.pi
        rho *= -1
    while theta < t - np.pi / 2:
        theta += np.pi
        rho *= -1
    if np.sign(rho) != np.sign(r):
        rho *= -1

    return rho, theta

def find_line_binary_image(im):
    rho, theta = cv2.HoughLines(im.astype(np.uint8), 1, np.pi / 180, 100, lines=1)[0, 0]
    return rho, theta

def find_line_intersection(line1_rt, line2_rt):
    r1, t1 = line1_rt
    r2, t2 = line2_rt

    arr = np.array([
        [np.cos(t1), np.sin(t1)],
        [np.cos(t2), np.sin(t2)]])
    arr_inv = np.linalg.inv(arr)
    return np.matmul(arr_inv, np.array([r1, r2]))

def get_intersection_points(proj_im_dir, camera, shape=(15, 8)):
    base_im, x_ims, y_ims = read_projector_calibration_images(proj_im_dir, camera)

    x_lines = [find_line(base_im, im) for im in x_ims]
    y_lines = [find_line(base_im, im) for im in y_ims]

    cam_points = np.zeros([shape[0], shape[1], 2])
    proj_points = np.zeros([shape[0], shape[1], 2])

    for x in range(shape[0]):
        for y in range(shape[1]):
            proj_points[x, y] = [1366 * (x+1) / 16.0, 768 * (y+1) / 9.0]
            cam_points[x, y] = find_line_intersection(x_lines[x], y_lines[y])

    return proj_points, cam_points

if __name__ == '__main__':
    points = get_intersection_points('calibration_images/proj_calib_2', 'kinect_calib.json')
    plt.scatter(points[..., 0], points[..., 1])
    plt.show()


# exercice 9:
# les diviseurs d'un nombre

def diviseurs(n):
    l = []

    for i in range(1,n+1):
        if i not in l and n%i==0:
            l.append(i)

    return l

n = int(input("saisissez une valeur pour n : "))

print(diviseurs(n))


import os
import argparse
import shutil
import random
import numpy as np
from itertools import chain
from joblib import load as skload

from mignn.dataset import PathLightDataset
from mignn.processing.scalers import ScalersManager
from mignn.processing import ScalerTransform, SignalEncoder

import torch_geometric.transforms as GeoT

from joblib import dump as skdump

import tqdm
from multiprocessing.pool import ThreadPool

from utils import scale_subset, merge_by_chunk

import config as MIGNNConf

def main():

    parser = argparse.ArgumentParser(description="Scaled data from config using GNN data")
    parser.add_argument('--data', type=str, help="GNN data", required=True)
    parser.add_argument('--output', type=str, help="output dataset folder", required=True)

    args = parser.parse_args()

    output_folder     = args.output
    input_containers  = args.data

    # Some MIGNN params
    dataset_percent   = MIGNNConf.DATASET_PERCENT

    os.makedirs(output_folder, exist_ok=True)
    dataset_path = f'{output_folder}/data'

    scalers_folder = f'{output_folder}/scalers'

    # intermediates datasets when fitting scalers
    output_temp_scaled = f'{output_folder}/intermediates/scaled'
    os.makedirs(output_temp_scaled, exist_ok=True)

    if not os.path.exists(scalers_folder):

        # save intermediate PathLightDataset
        # Then fusion PathLightDatasets into only one
        train_viewpoints = os.path.join(input_containers, 'train')
        test_viewpoints = os.path.join(input_containers, 'test')

        # initialize scalers from config using manager
        scalers = ScalersManager(config=MIGNNConf.NORMALIZERS)

        train_viewpoints_folder = [ os.path.join(train_viewpoints, v_name) for v_name in os.listdir(train_viewpoints) ]
        test_viewpoints_folder = [ os.path.join(test_viewpoints, v_name) for v_name in os.listdir(test_viewpoints) ]

        n_train_viewpoints = len(train_viewpoints_folder)
        print(f'[Processing] fit scalers on {n_train_viewpoints} viewpoints')

        if not scalers.enable_partial:
            raise AttributeError('[Unsupported] specified normalizers require the storage of all training data. \
                This can cause a memory surge.')

        # retrieve all containers filename
        train_viewpoints_files = list(chain(*[
                [ os.path.join(viewpoint_path, c_file) for c_file in os.listdir(viewpoint_path) ]
                for viewpoint_path in train_viewpoints_folder
            ]))

        n_graphs = 0

        for idx, c_dataset_path in enumerate(train_viewpoints_files):

            subset = PathLightDataset(root=c_dataset_path)

            # record data
            n_graphs += len(subset)

            # partial fit on train set when possible
            scalers.partial_fit(subset)

            # # reset train data list if necessary
            # if scalers.enable_partial:
            #     train_data = []

            # # always clear test data
            # test_data = []

            print(f'[Processing] fit scalers progress: {(idx + 1) / len(train_viewpoints_files) * 100.:.2f}%', end='\r')

        print(f'[Information] {n_graphs} graphs have been managed.')

        # For the moment we avoid total fit scalers and raise issue
        # ensure normalization using scalers with no partial fit method
        # if not scalers.enable_partial:

        #     c_dataset, _ = PathLightDataset.collate(train_data)
        #     scalers.fit(c_dataset)

        # save scalers
        os.makedirs(scalers_folder, exist_ok=True)

        skdump(scalers, f'{scalers_folder}/scalers.bin', compress=True)

    # reload scalers
    scalers_path = f'{scalers_folder}/scalers.bin'
    print('[Scaling] start preparing scaled data...')

    scalers = skload(scalers_path)
    transforms_list = [ScalerTransform(scalers)]

    if MIGNNConf.ENCODING_SIZE is not None:
        transforms_list.append(SignalEncoder(MIGNNConf.ENCODING_SIZE, MIGNNConf.ENCODING_MASK))

    applied_transforms = GeoT.Compose(transforms_list)

    # applied transformations over all intermediate path light dataset
    # avoid memory overhead
    if not os.path.exists(f'{dataset_path}/train'):

        output_scaled_temp_train = f'{output_temp_scaled}/train'
        output_scaled_temp_test = f'{output_temp_scaled}/test'
        os.makedirs(output_scaled_temp_train, exist_ok=True)
        os.makedirs(output_scaled_temp_test, exist_ok=True)

        train_dataset_path = f'{dataset_path}/train'
        test_dataset_path = f'{dataset_path}/test'
        os.makedirs(train_dataset_path, exist_ok=True)
        os.makedirs(test_dataset_path, exist_ok=True)

        subset_params = list(chain(*[
            [
                (
                    os.path.join(v_path, v_file),
                    scalers_path,
                    os.path.join(output_scaled_temp_train, os.path.split(v_path)[-1]),
                )
                for v_file in os.listdir(v_path)
            ]
            for v_path in train_viewpoints_folder
        ]))

        subset_params += list(chain(*[
            [
                (
                    os.path.join(v_path, v_file),
                    scalers_path,
                    os.path.join(output_scaled_temp_test, os.path.split(v_path)[-1]),
                )
                for v_file in os.listdir(v_path)
            ]
            for v_path in test_viewpoints_folder
        ]))

        # multi-process scale of dataset
        pool_obj_scaled = ThreadPool()
        pool_results = []

        for result in tqdm.tqdm(pool_obj_scaled.imap(scale_subset, subset_params), total=len(subset_params)):
            pool_results.append(result)

        # merge chunk for each viewpoin
        temp_viewpoints = [
                        (
                            os.path.join(output_scaled_temp_train, v_temp),
                            os.path.join(train_dataset_path, v_temp)
                        )
                        for v_temp in os.listdir(output_scaled_temp_train)
                    ]

        temp_viewpoints += [
                        (
                            os.path.join(output_scaled_temp_test, v_temp),
                            os.path.join(test_dataset_path, v_temp)
                        )
                        for v_temp in os.listdir(output_scaled_temp_test)
                    ]

        for idx, (temp_folder, dataset_folder) in enumerate(temp_viewpoints):

            # retrieve all scaled subset for this viewpoint
            scaled_subsets = [ os.path.join(temp_folder, v_file) for v_file in os.listdir(temp_folder) ]

            # do chunks
            merge_by_chunk(scaled_subsets, dataset_folder, applied_transforms)

            print(f'[Data processing] chunk progress: {(idx + 1) / len(temp_viewpoints) * 100.:.2f}%', end='\r')
        print('[Data processing] prepared chunked data done...')

    else:
        print(f'[Information] {dataset_path} already generated')

if __name__ == "__main__":
    main()


from json import loads
from queue import PriorityQueue
import sys
import os

'''
 --------------------------------------------------------------------------------
 @What it does?
    The following Python module is designed to ingest a CamFlow log and sort it
    in a way that all vertices are piled up at the top followed by all the edges
    sorted in a sequential order based on their 'relation_id'.

 @When should you use it?
    If there is a strict need in a module that requires a vertex to be present
    before the edge of that vertex is encountered in a log then this Python
    module can help you bring your CamFlow log in a desired shape.

 @authors
    Shahpar Khan, Mashal Abbas
 --------------------------------------------------------------------------------
'''

# Function that returns True if the current JSON object is a vertex --- False if it is an edge
def isVertex(obj):

    if (obj["type"] == "Entity") or (obj["type"] == "Activity"):
        return True

    return False

# Function that extract relation_ids from the JSON object
def extractRelationID(obj):

    try:
        return int(obj["annotations"]["relation_id"])
    except:
        print("Error in reading relation_id of the following JSON object:")
        print(obj)

    return None

# Function to dump all the edges in priority queue based on the relation_ids
def dumpPriorityQueue(pq, output_file):

    while not pq.empty():
        (_, line) = pq.get()
        output_file.write(line)

# Function to read log at a given path
def readWriteLog(input_log_path, output_log_name):

    # Opening files
    try:
        input_file = open(input_log_path, 'r')
    except:
        print("Error in opening file at path:", input_log_path)

    try:
        output_file = open(output_log_name, "a")
    except:
        print("Error in opening file with name:", output_log_name)

    # Initializing priority queue
    pq = PriorityQueue()

    line = input_file.readline()
    while line:
        try:
            obj = loads(line)
            if isVertex(obj):
                output_file.write(line)
            else:
                relation_id = extractRelationID(obj)
                pq.put((relation_id, line))
        except:
            print("Error in ingesting the following line:")
            print(line)

        line = input_file.readline()

    dumpPriorityQueue(pq, output_file)

    output_file.close()
    input_file.close()

def main(input_log_path, output_log_name):

    readWriteLog(input_log_path, output_log_name)
    print("Done...")

if __name__ == '__main__':
    try:
        if len(sys.argv) != 3:
            raise Exception("run python3 sortlog.py <input_log_path> <output_log_name>")
        else:
            print("Starting...")
            print("Input log path:", sys.argv[1])
            print("Output log name:", sys.argv[2])
            main(sys.argv[1], sys.argv[2])

    except KeyboardInterrupt:
        print("Exiting...")
        try:
            sys.exit(0)
        except SystemExit:
            os._exit(0)


import requests
import pprint
import time
import json
import logging

BASE_URL = "https://dadosabertos.camara.leg.br/api/v2"
N_ITENS_PER_PAGE = 100
MIN_YEAR = 1990

LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
logging.basicConfig(
    level=logging.INFO,
    format=LOG_FORMAT,
)

if __name__ == "__main__":
    params = {
        "itens": N_ITENS_PER_PAGE,
        "pagina": 1,
        "ordem": "DESC",
        "ordenarPor": "ano",
        "ano": 2023,
        "siglaTipo": "PL",
    }
    proposicoes = []
    in_keep_querying_prepositions = True

    logger = logging.getLogger(__name__)

    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.INFO)
    stream_handler.setFormatter(logging.Formatter(LOG_FORMAT))

    #logger.addHandler(stream_handler)
    logger.info("Iniciando script - get_propostas_camara.py")
    logger.info("BASIC PARAMS: " + json.dumps(params))

    while in_keep_querying_prepositions:

        logger.info(f"Fazendo requisição para o ano de {params['ano']}")

        params["pagina"] = 1
        while True:
            logger.info(f'Fazendo requisição para a página {params["pagina"]}')
            response = requests.get(f"{BASE_URL}/proposicoes", params=params)

            if response.status_code == 200:
                response_json = response.json()
                data = response_json["dados"]
                if not data:
                    logger.info(f"Não há mais proposições para o ano de {params['ano']}")
                    break
                proposicoes = proposicoes + data
            else:
                logger.error(f"Erro na requisição. Status code: {response.status_code}")
            params["pagina"] += 1

        params["ano"] -= 1

        if params["ano"] == MIN_YEAR:
            in_keep_querying_prepositions = False

        time.sleep(0.5)

    with open(f"./data/proposicoes.json", 'w') as file:
        json.dump(proposicoes, file, indent=4, ensure_ascii=False)


import sys
input=sys.stdin.readline
import heapq
M,N=map(int,input().split())
List=[list(map(int,input().split()))for _ in range(M)]
List_=[[float('inf') for _ in range(N)] for _ in range(M)]
q=[]
heapq.heapify(q)
dx,dy=[-1,1,0,0],[0,0,-1,1]
heapq.heappush(q,[List[0][0],0,0])
List_[0][0]=List[0][0]
while q:
    val,x,y=heapq.heappop(q)
    if List[x][y]==-1: continue
    if List_[x][y]!=val: continue
    for i in range(4):
        nx=x+dx[i]; ny=y+dy[i]
        if 0<=nx<M and 0<=ny<N and List[nx][ny]!=-1:
            if List_[nx][ny]>List_[x][y]+List[nx][ny]:
                List_[nx][ny]=List_[x][y]+List[nx][ny]
                heapq.heappush(q,[List_[nx][ny],nx,ny])

if List_[M-1][N-1]==float('inf'): print(-1)
else:
    print(List_[M-1][N-1])


import os
import logging
from IPython import get_ipython
import subprocess
from subprocess import Popen, PIPE
from pathlib import Path
import shlex

logger = logging.getLogger(__name__)

def _run_command(cmd):
    with Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, bufsize=1, universal_newlines=True) as p:
        while True:
            line = p.stdout.readline()
            if not line:
                break
            print(line, end="")
        exit_code = p.poll()
    return exit_code

def executeCmd(cmd, cwd='.', verbose=True, backend=None):
    currentDir = Path.cwd()
    os.chdir(cwd)
    try:
        if not verbose:
            ret = subprocess.run(cmd, shell=True, stdout=PIPE, stderr=PIPE)
        elif backend is None or backend == "ipython":
            get_ipython().system(cmd)
        else:
            _run_command(cmd)
    finally:
        os.chdir(currentDir)


from pathlib import Path

def linesFromFileToList(path:str):
    lines = []
    with open(Path(__file__).parent /
              path, 'r') as file:
        lines = file.readlines()
    return lines

def groupAndSumBetweenEmpty(lines: list):
    elvesCalories = [None] * len(lines)
    elfIdx = 0

    for line in lines:
        if line == "\n":
            elfIdx += 1
            continue
        else:
            line.strip("\n")
            if elvesCalories[elfIdx] == None:
                elvesCalories[elfIdx] = 0
            try:
                elvesCalories[elfIdx] += int(line)
            except ValueError:
                continue

    return elvesCalories

def findMaxValueIndex(list:list):
    index = 0

    for i in range(len(list)):
        currMax = list[index]
        currItem = list[i]

        if currItem != None and currItem > currMax:
            index = i

    return index

def main():
    lines = linesFromFileToList('./input.txt')
    elvesCalories = groupAndSumBetweenEmpty(lines)

    elvesCalories = [x for x in elvesCalories if x != None]

    elfWIthMostCalories = findMaxValueIndex(elvesCalories)

    print("Elf with most calories is elf number "+str(elfWIthMostCalories+1)+" with total of "  + str(elvesCalories[elfWIthMostCalories]) + " calories")

    elvesCalories.sort()
    elvesCalories.reverse()

    print(sum(elvesCalories[0:3]))

if __name__ == "__main__":
    main()


from nnsim.module import Module
from nnsim.ram import SRAM, RD, WR
from nnsim.channel import Channel

import numpy as np

class IFMapWeightsGLB(Module):
    def instantiate(self, ifmap_wr_chn, ifmap_rd_chn, weights_wr_chn, weights_rd_chn,\
            arr_y, ifmap_glb_depth, weights_glb_depth, \
            block_size, num_nonzero):
        self.ifmap_wr_chn = ifmap_wr_chn
        self.ifmap_rd_chn = ifmap_rd_chn
        self.weights_wr_chn = weights_wr_chn
        self.weights_rd_chn = weights_rd_chn
        self.arr_y = arr_y
        self.block_size = block_size
        self.num_nonzero = num_nonzero
        self.name = 'ifmap_weights_glb'

        self.in_chn = 0
        self.out_chn = 0

        self.stat_type = 'show'
        self.raw_stats = {'size' : (ifmap_glb_depth, num_nonzero*3), 'rd': 0, 'wr': 0}

        self.isram = SRAM(ifmap_glb_depth, num_nonzero*3, dtype=np.float16)
        self.ilast_read = Channel(3)
        self.ifmap_glb_depth = ifmap_glb_depth

        self.wsram = SRAM(weights_glb_depth, block_size, dtype=np.float16)
        self.wlast_read = Channel(1)
        # Channel depth of one here prevents SRAM reads from colliding
        # was having issues with a later read 'replacing' an earlier one
        # and thus getting the wrong data
        # having only one extant write on an SRAM at a time prevents this
        self.weights_glb_depth = weights_glb_depth

        # Channel to hold indices of weights that need to be sent
        # to NoC
        self.weights_to_send = Channel(3)

        self.image_size = (0, 0)
        self.filter_size = (0, 0)
        self.fmap_sets = 0
        self.fmap_per_iteration = 0

        self.curr_set = 0
        self.fmap_idx = 0
        self.iteration = 0
        self.iwr_done = False
        self.wwr_done = False

        # For managing convolution
        self.curr_x = 0
        self.curr_y = 0
        self.curr_chn = 0
        self.request_idx = 0
        self.send_idx = 0
        #self.curr_filt_x = 0
        #self.curr_filt_y = 0
        self.ifmap_done = False

        # for weights
        self.addr = 0
        self.base_addr = 0 # to store values from self.weights_to_send
        self.base_addr_wo_chn = -1 # to keep track of current position within 3x3 filter

        # invalid weights and inputs to use at the end to flush out last outputs
        self.weights_to_flush = 0
        self.inputs_to_flush = 0

        self.needed_addr = 0
        self.ready_to_output = False # ready to output a filter_size block of inputs
        self.curr_data = [0 for i in range(3*num_nonzero)]
        self.curr_weights = [0 for i in range(block_size)]
        self.data_idx = num_nonzero # block other operations while actively working through data
        # send one data point at a time (of num_nonzero)

    def configure(self, image_size, filter_size, in_chn, out_chn, fmap_per_iteration):
        self.wr_done = False

        self.image_size = image_size
        self.filter_size = filter_size
        self.in_chn = in_chn
        self.out_chn = out_chn
        self.fmap_per_iteration = fmap_per_iteration

        # For managing convolution
        self.curr_x = 0
        self.curr_y = 0
        self.curr_chn = 0
        self.request_idx = 0
        self.send_idx = 0
        self.curr_filt_x = 0
        self.curr_filt_y = 0
        self.curr_filt_set = 0
        self.ifmap_done = False

        offset_x = (self.filter_size[0] - 1)//2
        offset_y = (self.filter_size[1] - 1)//2
        # The first address needed to be filled in order to start sending
        #self.needed_addr = (self.image_size[0]*(1+offset_y) + 1+offset_x) *\
        #    (self.in_chn // self.block_size) - 1
        self.needed_addr = (self.image_size[0]*(offset_y) + 1+offset_x) *\
            (self.in_chn // self.block_size) - 1
        # Goes high to transfer sram control to output
        # Doing them synchronously would be better, but complicates things
        self.ready_to_output = False

    def tick(self):

        # WEIGHTS-------------------------------------------------------------------
        num_iterations = self.image_size[0]*self.image_size[1]*self.in_chn//self.block_size
        max_addr = self.filter_size[0]*self.filter_size[1]*self.in_chn*self.out_chn//self.block_size

        verbose = False

        if not self.wwr_done:
            if self.weights_wr_chn.valid():
                data = self.weights_wr_chn.pop()
                self.raw_stats['wr'] += len(data)
                #print(self.addr)
                #print("weights (iw glb) at addr {}".format(self.addr))
                #print(data)
                #print(self.addr)
                self.wsram.request(WR, self.addr, np.asarray(data))
                self.addr += 1
                #print("storing weights (wi glb)")
                #print(data)
                if (self.addr == max_addr):
                    self.addr = self.out_chn // self.block_size
                    self.wwr_done = True
                    #print("Done storing weights (wi glb)")
                    #print("--------------------------")
                    #print(self.wsram.data)
                    #print("--------------------------")

        # within this block of code self.addr is re-used
        # here it is more analogous to curr_set
        # and refers to the current block of filters (last index of the four)
        # that is being read
        else:
            # Catch addresses that correspond to nonzero inputs
            # search "self.weights_to_send.push(waddr)" below
            if (self.weights_to_send.valid() and self.addr == self.out_chn // self.block_size):
                self.base_addr = self.weights_to_send.pop()
                self.addr = 0
            # cycle through channels using self.addr
            # make requests to memory; will pick these up in the next if statement below
            elif (self.wlast_read.vacancy() and not self.addr == self.out_chn // self.block_size):
                full_addr = self.base_addr + self.addr
                self.wsram.request(RD, full_addr)
                self.wlast_read.push(False)
                #print("Request weights (wi glb):")
                #print(full_addr)
                self.addr += 1
        # catch requests from memory; send results to WeightsNoC
        if self.wlast_read.valid() and self.weights_rd_chn.vacancy(1):
            is_zero = self.wlast_read.pop()
            data = [e for e in self.wsram.response()]
            self.weights_rd_chn.push(data)
            #print("weights sent (from iw glb)")
            #print(data)
            self.raw_stats['rd'] += len(data)

        # these two if statements take care of an issue that occurs at the end
        # PEs don't automatically detect the end of the computation without inputs
        # from another location
        # So we send in some dummy inputs to flush out the last outputs
        if self.weights_rd_chn.vacancy(1) and not self.wlast_read.valid() and \
            not self.weights_to_send.valid() and self.addr == self.out_chn // self.block_size\
            and self.weights_to_flush > 0:
            self.weights_to_flush -= 1
            self.weights_rd_chn.push([0 for i in range(self.block_size)])
        if self.ifmap_done and self.inputs_to_flush > 0 and self.ifmap_rd_chn.vacancy(1):
            self.inputs_to_flush -= 1
            self.ifmap_rd_chn.push([-1, 0, 0])

        # IFMAP-------------------------------------------------------------------
        if not (self.ifmap_done and not self.ilast_read.valid() and not self.ready_to_output):
            verbose = False

            # shorthand values that will be useful later
            num_iteration = self.filter_size[0]*self.filter_size[1]
            offset_x = (self.filter_size[0] - 1)//2
            offset_y = (self.filter_size[1] - 1)//2
            filter_x = self.iteration % self.filter_size[0] - offset_x
            filter_y = self.iteration // self.filter_size[0] - offset_y
            in_sets = self.in_chn // self.block_size
            out_sets = self.out_chn // self.block_size
            if not self.iwr_done and not self.ready_to_output:
                # Write to GLB
                if self.ifmap_wr_chn.valid():
                    data = self.ifmap_wr_chn.pop()
                    data = np.reshape(np.asarray(data), (-1))

                    full_addr = in_sets*self.fmap_idx + self.curr_set
                    #print("ifmap (wi glb) received data:")
                    #print(data)
                    #print("{} >?= {}".format(full_addr, self.needed_addr))
                    self.curr_set += 1
                    addr = full_addr % self.ifmap_glb_depth

                    # if we have enough inputs in memory to start sending
                    if (full_addr == self.needed_addr):
                        self.ready_to_output = True
                        self.needed_addr += in_sets

                    self.isram.request(WR, addr, data)
                    self.raw_stats['wr'] += len(data)
                    #print("ifmap, iw glb")
                    #print("{} written to {}".format(data, addr))

                    if self.curr_set == self.fmap_sets:
                        self.curr_set = 0
                        self.fmap_idx += 1
                    if self.fmap_idx == self.fmap_per_iteration:
                        # Done initializing ifmaps and psums
                        print("iw glb: Finished filling ifmap buffer")
                        self.fmap_idx = 0
                        self.iwr_done = True
            elif self.ready_to_output:
                increment_vals = False
                # send data to NoC
                if (self.ilast_read.valid() and self.ifmap_rd_chn.vacancy(1)):
                    is_zero = self.ilast_read.pop()
                    #print(is_zero)
                    if (not is_zero):
                        self.curr_data = [e for e in self.isram.response()]
                        self.data_idx = 0
                    else:
                        increment_vals = True
                elif (not self.data_idx == self.num_nonzero and self.weights_to_send.vacancy() and self.base_addr_wo_chn >= 0):
                    data = [self.curr_data[i] for i in \
                        range(self.data_idx*3, self.data_idx*3 + 3)]
                    data_mod = [self.curr_x*self.image_size[1]+self.curr_y,\
                        data[1], data[2]]
                    self.ifmap_rd_chn.push(data_mod)
                    #print("iw glb inputs sent: {},{},{}".format(self.curr_x, self.curr_y, self.curr_chn))
                    #print(data_mod)
                    self.raw_stats['rd'] += 1

                    # Assertion checks that we will not attempt to read data that
                    # has not yet been stored in memory
                    waddr = self.base_addr_wo_chn + int(data[0])*out_sets
                    assert (self.wwr_done or waddr < self.addr)
                    #self.wsram.request(RD, waddr)
                    #self.wlast_read.push(False)
                    self.weights_to_send.push(waddr)
                    #print("Send request (wi glb):")
                    #print(waddr)

                    self.data_idx += 1
                    #if (self.data_idx == self.num_nonzero):
                    if (data[2] == 1):
                        self.data_idx = self.num_nonzero
                        increment_vals = True
                    if (self.data_idx == self.num_nonzero):
                        self.base_addr_wo_chn = -1
                    #print(self.data_idx)
                if (increment_vals):
                    #print(self.send_idx)
                    self.curr_chn += 1
                    if (self.curr_chn == in_sets):
                        self.curr_chn = 0
                        self.send_idx += 1
                    if (self.send_idx == self.filter_size[0]*self.filter_size[1]):
                        #print("Ready to shift input glb frame ({},{})".format(self.curr_x, self.curr_y))
                        self.send_idx = 0
                        self.curr_y += 1
                        if (self.curr_y == self.image_size[1]):
                            self.curr_y = 0
                            self.curr_x += 1
                        if (self.curr_x == self.image_size[0]):
                            self.curr_x = 0
                            self.ifmap_done = True
                            #print("Done sending inputs from iw glb")
                            self.ready_to_output = False
                            self.inputs_to_flush = 1
                            self.weights_to_flush = self.arr_y // self.block_size
                        elif (not self.iwr_done):
                            self.ready_to_output = False
                        #print(self.ifmap_wr_chn.valid())

                # stage one of these at a time
                # request data from SRAM
                if (not self.ifmap_done and self.ilast_read.vacancy(1) and \
                    self.data_idx == self.num_nonzero and self.weights_to_send.vacancy()\
                    and self.base_addr_wo_chn == -1):
                    # and not (self.curr_x == self.image_size[0]):
                    x_adj = (self.curr_x + self.curr_filt_x - offset_x)
                    y_adj = self.curr_y + self.curr_filt_y - offset_y
                    idx = x_adj*self.image_size[1] + y_adj
                    #print("{},{},{} input requested".format(x_adj, y_adj, self.curr_filt_set))
                    #print(idx)
                    if (x_adj < 0 or x_adj >= self.image_size[0] or y_adj < 0 or y_adj >= self.image_size[1]):
                        self.ilast_read.push(True)
                    else:
                        addr = (idx * in_sets + self.curr_filt_set) % self.ifmap_glb_depth
                        self.isram.request(RD, addr)
                        self.ilast_read.push(False)
                        # set up for corresponding weights to be sent later
                        self.base_addr_wo_chn = self.curr_filt_x*self.filter_size[1]\
                            *self.in_chn*out_sets + \
                            self.curr_filt_y*self.in_chn*out_sets + \
                            self.curr_filt_set*self.block_size*out_sets
                        #print("Next base addr = {}".format(self.base_addr_wo_chn))

                    self.curr_filt_set += 1
                    if (self.curr_filt_set == in_sets):
                        self.curr_filt_set = 0
                        self.curr_filt_y += 1
                    if (self.curr_filt_y == self.filter_size[1]):
                        self.curr_filt_y = 0
                        self.curr_filt_x += 1
                    if (self.curr_filt_x == self.filter_size[0]):
                        self.curr_filt_x = 0

class IFMapGLB(Module):
    def instantiate(self, wr_chn, rd_chn, arr_y, glb_depth, block_size, num_nonzero):
        self.wr_chn = wr_chn
        self.rd_chn = rd_chn
        self.arr_y = arr_y
        self.block_size = block_size
        self.num_nonzero = num_nonzero
        self.name = 'ifmap_glb'

        self.stat_type = 'show'
        self.raw_stats = {'size' : (glb_depth, num_nonzero), 'rd': 0, 'wr': 0}

        self.sram = SRAM(glb_depth, num_nonzero*3)
        self.last_read = Channel(3)
        self.glb_depth = glb_depth

        self.image_size = (0, 0)
        self.filter_size = (0, 0)
        self.fmap_sets = 0
        self.fmap_per_iteration = 0

        self.curr_set = 0
        self.fmap_idx = 0
        self.iteration = 0
        self.wr_done = False

        # For managing convolution
        self.curr_x = 0
        self.curr_y = 0
        self.curr_chn = 0
        self.request_idx = 0
        self.send_idx = 0
        #self.curr_filt_x = 0
        #self.curr_filt_y = 0
        self.ifmap_done = False

        self.needed_addr = 0
        self.ready_to_output = False # ready to output a filter_size block of inputs
        self.curr_data = [0 for i in range(3*num_nonzero)]
        self.data_idx = num_nonzero # block other operations while actively working through data
        # send one data point at a time (of num_nonzero)

    def configure(self, image_size, filter_size, in_chn, fmap_per_iteration):
        self.wr_done = False

        self.image_size = image_size
        self.filter_size = filter_size
        self.in_chn = in_chn
        self.fmap_per_iteration = fmap_per_iteration

        # For managing convolution
        self.curr_x = 0
        self.curr_y = 0
        self.curr_chn = 0
        self.request_idx = 0
        self.send_idx = 0
        self.curr_filt_x = 0
        self.curr_filt_y = 0
        self.curr_filt_set = 0
        self.ifmap_done = False

        offset_x = (self.filter_size[0] - 1)//2
        offset_y = (self.filter_size[1] - 1)//2
        # The first address needed to be filled in order to start sending
        self.needed_addr = (self.image_size[0]*(1+offset_y) + 1+offset_x) *\
            (self.in_chn // self.block_size) - 1
        # Goes high to transfer sram control to output
        # Doing them synchronously would be better, but complicates things
        self.ready_to_output = False

    def tick(self):
        if (self.ifmap_done and not self.last_read.valid() and not self.ready_to_output):
            return

        verbose = False

        num_iteration = self.filter_size[0]*self.filter_size[1]
        offset_x = (self.filter_size[0] - 1)//2
        offset_y = (self.filter_size[1] - 1)//2
        filter_x = self.iteration % self.filter_size[0] - offset_x
        filter_y = self.iteration // self.filter_size[0] - offset_y
        in_sets = self.in_chn // self.block_size
        #print(filter_x)

        if not self.wr_done and not self.ready_to_output:
            # Write to GLB
            if self.wr_chn.valid():
                data = self.wr_chn.pop()
                data = np.reshape(np.asarray(data), (-1))
                # Write ifmap to glb
                #print("ifmap_glb")
                #print(data)

                full_addr = in_sets*self.fmap_idx + self.curr_set
                self.curr_set += 1
                addr = full_addr % self.glb_depth

                # if we have enough inputs in memory to start sending
                if (full_addr == self.needed_addr):
                    self.ready_to_output = True
                    self.needed_addr += in_sets

                self.sram.request(WR, addr, data)
                self.raw_stats['wr'] += len(data)
                if self.curr_set == self.fmap_sets:
                    self.curr_set = 0
                    self.fmap_idx += 1
                if self.fmap_idx == self.fmap_per_iteration:
                    # Done initializing ifmaps and psums
                    # self.sram.dump()
                    self.fmap_idx = 0
                    self.wr_done = True
        elif self.ready_to_output:
            # send data to NoC
            if (self.last_read.valid() and self.rd_chn.vacancy(1) and self.data_idx == 0):
                xmin = self.curr_filt_x
                xmax = xmin + self.arr_y
                #print("{}-{},{},{}".format(xmin, xmax, self.holder_y, self.curr_chn))
                #data = [self.holder[x][self.holder_y][self.curr_chn] for x in range(xmin, xmax)]
                is_zero = self.last_read.pop()
                if (not is_zero):
                    self.curr_data = [e for e in self.sram.response()]
                    self.data_idx = 0

            if (not self.data_idx == self.num_nonzero):
                data = [self.curr_data[i] for i in \
                    range(self.data_idx*3, self.data_idx*3 + 3)]
                self.rd_chn.push(data)
                self.raw_stats['rd'] += len(data)
                self.data_idx += 1
                if (self.data_idx == num_nonzero):
                    self.data_idx = 0
                    self.curr_chn += 1
                    if (self.curr_chn == self.arr_y):
                        self.curr_chn = 0
                        self.send_idx += 1
                    if (self.send_idx == self.filter_size[0]*self.filter_size[1]):
                        if (verbose):
                            print("Ready to shift input glb frame")
                        self.send_idx = 0
                        self.curr_y += 1
                        if (self.curr_y == self.image_size[1]):
                            self.curr_y = 0
                            self.curr_x += 1
                        if (self.curr_x == self.image_size[0]):
                            self.curr_x = 0
                            self.ifmap_done = True
                            self.ready_to_output = False
                        elif (not self.wr_done):
                            self.ready_to_output = False

                #print("{},{},{}".format(self.holder_x, self.holder_y, self.curr_chn))
                #print(data)

            # stage one of these at a time
            # request data from SRAM
            if (not self.ifmap_done and self.last_read.vacancy(1) and self.data_idx == num_nonzero):
                # and not (self.curr_x == self.image_size[0]):
                idx = (self.curr_x + self.curr_filt_x - offset_x)*self.image_size[1] + self.curr_y + self.curr_filt_y - offset_y
                #print(idx)
                if (idx >= self.image_size[0]*self.image_size[1] or idx < 0):
                    self.last_read.push(True)
                else:
                    addr = idx * in_sets + self.curr_filt_set
                    self.sram.request(RD, addr)
                    self.last_read.push(False)

                self.curr_filt_set += 1
                if (self.curr_filt_set == in_sets):
                    self.curr_filt_set = 0
                    self.curr_filt_y += 1
                if (self.curr_filt_y == self.filter_size[1]):
                    self.curr_filt_y = 0
                    self.curr_filt_x += 1
                if (self.curr_filt_x == self.filter_size[0]):
                    self.curr_filt_x = 0

class PSumGLB(Module):
    def instantiate(self, dram_wr_chn, noc_wr_chn, rd_chn, glb_depth, block_size, num_nonzero):
        self.dram_wr_chn = dram_wr_chn
        self.noc_wr_chn = noc_wr_chn
        self.rd_chn = rd_chn
        self.name = 'psum_glb'
        self.block_size = block_size
        self.num_nonzero = num_nonzero

        self.stat_type = 'show'
        self.raw_stats = {'size' : (glb_depth, block_size), 'rd': 0, 'wr': 0}

        self.sram = SRAM(glb_depth, block_size, nports=2, dtype=np.float16)
        self.last_read = Channel(3)

        self.filter_size = (0, 0)
        self.fmap_sets = 0
        self.fmap_per_iteration = 0

        self.rd_set = 0
        self.fmap_rd_idx = 0
        self.iteration = 0

        self.wr_set = 0
        self.fmap_wr_idx = 0
        self.wr_done = False

    def configure(self, filter_size, out_chn, fmap_per_iteration):
        self.wr_done = False

        self.filter_size = filter_size
        self.out_chn = out_chn
        self.fmap_per_iteration = fmap_per_iteration

        self.rd_set = 0
        self.fmap_rd_idx = 0
        self.iteration = 0

        self.wr_set = 0
        self.fmap_wr_idx = 0
        self.wr_done = False

    def tick(self):
        num_iteration = 1#self.filter_size[0]*self.filter_size[1]

        if not self.wr_done:
            # Write to GLB
            if self.dram_wr_chn.valid():
                data = self.dram_wr_chn.pop()
                # Write ifmap to glb
                #print("psum_glb")
                #print(data)
                #addr = self.fmap_sets*self.fmap_wr_idx + self.wr_set
                addr = self.wr_set
                self.wr_set += 1
                self.sram.request(WR, addr, data, port=0)
                self.raw_stats['wr'] += len(data)
                if self.wr_set == self.out_chn//self.block_size:
                    self.wr_set = 0
                    self.wr_done = True
                    #self.fmap_wr_idx += 1
                #if self.fmap_wr_idx == self.fmap_per_iteration:
                #    # Done initializing ifmaps and psums
                #    # self.sram.dump()
                #    #print("done!")
                #    self.fmap_wr_idx = 0
                #    self.wr_done = True

        else:
            # Read from GLB and deal with SRAM latency
            if self.rd_chn.vacancy(1) and self.iteration < num_iteration:
                #addr = self.fmap_sets*self.fmap_rd_idx + self.rd_set
                addr = self.rd_set
                self.sram.request(RD, addr, port=0)
                self.last_read.push(False)
                self.rd_set += 1
                if self.rd_set == self.out_chn//self.block_size:
                    self.rd_set = 0
                    self.fmap_rd_idx += 1
                if self.fmap_rd_idx == self.fmap_per_iteration:
                    self.fmap_rd_idx = 0
                    self.iteration += 1

            # Process the last read sent to the GLB SRAM
            if self.last_read.valid():
                is_zero = self.last_read.pop()
                data = [0]*self.block_size if is_zero else \
                        [e for e in self.sram.response()]
                self.rd_chn.push(data)
                self.raw_stats['rd'] += len(data)

class WeightsGLB(Module):
    def instantiate(self, wr_chn, rd_chn, glb_depth, block_size):
        self.wr_chn = wr_chn
        self.rd_chn = rd_chn
        self.name = 'weight_glb'

        self.filter_size = (0,0)
        self.image_size = (0,0)
        self.wr_done = False
        self.iteration = 0
        self.addr = 0
        self.in_chn = 0
        self.out_chn = 0
        #self.arr_y = 0
        #self.out_sets = 0
        self.block_size = block_size

        self.sram = SRAM(glb_depth, block_size)
        self.last_read = Channel(3)

        self.stat_type = 'show'
        self.raw_stats = {'size' : (glb_depth, block_size), 'rd': 0, 'wr': 0}

    def configure(self, filter_size, image_size, in_chn, out_chn):
        self.filter_size = filter_size
        self.image_size = image_size
        self.iteration = 0
        self.addr = 0
        self.in_chn = in_chn
        self.out_chn = out_chn
        #self.arr_y = arr_y
        #self.out_sets = out_sets

        self.wr_done = False

    def tick(self):
        # num_iterations = times to read out all weights
        # max_addr = number of slots to hold all blocks of weights
        num_iterations = self.image_size[0]*self.image_size[1]*self.in_chn//self.block_size
        max_addr = self.filter_size[0]*self.filter_size[1]*self.in_chn*self.out_chn//self.block_size

        verbose = False

        if not self.wr_done:
            if self.wr_chn.valid():
                data = self.wr_chn.pop()
                self.raw_stats['wr'] += len(data)
                #print(self.addr)
                #print(data)
                #print(type(data))
                self.sram.request(WR, self.addr, np.asarray(data))
                self.addr += 1
                if (self.addr == max_addr):
                    self.addr = 0
                    self.wr_done = True
                if (verbose):
                    print("weight_glb")
                    print(data)

        elif self.rd_chn.vacancy(1):
            if (self.iteration < num_iterations):
                self.sram.request(RD, self.addr)
                self.last_read.push(False)
                self.addr += 1
                if (self.addr == max_addr):
                    self.addr = 0
                    self.iteration += 1
            #self.rd_chn.push(data)
            #self.raw_stats['rd'] += len(data)
        if self.last_read.valid():
            is_zero = self.last_read.pop()
            data = [e for e in self.sram.response()]
            self.rd_chn.push(data)
            #print(self.iteration)
            #print(data)
            self.raw_stats['rd'] += len(data)


import pandas as pd
import numpy as np
import pickle
import os
import sys
import time
import numpy as np
from sklearn.model_selection import cross_validate
from collections import Counter
from imblearn.over_sampling import RandomOverSampler
import redis_connection

sys.path.append(str(os.environ.get("FLEXHTTP")) + "/client")
sys.path.append(str(os.environ.get("FLEXHTTP")))
from myUtil import RedisTable, RedisRow, sortFImp

def train_RF(data_X, data_Y, max_depth=4, n_jobs=12, out=False):
    ros = RandomOverSampler(random_state=0)

    data_X, data_Y = ros.fit_resample(data_X, data_Y)
    print(Counter(data_Y))
    print(data_X.shape)

    # need a per-trained global model
    # clf = RandomForestClassifier(max_depth=max_depth, n_jobs=12)
    _file = os.path.join(str(os.environ.get("FLEXHTTP")), "myUtil", "global_pool", "global_model.pkl")
    clf = pickle.load(open(_file, 'rb'))
    scores = cross_validate(clf, data_X, data_Y,
                            scoring=['accuracy', 'precision',
                                     'recall', 'f1', 'roc_auc'],
                            cv=10,
                            return_train_score=False)
    clf.fit(data_X, data_Y)
    fImp = sortFImp(clf.feature_importances_)

    return scores, clf

def update_global_model(global_train_data):
    data_X = global_train_data.values[:, 2:].astype(np.float64)
    print("data_X length: {}".format(len(data_X)))
    if len(data_X) == 0:
        print("Available trace list is empty!")
        return
    data_Y = global_train_data.values[:, 0].astype(np.int32)
    scores, global_model = train_RF(data_X, data_Y, max_depth=8)

    # store local model to file
    _file = os.path.join(str(os.environ.get("FLEXHTTP")), "myUtil", "global_pool", "global_model.pkl")
    pickle.dump(global_model, open(_file, 'wb'))

    return global_model

def get_format_training_data():
    global_trace_table = RedisTable(redis_connection.redis_pool, redis_connection.table_global_trace)
    global_trace_rows = global_trace_table.get_all_rows()
    global_trace_dicts = list()
    for primekey in global_trace_rows:
        data_row = RedisRow(redis_connection.redis_pool, primekey)
        data_dict = data_row.get_val_dict()
        label = _get_label_of_trace(data_dict)
        if label is None:
            continue
        else:
            data_dict.update({'label': label})
            global_trace_dicts.append(data_dict)

    return pd.DataFrame(global_trace_dicts, columns=[
        "label",
        "link",
        "all_cnt",
        "all_size",
        "text_cnt",
        "text_size",
        "css_cnt",
        "css_size",
        "js_cnt",
        "js_size",
        "img_cnt",
        "img_size",
        "rtt",
        "loss_rate",
        "bandwidth"
    ])

bandwidth_partitions = [  # 1, 10, 50, 100
    (0, 5),
    (5, 25),
    (25, 75),
    (75, 10000)
]
loss_rate_partitions = [  # 0.01, 0.10, 1.00, 5.00
    (0, 0.05),
    (0.05, 0.50),
    (0.50, 2.50),
    (2.50, 10000)
]
rtt_partitions = [  # 30, 100, 300, 500
    (0, 50),
    (50, 200),
    (200, 400),
    (400, 10000)
]

def _get_label_of_trace(trace):
    for (a, b) in bandwidth_partitions:
        if a <= float(trace['bandwidth']) < b:
            bandwidth_str = str((a, b))
            break
    for (a, b) in loss_rate_partitions:
        if a <= float(trace['loss_rate']) < b:
            loss_rate_str = str((a, b))
            break
    for (a, b) in rtt_partitions:
        if a <= float(trace['rtt']) < b:
            rtt_str = str((a, b))
            break

    try:
        global_trace_with_label_row = RedisRow(redis_connection.redis_pool, "label_{}_{}_{}_{}".format(
            trace['link'], bandwidth_str, loss_rate_str, rtt_str
        ))
        if float(global_trace_with_label_row.get_val_of_key('plt_http2')) < float(global_trace_with_label_row.get_val_of_key('plt_quic')) + 2*float(trace['rtt'])*0.001:
            return 0
        else:
            return 1
    except Exception as err:
        return None

if __name__ == "__main__":
    while True:
        time.sleep(600)  # sleep 10 minutes
        x = get_format_training_data()
        update_global_model(x)
        print('updated')