import os
import sys
import torch
from functools import partial
import numpy as np
import RNA
import itertools
import h5py
from torch.utils.data import Dataset, DataLoader

basedir = os.path.split(os.path.dirname(os.path.abspath(__file__)))[0]
sys.path.append(basedir)
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from baseline_models.GraphLSTMVAE import GraphEncoder
from jtvae_models.GraphEncoder import GraphEncoder as jtvae_GraphEncoder
from jtvae_models.TreeEncoder import TreeEncoder
from jtvae_models.BranchedTreeEncoder import BranchedTreeEncoder
from lib.tree_decomp import RNAJunctionTree

NUC_VOCAB = ['A', 'C', 'G', 'U']
LEN_NUC_VOCAB = len(NUC_VOCAB)
STRUCT_VOCAB = ['(', ')', '.']
LEN_STRUCT_VOCAB = len(STRUCT_VOCAB)

JOINT_VOCAB = [''.join(cand) for cand in itertools.product(NUC_VOCAB, STRUCT_VOCAB)]
FDIM_JOINT_VOCAB = len(JOINT_VOCAB)

# RNAcompete dataset information
rnacompete_train_datapath = os.path.join(basedir, 'data', 'RNAcompete_derived_datasets', 'full', '{}_data_full_A.txt')
rnacompete_test_datapath = os.path.join(basedir, 'data', 'RNAcompete_derived_datasets', 'full', '{}_data_full_B.txt')
rnacompete_all_rbps = ['Fusip', 'HuR', 'PTB', 'RBM4', 'SF2', 'SLM2', 'U1A', 'VTS1', 'YB1']

# RNAcompete-S dataset information
rnacompete_s_datapath = os.path.join(basedir, 'data', 'RNAcompete_S', 'predicted_centroid_structures', '{}.tab')
rnacompete_s_pool_datapath = os.path.join(basedir, 'data', 'RNAcompete_S', 'predicted_centroid_structures',
                                          '8_Pool.tab')
rnacompete_s_all_rbps = ['HuR', 'PTB', 'QKI', 'Vts1', 'RBMY', 'SF2', 'SLBP']

# ncRNA dataset information
ncRNA_train_datapath = os.path.join(basedir, 'data', 'ncRNA', 'dataset_Rfam_6320_13classes.fasta')
ncRNA_test_datapath = os.path.join(basedir, 'data', 'ncRNA', 'dataset_Rfam_validated_2600_13classes.fasta')
ncRNA_all_classes = ['miRNA', '5S_rRNA', '5_8S_rRNA', 'ribozyme', 'CD-box', 'HACA-box', 'scaRNA', 'tRNA', 'Intron_gpI',
                     'Intron_gpII', 'IRES', 'leader', 'riboswitch']

# in-vivo RBP binding dataset information
rbp_datapath = os.path.join(basedir, 'data', 'rbpdata', '{}')
rbp_dataset_options = {'data_RBPslow.h5', 'data_RBPsmed.h5', 'data_RBPshigh.h5'}

######################## for htr-selex multilabel dataset ########################
def read_htr_selex_fasta(filepath, nb_class):
    seqs, labels = [], []
    with open(filepath, 'r') as file:
        for line in file:
            if line.startswith('>'):
                _labels = [int(l) for l in line.split('label:')[-1].rstrip().split(',')]
                labels.append(np.array(list(map(lambda x: x in _labels, range(nb_class))), dtype=np.float32))
            else:
                seqs.append(line.rstrip())
    return seqs, labels

######################## for RNAcompte-S dataset ########################
def read_rnacompete_s_tab(filepath, seq_limit=np.inf):
    seqs = []
    with open(filepath, 'r') as file:
        for line in file:
            line = line.rstrip()
            seqs.append(line.split('\t')[1].replace('T', 'U'))
    if seq_limit > len(seqs):
        return seqs
    else:
        return list(np.random.choice(seqs, seq_limit, False))

def create_rnacompete_s_dataset(pos_raw_filepath, neg_raw_filepath, seq_limit=np.inf):
    pos_seq = read_rnacompete_s_tab(pos_raw_filepath, seq_limit)
    size_pos = len(pos_seq)
    neg_seq = read_rnacompete_s_tab(neg_raw_filepath, size_pos)
    size_neg = len(neg_seq)
    print('all_sizes', size_pos + size_neg)
    return pos_seq, neg_seq

def read_curated_rnacompete_s_dataset(filepath):
    pos_seq, neg_seq = [], []
    label = None
    with open(filepath, 'r') as file:
        for line in file:
            if line.startswith('>'):
                label = int(line.rstrip().split(':')[-1])
            else:
                seq = line.rstrip()
                if label == 1:
                    pos_seq.append(seq)
                else:
                    neg_seq.append(seq)

    return pos_seq, neg_seq

######################## for RNAcompete dataset ########################
def read_rnacompete_datafile(filepath):
    seqs, targets = [], []
    with open(filepath, 'r') as file:
        for line in file:
            line = line.rstrip()
            target, seq = line.split('\t')
            targets.append(float(target))
            seqs.append(seq.replace('T', 'U'))
    return seqs, targets

######################## for ncRNA dataset ########################
def read_ncRNA_fasta(filepath):
    seqs, labels = [], []
    seq = ''
    with open(filepath, 'r') as file:
        for line in file:
            line = line.rstrip()
            if line.startswith('>'):
                if len(seq) > 0:
                    seqs.append(seq)
                    seq = ''
                labels.append(ncRNA_all_classes.index(line.split(' ')[-1]))
            else:
                seq += line.replace('T', 'U')
        seqs.append(seq)
    return seqs, labels

######################## for in-vivo RBP binding dataset ########################
def read_rbp_h5py(filepath, mode='train'):
    with h5py.File(filepath, 'r') as file:
        all_np_seq = np.array(file['%s_in_seq' % (mode)]).transpose(0, 2, 1)
        all_label = np.array(file['%s_out' % (mode)])
    return all_np_seq, all_label

######################## lstm-seqonly encoding subroutine ########################
def lstm_seqonly_encoding_subroutine(seq):
    if type(seq) is np.ndarray:
        # convert numpy sequence to string
        seq = ''.join([NUC_VOCAB[nuc_idx] for nuc_idx in np.argmax(seq, axis=-1)])

    encoding = []
    label = []
    for seq_char in seq:
        onehot_enc = np.array(list(map(lambda x: x == seq_char, NUC_VOCAB)),
                              dtype=np.float32)
        encoding.append(onehot_enc)
        label.append(np.argmax(onehot_enc))
    return encoding, label

######################## lstm encoding subroutine ########################
def lstm_joint_encoding_subroutine(seq):
    if type(seq) is np.ndarray:
        # convert numpy sequence to string
        seq = ''.join([NUC_VOCAB[nuc_idx] for nuc_idx in np.argmax(seq, axis=-1)])

    struct = RNA.fold(seq)[0]
    joint_encoding = []
    label = []
    for seq_char, struct_char in zip(seq, struct):
        onehot_enc = np.array(list(map(lambda x: x == seq_char + struct_char, JOINT_VOCAB)),
                              dtype=np.float32)
        joint_encoding.append(onehot_enc)
        label.append(np.argmax(onehot_enc))
    return joint_encoding, label

######################## graph encoding subroutine ########################
def graph_encoding_subroutine(batch_seq):
    all_joint_encoding = []
    all_label = []
    all_pairs = []
    for seq in batch_seq:
        if type(seq) is np.ndarray:
            # convert numpy sequence to string
            seq = ''.join([NUC_VOCAB[nuc_idx] for nuc_idx in np.argmax(seq, axis=-1)])

        struct = RNA.fold(seq)[0]
        all_pairs.append([seq, struct])

        joint_encoding = []
        label = []
        for seq_char, struct_char in zip(seq, struct):
            onehot_enc = np.array(list(map(lambda x: x == seq_char + struct_char, JOINT_VOCAB)), dtype=np.float32)
            joint_encoding.append(onehot_enc)
            label.append(np.argmax(onehot_enc))
        all_joint_encoding.append(joint_encoding)
        all_label.append(label)

    graph_encoder_input = GraphEncoder.prepare_batch_data(all_pairs)

    return (graph_encoder_input, all_joint_encoding, all_label)

######################## jtvae encoding subroutine ########################
def jtvae_encoding_subroutine(batch_seq, tree_enc_type='baseline'):
    all_trees = []
    for seq in batch_seq:
        if type(seq) is np.ndarray:
            # convert numpy sequence to string
            seq = ''.join([NUC_VOCAB[nuc_idx] for nuc_idx in np.argmax(seq, axis=-1)])

        struct, mfe = RNA.fold(seq)
        tree = RNAJunctionTree(seq, struct, free_energy=float(mfe))
        all_trees.append(tree)
    graph_encoder_input = jtvae_GraphEncoder.prepare_batch_data(
        [(tree.rna_seq, tree.rna_struct) for tree in all_trees])
    if tree_enc_type == 'jtvae':
        tree_encoder_input = TreeEncoder.prepare_batch_data(all_trees)
    elif tree_enc_type == 'jtvae_branched':
        tree_encoder_input = BranchedTreeEncoder.prepare_batch_data(all_trees)

    return (all_trees, graph_encoder_input, tree_encoder_input)

class TaskFolder:

    def __init__(self, all_seq, all_labels, batch_size, num_workers=4, shuffle=True,
                 mode='train', preprocess_type='lstm'):
        assert preprocess_type in ['lstm', 'lstm_seqonly', 'graph_lstm', 'jtvae', 'jtvae_branched'], \
            'preprocess type {} not found in {}'.format(preprocess_type, ['lstm', 'lstm_seqonly', 'graph_lstm', 'jtvae',
                                                                          'jtvae_branched'])

        self.all_seq = all_seq
        self.all_labels = all_labels
        self.size = len(self.all_seq)
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.shuffle = shuffle
        self.mode = mode
        self.preprocess_type = preprocess_type

    def __iter__(self):

        if self.shuffle:
            shuffled_idx = np.random.permutation(self.size)
            all_seq = self.all_seq[shuffled_idx]
            all_labels = self.all_labels[shuffled_idx]
        else:
            all_seq = self.all_seq
            all_labels = self.all_labels

        batches = [[all_seq[i: i + self.batch_size], all_labels[i: i + self.batch_size]] for i in
                   range(0, self.size, self.batch_size)]

        dataset = TaskDataset(batches, self.preprocess_type)
        dataloader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=self.num_workers,
                                collate_fn=lambda x: x[0])

        for b in dataloader:
            yield b

        del batches, dataset, dataloader

class TaskDataset(Dataset):

    def __init__(self, data, preprocess_type):
        self.data = data
        self.preprocess_type = preprocess_type

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):

        batch_seq, batch_label = self.data[idx]
        with torch.no_grad():

            if self.preprocess_type == 'lstm':

                batch_joint_encodings = []
                batch_seq_label = []
                for seq in batch_seq:
                    joint_encoding, label = lstm_joint_encoding_subroutine(seq)
                    batch_joint_encodings.append(joint_encoding)
                    batch_seq_label.append(label)
                return (batch_joint_encodings, batch_seq_label), batch_label

            elif self.preprocess_type == 'lstm_seqonly':

                batch_encodings = []
                batch_seq_label = []
                for seq in batch_seq:
                    encoding, label = lstm_seqonly_encoding_subroutine(seq)
                    batch_encodings.append(encoding)
                    batch_seq_label.append(label)
                return (batch_encodings, batch_seq_label), batch_label

            elif self.preprocess_type == 'graph_lstm':

                batch_graph_input = graph_encoding_subroutine(batch_seq)
                return batch_graph_input, batch_label

            elif self.preprocess_type == 'jtvae' or self.preprocess_type == 'jtvae_branched':

                batch_jtvae_inputs = jtvae_encoding_subroutine(batch_seq, self.preprocess_type)
                return batch_jtvae_inputs, batch_label


import random

liste = ["Kobe Bryant","Lebron James","Kevin Durant","Michael Jordan","Larry Bird","Stephen Curry",
"Tim Duncan", "Wilt Chamberlain","Bill Russel","Giannis Antetokounmpo","Kareem Abdul-Jabbar","Magic Johnson","Shaq"]

liste = [i.upper() for i in liste]

answer = random.choice(liste)

counter= 5

print("""
***************************
GUESS THE TOP 10 NBA PLAYER

Guess one of the top 10 NBA Players of all time.
You have 5 guesses!

***************************
""")

while True:
    guess = (input("which top 10 nba player is this?"))

    if counter == 0:
       print("the answer is ", answer)
       break

    for i in range(len(guess)):
        guess = guess.upper()

    if guess == answer:
        print("congrats!")
        break
    elif guess != answer:
        counter -= 1
        print("Wrong! You have {} guesses left" .format(counter))


import os

import pytest

from plexapi.server import PlexServer
from fylmlib.pushover import init, Client

import fylmlib.config as config
import fylmlib.notify as notify

@pytest.mark.skip()
class TestPlex:

    def test_plex(self):

        # Note: this test is disabled by default, because it requires a Plex server and
        # credentials in order to pass. You can run it yourself by configuring config.yaml
        # with your own Plex server details and removing the comment on `@pytest.mark.skip()`

        # Set up config
        config.test = False
        config.quiet = False
        assert(config.test is False)
        assert(config.quiet is False)
        assert(config.plex.enabled is True)
        assert(config.plex.baseurl == "http://127.0.0.1:32400")
        assert(config.plex.token == "YOUR_TOKEN_HERE")
        assert("Movies" in config.plex.sections)

        plex = PlexServer(baseurl=config.plex.baseurl, token=config.plex.token, timeout=10)

        for section in (plex.library.section(section) for section in config.plex.sections):
            section.update()

    @pytest.mark.xfail(raises=(OSError, IOError))
    def test_plex_fail(self):

        plex = PlexServer(baseurl="http://127.0.0.1:12701", token="BAD_TOKEN", timeout=2)


from rest_framework import serializers
from SHOP_APP.models import Product, Order

class ProductSerializer(serializers.ModelSerializer):
	url = serializers.HyperlinkedIdentityField(
		view_name = 'product_detail_api_view',
		lookup_field = 'pk')
	class Meta:
		model = Product
		fields = [
		'url',
		'name',
		'price',
		'discount_price',
		'digital',
		'image',
		'slug',
		'description',
		'category',
		'quantity',
		'item_id',
		'pk'
		]

class UserOrderSerializer(serializers.ModelSerializer):
	user = serializers.StringRelatedField()
	products = serializers.StringRelatedField(many = True)
	url = serializers.HyperlinkedIdentityField(
		view_name = 'order_detail_api_view',
		lookup_field = 'transaction_id')
	#products = serializers.HyperlinkedRelatedField(
    #    many=True,
    #    read_only = True,
    #    view_name='product_detail_api_view',
    #    lookup_field = 'pk')
	class Meta:
		model = Order
		fields = [
		'url',
		'user',
		'date_ordered',
		'complete',
		'transaction_id',
		'products',
		'payment_method',
		'total_cost'
		]


import os, re, sys
sys.dont_write_bytecode = 1

import gen_mul4 as P

g_n = int(sys.argv[1])
g_tgt = sys.argv[2]

def do_it(tgt):
    data = {
            'macro_name': 'mpn_le_%s' % g_n,
            'input': ['ap a_p', 'bp b_p'],
            'scratch': ['re tgt', 'w0 scratch'],
            'clobber': 'cc',
            'source': os.path.basename(sys.argv[0]),
            'code_language': 'asm',
            'vars_type': {'scratch': 'uint64_t'},
            'macro_parameters': 'tgt a_p b_p',
            }
    ofs = 8 * (g_n - 1)
    code = ['movq %s(ap), w0' % ofs, 'xorq re, re', 'subq %s(bp), w0' % ofs, 'jnz done']
    # carry flag is set iff subtraction result is negative

    i = g_n - 2
    while i > 0:
        ofs = 8 * i
        code += ['movq %s(ap), w0' % ofs, 'subq %s(bp), w0' % ofs, 'jnz done']
        i -= 1

    code += ['movq (ap), w0', 'subq (bp), w0', 'done:', 'adcq $0, re']

    code = '\n'.join(code)

    all_vars = P.extract_int_vars_name(data['scratch']) + \
            P.extract_int_vars_name(data['input'])
    for v in all_vars:
        code = re.sub(r'\b%s\b' % v, '%%[%s]' % v, code)
    code = re.sub(r'\bdone\b', 'done%=', code)

    P.write_cpp_code(tgt, code, data)

with open(g_tgt, 'wb') as g_o:
    do_it(g_o)


import circle

def main():
    r = float(input("what is the radius: "))
    ar = circle.ar_circle(r)   # circle.py -> ar_circle()
    print("Area:", ar)
    ci = circle.ci_circle(r)
    print("Circumference: ", ci)

main()


import numpy as np
import Cython_resources.build.simulation_instance as simulation_instance
import Cython_resources.build.simulation_instance_comms as simulation_instance_comms
import time
import multiprocessing

def simulate_tofile(filename, runs_per_thread, generations, Z, socialnorm, spread):

    runs = runs_per_thread
    population_size = Z
    mutation_rate = float(np.power(float(10 * Z), float(-1)))
    execution_error = 0.08
    reputation_assignment_error = 0.01
    private_assessment_error = 0.01
    reputation_update_prob = 1.0
    cost = 1
    benefit = 5
    num_threads = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(num_threads)
    cooperation_index_average = float(0)
    results = [pool.apply_async(simulation_instance_comms.run_instance,
                                args=(runs, generations, population_size, mutation_rate,
                                      execution_error, reputation_assignment_error,
                                      private_assessment_error, reputation_update_prob, spread,
                                      np.array(socialnorm), cost, benefit)) for _ in range(num_threads)]
    """
    result is the [cooperation index, Z, socialnorm]
    """
    for result in results:
        cooperation_index_values_i = result.get()
        cooperation_index_average += float(cooperation_index_values_i[0])
        out_string = str(cooperation_index_values_i[0]) + ", Z: " + \
            str(cooperation_index_values_i[1]) + " of " + str(np.ndarray.tolist(cooperation_index_values_i[2])) +\
            ", Spread: " + str(spread)
        file_out = open(filename, 'a')
        file_out.write(out_string + "\n")
        file_out.close()
        print(out_string)
    cooperation_index_average /= float(num_threads)
    pool.close()
    return cooperation_index_average

if __name__ == '__main__':
    start = time.clock()

    SJ = [[1, 0], [0, 1]]
    SS = [[1, 1], [0, 1]]
    ZERO = [[0, 0], [0, 0]]
    IS = [[1, 1], [0, 0]]

    variable_assessment_folder = "Data/VariableAssessmentError/"

    run_number = 100
    generation_number = 3 * np.power(10, 5)
    simulation_data = [

        [variable_assessment_folder + "Spread05.txt", 25, SJ, 0.5],
        [variable_assessment_folder + "Spread1.txt", 25, SJ, 1.0],
        [variable_assessment_folder + "Spread175.txt", 25, SJ, 1.75],
        [variable_assessment_folder + "Spread3.txt", 25, SJ, 3.0],
        [variable_assessment_folder + "Spread6.txt", 25, SJ, 6.0],

        # Simple Standing
        [variable_assessment_folder + "Spread05.txt", 25, SS, 0.5],
        [variable_assessment_folder + "Spread1.txt", 25, SS, 1.0],
        [variable_assessment_folder + "Spread175.txt", 25, SS, 1.75],
        [variable_assessment_folder + "Spread3.txt", 25, SS, 3.0],
        [variable_assessment_folder + "Spread6.txt", 25, SS, 6.0],

    ]

    for data in simulation_data:
        simulate_tofile(data[0], run_number, generation_number, data[1], data[2], data[3])
    end = time.clock()
    print("\nSimulation completed in " + str(end - start))


# -*- coding: utf-8 -*-
"""
Created on Tue Jan 14 22:40:23 2020

@author: COM
"""

infile=open("포켓몬.txt")
outfile=open("포켓몬순서.txt","w")
i=1
for line in infile:
    outfile.write(str(i)+":"+line)
    i+=1

infile.close()
outfile.close()

python 07 상속과 다형성, 파일과 예외처리/번호메기기.py


from bs4 import BeautifulSoup
from get_daily_price import *
from get_info import get_information
from edit_and_predict import *

def get_data_frame(url):
    soup = BeautifulSoup(url, 'html.parser')
    card_name = list()
    card_id = list()
    rarity = list()
    prices = list()
    link_to_card = list()
    day_1 = list()
    day_2 = list()
    day_3 = list()
    day_4 = list()
    day_5 = list()
    day_6 = list()
    day_7 = list()
    day_8 = list()

    new_url = soup.find('select', attrs={'name': 'set'})
    for option in new_url.findAll('option'):
        add = option["value"]
        card_name, card_id, rarity, link_to_card, prices = get_information(
            getHTMLdocument(f"https://shop.tcgplayer.com/price-guide/yugioh/" + add), card_name, card_id, rarity,
            link_to_card, prices)

        if len(card_id) > 4000:
            break

    day_8, day_7, day_6, day_5, day_4, day_3, day_2, day_1 = fill_list(card_name, day_8, day_7, day_6,
                                                                                day_5, day_4, day_3, day_2,
                                                                                day_1)

    day_8, day_7, day_6, day_5, day_4, day_3, day_2, day_1 = get_daily_price(link_to_card, day_8,
                                                                             day_7, day_6, day_5, day_4,
                                                                             day_3, day_2, day_1)
    data = pd.DataFrame(
        {"card_name": card_name, "card_id": card_id, "rarity": rarity, "link": link_to_card, "price": prices,
         "day 1": day_1, "day 2": day_2, "day 3": day_3, "day 4": day_4, "day 5": day_5, "day 6": day_6,
         "day 7": day_7, "day 8": day_8})

    data = edit_df(data)

    day_9 = predict_day_9(data)
    data['day 9'] = day_9
    return data


from django.shortcuts import render
from django.shortcuts import render_to_response

# Create your views here.
from django.http import HttpResponse
from django.http import HttpResponseRedirect

from django.contrib.auth import authenticate, login
from django.contrib.auth.models import User
from django.contrib.auth.forms import AuthenticationForm

def index(request):
    return render(request,'index.html');

from myapplication.forms import UserForm
from django.template import RequestContext

def register(request):
    # Like before, get the request's context.
    context = RequestContext(request)

    # A boolean value for telling the template whether the registration was successful.
    # Set to False initially. Code changes value to True when registration succeeds.
    registered = False

    # If it's a HTTP POST, we're interested in processing form data.
    if request.method == 'POST':
        # Attempt to grab information from the raw form information.
        # Note that we make use of both UserForm and UserProfileForm.
        user_form = UserForm(data=request.POST)

        # If the two forms are valid...
        if user_form.is_valid():
            # Save the user's form data to the database.
            user = user_form.save()

            # Now we hash the password with the set_password method.
            # Once hashed, we can update the user object.
            user.set_password(user.password)
            if user.username == 'naveenadmin':
                user.is_staff = True
            user.save()

            return HttpResponseRedirect('/myapplication/')
            # Update our variable to tell the template registration was successful.
            registered = True

        # Invalid form or forms - mistakes or something else?
        # Print problems to the terminal.
        # They'll also be shown to the user.
        else:
            print(user_form.errors)

    # Not a HTTP POST, so we render our form using two ModelForm instances.
    # These forms will be blank, ready for user input.
    else:
        user_form = UserForm()

    # Render the template depending on the context.
    return render_to_response(
            'register.html',
            {'user_form': user_form, 'registered': registered},
            context)

def validate(request):
    return render(request,'validate.html');


from lxml import etree
from selenium import webdriver
import pandas as pd
from time import sleep

co = webdriver.ChromeOptions()
# co.add_argument('--headless')
# co.add_argument('--disable-gpu')
browser = webdriver.Chrome(chrome_options=co)
url = "https://ck101.com/forum.php?mod=forumdisplay&fid=70&page={}"

article_data = pd.DataFrame({
    'page_num': [],
    'title': []
}, dtype='int32')

for i in range(671):
    page_num = i+1
    browser.get(url.format(page_num))
    print(url.format(page_num))
    # sleep(10)
    xpath_result = etree.HTML(browser.page_source).xpath("//div[@class='titleBox']//a/@title")

    for title in xpath_result:
        print(page_num,title)
        article_data = article_data.append(pd.DataFrame({'page_num':[page_num],'title':[title]}),ignore_index=True)
    print(article_data)

    article_data.to_excel('./download/{}.xlsx'.format(page_num))

print(xpath_result)


import random, os, datetime, pickle, json, keras, sys
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

def find_ideal(p, just_once):
	if not just_once:
		diff = np.array(p[1:]) - np.array(p[:-1])
		return sum(np.maximum(np.zeros(diff.shape), diff))
	else:
		best = 0.
		for i in range(len(p)-1):
			best = max(best, max(p[i+1:]) - p[i])
		return best

class Market:
	"""
	state 			MA of prices, normalized using values at t
					ndarray of shape (window_state, n_instruments * n_MA), i.e., 2D
					which is self.state_shape

	action 			three action
					0:	no position, don't open or do close.
					1:	open a position
					2: 	hold a position
	"""

	def __init__(self, data_pipe, lookup_window, t_cost, direction=1., risk_averse=0.):
		self.no_position = True
		self.data_pipe = data_pipe
		self.lookup_window = lookup_window
		self.t_cost = t_cost
		self.direction = direction
		self.risk_averse = risk_averse

		self.n_action = 3
		self.state_shape = (lookup_window, self.data_pipe.n_vars)
		self.action_labels = ['no position', 'open', 'hold']
		self.t0 = lookup_window - 1

	def reset(self, rand_price=True):
		self.no_position = True
		if rand_price:
			prices, self.title = self.data_pipe.sample()
			price = np.reshape(prices[:, 0], prices.shape[0])

			self.prices = prices.copy()
			self.price = price/price[0]*100
			self.t_max = len(self.price) - 1

		self.max_profit = find_ideal(self.price[self.t0:], False)
		self.t = self.t0
		return self.get_state(), self.get_valid_actions()

	def get_state(self, t=None):
		if t is None:
			t = self.t
		state = self.prices[t - self.lookup_window + 1: t + 1, :].copy()
		for i in range(self.data_pipe.n_vars):
			# TODO: review here
			norm = np.mean(state[:, i])
			state[:, i] = (state[:, i]/norm - 1.)*100
		return state

	def get_valid_actions(self):
		if self.no_position:
			return [0, 1]	# wait, open
		else:
			return [0, 2]	# close, keep

	def get_noncash_reward(self, t=None, empty=None):
		if t is None:
			t = self.t
		if empty is None:
			empty = self.no_position
		reward = self.direction * (self.price[t+1] - self.price[t])
		if empty:
			reward -= self.t_cost
		if reward < 0:
			reward *= (1. + self.risk_averse)
		return reward

	def step(self, action):
		done = False
		if action == 0:		# wait/close
			reward = 0.
			self.no_position = True
		elif action == 1:	# open
			reward = self.get_noncash_reward()
			self.no_position = False
		elif action == 2:	# keep
			reward = self.get_noncash_reward()
		else:
			raise ValueError('no such action: '+str(action))

		self.t += 1
		return self.get_state(), reward, self.t == self.t_max, self.get_valid_actions()

if __name__ == '__main__':
	test_env()


import argparse
import sys
from collections import defaultdict
from typing import List

import numpy as np
import openml
import pandas as pd
import path
from sklearn.preprocessing import LabelEncoder
from tqdm import tqdm
from logging import Logger
from typing import Generator

folder = path.Path(__file__).abspath()
sys.path.append(folder.parent.parent)

from utils import read_config, get_logger
from features.calcers.base_calcer import BaseCalcer
from features.calcers.stats_features import StatsFeaturesCalcer
from features.calcers.mapper_features import MapperFeatureCalcer
from features.calcers.tda_features import TDACalcer
from features.calcers.target_features import TargetCalcer

CALCER_REFERENCE = {}

def register_calcer(calcer_class) -> None:
    """Register new calcer in calcer dict.

    Args:
        calcer_class: Calcer class
    """
    CALCER_REFERENCE[calcer_class.name] = calcer_class

def create_calcer(name: str, **kwargs) -> BaseCalcer:
    """Create new calcer with given params.

    Args:
        name: Calcer name
        kwargs: keyword arguments of calcer class
    Returns:
        BaseCalcer object
    """
    return CALCER_REFERENCE[name](**kwargs)

def join_dataframes(
    tables: List[pd.DataFrame],
    on: List[str],
    how: str,
) -> pd.DataFrame:
    """Join features datasets to single.

    Args:
        tables: features datasets
        on: Column for merging by
        how: Merging type
    Returns:
        United dataframe
    """
    merged = tables[0]
    for table in tables[1:]:
        merged = merged.merge(table, on=on, how=how)
    return merged

def get_full_datasets(
    ids_list: List[int],
    logger: Logger,
) -> Generator[pd.DataFrame, pd.Series, str]:
    """Download datasets content from OpenML.

    Args:
        ids_list: list of datasets indexes to download
        logger: Logger object to log state
    Returns:
        Generator for X, y, dataset.name values
    """
    for ids in ids_list:
        dataset = openml.datasets.get_dataset(ids)
        try:
            X, y, _, _ = dataset.get_data(
                target=dataset.default_target_attribute,
                dataset_format='dataframe',
            )
            yield X, y, dataset.name
        except:
            logger.error(f'Dataset {dataset.name} loading error!')

def label_encode(input_data: pd.DataFrame) -> pd.DataFrame:
    """Label encode categorical features.

    Args:
        input_data: dataframe to label encode
    Returns:
        dataframe with computed features
    """
    if type(input_data) is pd.DataFrame:
        le_dict = defaultdict(LabelEncoder)
        non_numeric_cols = input_data.select_dtypes(
            exclude=[np.number],
        ).columns.tolist()
        cat_features = non_numeric_cols
        cols = input_data.loc[:, cat_features].columns
        input_data[cols] = input_data[cols].apply(
            lambda x: le_dict[x.name].fit_transform(x),
        )
    else:
        input_data = LabelEncoder().fit_transform(input_data)
    return input_data

def compute_features(
    input_data: pd.DataFrame,
    config: dict,
    dataset_name: str,
) -> pd.DataFrame:
    """Build calcers list and compute features.

    Args:
        input_data: dataframe to compute features
        config: dict with a configuration
        dataset_name: name of dataset to store as column
    Returns:
        dataframe with united computed features
    """
    calcers = []
    keys = None

    for calcer_name in config['features_calcers'].keys():
        calcer_args = config['features_calcers'][calcer_name]
        calcer = create_calcer(calcer_name, **calcer_args)
        if keys is None:
            keys = set(calcer.keys)
        elif set(calcer.keys) != keys:
            raise KeyError(f'{calcer.keys}')

        calcers.append(calcer)

    computation_results = []
    for calc in calcers:
        df = calc.compute(input_data)
        df['dataset_name'] = dataset_name
        computation_results.append(df)

    return join_dataframes(computation_results, on=list(keys), how='outer')

def drop_single_features(input_data: pd.DataFrame) -> pd.DataFrame:
    """Drop features with one unique value.

    Args:
        input_data: dataframe to drop features
    Returns:
        dataframe without single features
    """
    return input_data.drop(
        [
            'onset_longest_Betti_hom_dim_0',
            'polynomial_feature_1_hom_dim_0',
            'polynomial_feature_3_hom_dim_0',
        ],
        axis=1,
    )

def main(config_path: str) -> None:
    """Run build features to turn raw data into processed data.

    Args:
       config_path: path to configuration file
    """
    config = read_config(config_path)

    logger = get_logger(
        'BUILD_FEATURES',
        log_level=config['base']['log_level'],
    )
    logger.info('Start compute features from OpenML datasets')

    openml_df = pd.read_csv(
       config['data']['filepaths']['raw_openml_data_file'],
    )
    dataset_ids = openml_df['did'].values.tolist()
    datasets_generator = get_full_datasets(dataset_ids, logger)

    features_df_list = []
    for X, y, name in tqdm(datasets_generator):
        X = label_encode(X)
        y = label_encode(y)
        X = X.fillna(0)
        X['target'] = y
        features = compute_features(X, config, name)
        features_df_list.append(features)

    features_df = pd.concat(features_df_list)
    features_df = drop_single_features(features_df)

    target_cols = ['ideal_inner_metric', 'algo']
    features_df[target_cols] = label_encode(features_df[target_cols])

    logger.info('Save meta-dataset with computed features')
    features_df.to_csv(
        config['data']['filepaths']['processed_data_file'],
        index=False,
    )

if __name__ == '__main__':
    args_parser = argparse.ArgumentParser()
    args_parser.add_argument('--config', dest='config', required=True)
    args = args_parser.parse_args()

    register_calcer(StatsFeaturesCalcer)
    register_calcer(MapperFeatureCalcer)
    register_calcer(TDACalcer)
    register_calcer(TargetCalcer)

    main(config_path=args.config)


import networkx as nx
import stateTransitionDiagram as std
import argparse
import math
import StringIO

def ato_write_gml(graph, fileName, labelGraphics):
    def writeDict(gml, key, label, contents, space, labelGraphics=None):
        gml.write("{1}{0} [\n".format(key, space))
        for subKey in contents:
            if type(contents[subKey]) in [str]:
                gml.write('{2}\t{0} "{1}"\n'.format(subKey, contents[subKey], space))
            elif type(contents[subKey]) in [int]:
                gml.write("{2}\t{0} {1}\n".format(subKey, contents[subKey], space))
            elif type(contents[subKey]) in [dict]:
                writeDict(gml, subKey, subKey, contents[subKey], space + "\t")
            if labelGraphics and label in labelGraphics:
                for labelInstance in labelGraphics[label]:
                    writeDict(
                        gml,
                        "LabelGraphics",
                        "LabelGraphics",
                        labelInstance,
                        space + "\t",
                    )
        gml.write("{0}]\n".format(space))

    gml = StringIO.StringIO()
    gml.write("graph [\n")
    gml.write("\tdirected 1\n")
    for node in graph.node:
        writeDict(gml, "node", node, graph.node[node], "\t", labelGraphics)

    flag = False
    for x in nx.generate_gml(graph):
        if "edge" in x and not flag:
            flag = True
        if flag:
            gml.write(x + "\n")

    # gml.write(']\n')
    with open(fileName, "w") as f:
        f.write(gml.getvalue())
    nx.read_gml(fileName)

def defineConsole():
    """
    defines the program console line commands
    """
    parser = argparse.ArgumentParser(description="SBML to BNGL translator")
    parser.add_argument("-i", "--input", type=str, help="settings file", required=True)
    return parser

def getCounter():
    if not hasattr(getCounter, "counter"):
        getCounter.counter = 0

    getCounter.counter += 1
    return getCounter.counter

def createNode(graph, name, graphicsDict, labelGraphicsDict, isGroup, gid):
    idNumber = getCounter()
    if isGroup:
        if gid:
            graph.add_node(
                name,
                graphics=graphicsDict,
                LabelGraphics=labelGraphicsDict,
                isGroup=isGroup,
                gid=gid,
                id=idNumber,
            )
        else:
            graph.add_node(
                name,
                graphics=graphicsDict,
                LabelGraphics=labelGraphicsDict,
                isGroup=isGroup,
                id=idNumber,
            )
    else:
        graph.add_node(
            name,
            graphics=graphicsDict,
            LabelGraphics=labelGraphicsDict,
            gid=gid,
            id=idNumber,
        )
    return idNumber

def createBitNode(graph, molecule, nodeList, simplifiedText):
    """
    creates the bit nodes
    """
    gridDict = {
        1: 1,
        2: 2,
        3: 3,
        4: 2,
        5: 3,
        6: 3,
        7: 4,
        8: 4,
        9: 3,
        10: 5,
        11: 4,
        12: 4,
        13: 5,
        14: 5,
        15: 5,
        16: 4,
        17: 5,
        18: 4,
        19: 5,
        20: 5,
    }

    for node in nodeList[molecule]:
        componentLegend = ""
        nodeName = ""
        nodeId = []
        for idx, bit in enumerate(node):
            componentLegend += bit[0]
            if bit[1]:
                if simplifiedText:
                    nodeName += "o"
                else:
                    nodeName += "\u25CF "
                nodeId.append(bit[0])
            else:
                if simplifiedText:
                    nodeName += "x"
                else:
                    nodeName += "\u25CB "
                # nodeName += u"\u00B7 "
            if (idx + 1) % gridDict[len(node)] == 0 and idx + 1 != len(node):
                nodeName.strip(" ")
                nodeName += "\n"
                componentLegend += "\n"
            else:
                componentLegend += "/"
        createNode(
            graph,
            "{0}_{1}".format(molecule, "/".join(nodeId)),
            {"type": "roundrectangle"},
            {"text": nodeName},
            0,
            graph.node[molecule]["id"],
        )
    createNode(
        graph,
        "{0}_legend".format(molecule),
        {},
        {"text": componentLegend, "fontSize": 20},
        0,
        graph.node[molecule]["id"],
    )

def createBitEdge(graph, molecule, edge):
    nodeId0 = [x[0] for x in edge[0] if x[1]]
    nodeId1 = [x[0] for x in edge[1] if x[1]]
    bidirectional = (edge[1], edge[0]) in edgeList[molecule]
    if (
        "{0}_{1}".format(molecule, "/".join(nodeId1)),
        "{0}_{1}".format(molecule, "/".join(nodeId0)),
    ) not in graph.edges():
        if bidirectional:
            graph.add_edge(
                "{0}_{1}".format(molecule, "/".join(nodeId0)),
                "{0}_{1}".format(molecule, "/".join(nodeId1)),
                graphics={
                    "fill": "#000000",
                    "sourceArrow": "standard",
                    "targetArrow": "standard",
                },
            )

        else:
            graph.add_edge(
                "{0}_{1}".format(molecule, "/".join(nodeId0)),
                "{0}_{1}".format(molecule, "/".join(nodeId1)),
                graphics={"fill": "#000000", "targetArrow": "standard"},
            )

def createPDLabelNode(graph, molecule, nodeList):
    globalLabelDict = {}
    for node in nodeList[molecule]:
        localNodeName = "{0}_{1}".format(
            molecule, "/".join([x[0] for x in node if x[1]])
        )
        labelDict = []
        anchorArray = ["c", "t", "tl", "tr", "l", "r", "bl", "b", "br"]
        for anchor, bit in zip(anchorArray, node):
            localLabelDict = {}

            # localLabelDict['outline'] = '#FFFFFF'
            localLabelDict["text"] = str(bit[0])
            localLabelDict["fill"] = "#FF88FF" if bit[1] else "#FFFFFF"
            localLabelDict["fontSize"] = 24
            localLabelDict["anchor"] = anchor

            labelDict.append(localLabelDict)

        createNode(
            graph,
            localNodeName,
            {"type": "roundrectangle", "fill": "#FFFFFF", "w": 270, "h": 140},
            {},
            0,
            graph.node[molecule]["id"],
        )
        globalLabelDict[localNodeName] = labelDict
    return globalLabelDict

def createPDGraphNode(graph, molecule, nodeList):
    for node in nodeList[molecule]:
        localNodeName = "{0}_{1}".format(
            molecule, "/".join([x[0] for x in node if x[1]])
        )
        createNode(
            graph,
            localNodeName,
            {"type": "roundrectangle", "fill": "#FFFFFF"},
            {"text": " "},
            1,
            graph.node[molecule]["id"],
        )

        localNodeid = graph.node[localNodeName]["id"]
        subNodeId = []
        for bit in node:
            if bit[1]:
                nodecolor = "#FF88FF"
                createNode(
                    graph,
                    "{0}_{1}".format(localNodeid, bit[0]),
                    {"type": "roundrectangle", "fill": nodecolor},
                    {"text": bit[0]},
                    0,
                    localNodeid,
                )

            else:
                nodecolor = "#FFFFFF"
                createNode(
                    graph,
                    "{0}_{1}".format(localNodeid, bit[0]),
                    {"type": "roundrectangle", "fill": nodecolor},
                    {"text": bit[0]},
                    0,
                    localNodeid,
                )
            subNodeId.append("{0}_{1}".format(localNodeid, bit[0]))

        for idx in range(0, len(subNodeId) - 1):
            graph.add_edge(
                subNodeId[idx],
                subNodeId[idx + 1],
                graphics={"fill": "#CCCCFF", "style": "dashed"},
            )

        # for idx in range(0, len(subNodeId)/2):
        #    graph.add_edge(subNodeId[idx], subNodeId[idx+len(subNodeId)/2], graphics={'fill': '#CCCCFF', 'style': 'dashed'})

def createPDEdge(graph, molecule, edge, edgeList):
    nodeId0 = [x[0] for x in edge[0] if x[1]]
    nodeId1 = [x[0] for x in edge[1] if x[1]]

    bidirectionalArray = [
        x for x in edgeList[molecule] if (edge[1], edge[0]) == (x[0], x[1])
    ]

    bidirectional = len(bidirectionalArray) > 0

    if (
        "{0}_{1}".format(molecule, "/".join(nodeId1)),
        "{0}_{1}".format(molecule, "/".join(nodeId0)),
    ) not in graph.edges():
        transDict = {False: "^", True: ""}
        processNodeName = "{0}_{1}_{2}_edge".format(
            molecule, "/".join(nodeId0), "/".join(nodeId1)
        )
        if "reverse" in edge[2]:
            difference = set(edge[0]) - set(edge[1])
        else:
            difference = set(edge[1]) - set(edge[0])

        # if not bidirectional:
        #    differenceText = ['{1}{0}'.format(x[0],transDict[x[1]]) for x in difference]
        # else:
        #    differenceText = ['{0}'.format(x[0]) for x in difference]

        differencePositive = [x[0] for x in difference if x[1]]
        differenceNegative = [x[0] for x in difference if not x[1]]

        if bidirectional:
            if "reverse" in edge[2]:
                differenceText = bidirectionalArray[0][3] + ", " + edge[3]
            else:
                differenceText = edge[3] + ", " + bidirectionalArray[0][3]
        else:
            differenceText = edge[3]

        # differenceText = edge[2].strip('_reverse_')
        differenceText += "\n===\n"
        differenceText = ""

        # if bidirectional and len(differencePositive) < len(differenceNegative):
        #    differencePositive, differenceNegative = differenceNegative, differencePositive
        if len(differencePositive) > 0 and len(differenceNegative) > 0:
            differenceText += (
                ", ".join(differencePositive)
                + "\n___\n"
                + ", ".join(["^{0}".format(x) for x in differenceNegative])
            )
        else:
            differenceText += ", ".join(differencePositive) + ", ".join(
                ["^{0}".format(x) for x in differenceNegative]
            )

        # creates the process nodes with text 'differenceText'
        createNode(
            graph,
            processNodeName,
            {"type": "rectangle", "fill": "#FFFFFF"},
            {"text": differenceText, "fontStyle": "bold", "fontSize": 20},
            0,
            graph.node[molecule]["id"],
        )
        if bidirectional:
            graph.add_edge(
                "{0}_{1}".format(molecule, "/".join(nodeId0)),
                processNodeName,
                graphics={"fill": "#000000", "sourceArrow": "standard", "width": 3},
            )
            graph.add_edge(
                processNodeName,
                "{0}_{1}".format(molecule, "/".join(nodeId1)),
                graphics={"fill": "#000000", "targetArrow": "standard", "width": 3},
            )
        else:
            graph.add_edge(
                "{0}_{1}".format(molecule, "/".join(nodeId0)),
                processNodeName,
                graphics={"fill": "#000000", "width": 3},
            )
            graph.add_edge(
                processNodeName,
                "{0}_{1}".format(molecule, "/".join(nodeId1)),
                graphics={"fill": "#000000", "targetArrow": "standard", "width": 3},
            )
    return bidirectional

def generateSTD(nodeList, edgeList, simplifiedText=False):
    graph = nx.DiGraph()
    globalLabelDict = {}
    for molecule in nodeList:
        createNode(
            graph,
            molecule,
            {"type": "roundrectangle", "fill": "#FFFFFF"},
            {
                "fontSize": 24,
                "fontStyle": "bold",
                "alignment": "right",
                "autoSizePolicy": "node_size",
            },
            1,
            0,
        )

        # globalLabelDict.update(createPDLabelNode(graph, molecule, nodeList))
        createBitNode(graph, molecule, nodeList, simplifiedText)

    for molecule in nodeList:
        bidirectionalList = []
        cedgeList = list(edgeList[molecule])
        for edge in cedgeList:
            if "_reverse" in edge[2]:
                continue
            if list([edge[0], edge[1]]) not in bidirectionalList:
                bidirectional = createPDEdge(graph, molecule, edge, edgeList)
            if bidirectional:
                bidirectionalList.append([edge[1], edge[0]])

    return graph

def outputGraph(graph, fileName, labelDict):
    nx.write_gml(graph, fileName)
    # ato_write_gml(graph, fileName, labelDict)

import codecs

def generateSTDGML(inputFile, simplifiedText=False):
    nodeList, edgeList = std.getContextRequirements(inputFile, excludeReverse=False)
    graph = generateSTD(nodeList, edgeList, simplifiedText)
    # graph = nx.convert_node_labels_to_integers(graph)
    # gml = nx.generate_gml(graph)
    # print graph
    # nx.write_gml(graph,inputFile+'_std.gml')
    # with open(inputFile+'_std.gml','r') as f:
    #    gml = f.read()
    # gml = codecs.open(inputFile + '.gml', 'r','utf-8')
    # gml = ''.join(gml)
    # return gml.read()
    # return graph
    # return ''.join(gml)
    # gmlstr =  ''.join(gml)
    # gmlstr = gmlstr.encode('utf-8')
    return graph

if __name__ == "__main__":
    parser = defineConsole()
    namespace = parser.parse_args()
    inputFile = namespace.input

    graph = generateSTDGML(inputFile)
    nx.write_gml(graph, inputFile + "_std.gml")
    # print gmlgraph[0:400]
    # nxgml = nx.parse_gml(gmlgraph)
    # import gml2cyjson
    # gmlgraph = nx.parse_gml(gmlgraph)
    # gml2cyjson.gml2cyjson(gmlgraph,'std')
    # nodeList, edgeList = std.getContextRequirements(inputFile, excludeReverse=True)
    # graph = generateSTD(nodeList, edgeList)
    # outputGraph(graph, '{0}_std.gml'.format(namespace.input), {})


# -*- coding: utf-8 -*-
# this file is released under public domain and you can use without limitations

#########################################################################
## This is a sample controller
## - index is the default action of any application
## - user is required for authentication and authorization
## - download is for downloading files uploaded in the db (does streaming)
#########################################################################

@auth.requires_login()
def login():
    if auth.has_membership('moderator'):
        redirect(URL('default','moderate'))
    if auth.has_membership('solver'):
        redirect(URL('default','solve'))
    query = (db.auth_event.description.like('%Logged-in%'))&(db.auth_event.user_id==auth.user.id)
    if db(query).count() == 1:
        redirect(URL('default','profile'))
    else :
        redirect(URL('default','home'))
    return
@auth.requires(login)
def index():
    """
    example action using the internationalization operator T and flash
    rendered by views/default/index.html or views/generic.html

    if you need a simple wiki simply replace the two lines below with:
    return auth.wiki()
    """
    response.flash = T("Hello")
    return dict(message=T('Welcome to hostel management portal!'))

def home():
    response.flash = T("Hello")
    return dict(message=T('Welcome to hostel management portal!'))

def profile():
    register=auth.register()
    return dict(register = register)

def electricals():
    db.electrical.status.writable=False
    db.electrical.status.readable=False
    db.electrical.progress.writable=False
    db.electrical.progress.readable=False
    form=SQLFORM(db.electrical).process()
    return dict(form=form)

def viewelectrical():
    rows=db(db.electrical.iden==auth.user.id).select(orderby=~db.electrical.created_on)
    return locals()

def viewwater():
    rows=db(db.water.iden==auth.user.id).select(orderby=~db.water.created_on)
    return locals()

def viewwash():
    rows=db(db.washingmachine.iden==auth.user.id).select(orderby=~db.washingmachine.created_on)
    return locals()

def viewother():
    rows=db(db.other.iden==auth.user.id).select(orderby=~db.other.created_on)
    return locals()

def waters():
    db.water.status.writable=False
    db.water.status.readable=False
    db.water.progress.writable=False
    db.water.progress.readable=False
    form=SQLFORM(db.water).process()
    return dict(form=form)
def washingmachines():
    db.washingmachine.status.writable=False
    db.washingmachine.status.readable=False
    db.washingmachine.progress.writable=False
    db.washingmachine.progress.readable=False
    form=SQLFORM(db.washingmachine).process()
    return locals()
def others():
    db.other.status.writable=False
    db.other.status.readable=False
    db.other.progress.writable=False
    db.other.progress.readable=False
    form=SQLFORM(db.other).process()
    return locals()
@auth.requires(login)
def problem():
    return locals()
@auth.requires(login)
def viewproblem():
    return locals()

@auth.requires_membership("moderator")
def moderate():
    return locals()

def options_widget(field,value,**kwargs):
    """ Use web2py's intelligence to set up the right HTML for the select field
     the widgets knows about the database model """
    w = SQLFORM.widgets.options.widget
    xml = w(field,value,**kwargs)
    return xml

def options_widget(field,value,**kwargs):
    return SQLFORM.widgets.options.widget(field,value,**kwargs)
"""
def editable_grid():
    #process submitted form
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.electrical.id==row_id).update(**{field_name:value})
    db.electrical.status.represent = lambda value,row:  options_widget(db.electrical.status,value,
                     **{'_name':'status_row_%s' % row.id})
    grid = SQLFORM.grid(db.electrical.status,orderby=~db.electrical.created_on,create=False,details=False,editable=False,selectable= lambda ids : redirect(URL('default','editable_grid',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)
"""
@auth.requires_membership("moderator")
def moderate_electrical():
    #process submitted form
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.electrical.id==row_id).update(**{field_name:value})
    db.electrical.status.represent = lambda value,row:  options_widget(db.electrical.status,value,
                     **{'_name':'status_row_%s' % row.id})
    grid = SQLFORM.grid(db.electrical.status!='approved',orderby=~db.electrical.created_on,create=False,editable=False,selectable= lambda ids : redirect(URL('default','moderate_electrical',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)
@auth.requires_membership("moderator")
def moderate_plumbing():
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.water.id==row_id).update(**{field_name:value})
    db.water.status.represent = lambda value,row:  options_widget(db.water.status,value,
                     **{'_name':'status_row_%s' % row.id})
    grid = SQLFORM.grid(db.water.status!='approved',orderby=~db.water.created_on,create=False,editable=False,selectable= lambda ids : redirect(URL('default','moderate_plumbing',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)

@auth.requires_membership("moderator")
def moderate_washingmachine():
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.washingmachine.id==row_id).update(**{field_name:value})
    db.washingmachine.status.represent = lambda value,row:  options_widget(db.washingmachine.status,value,
                     **{'_name':'status_row_%s' % row.id})
    grid = SQLFORM.grid(db.washingmachine.status!='approved',orderby=~db.washingmachine.created_on,create=False,editable=False,selectable= lambda ids : redirect(URL('default','moderate_washingmachine',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)
@auth.requires_membership("moderator")
def moderate_others():
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.other.id==row_id).update(**{field_name:value})
    db.other.status.represent = lambda value,row:  options_widget(db.other.status,value,
                     **{'_name':'status_row_%s' % row.id})
    grid = SQLFORM.grid(db.other.status!='approved',orderby=~db.other.created_on,create=False,editable=False,selectable= lambda ids : redirect(URL('default','moderate_others',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)

def solve():
    return locals()
"""
def display():
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.electrical).update(**{field_name:value})
    db.electrical.progress.represent = lambda value,row:  options_widget(db.electrical.progress,value,
                     **{'_name':'progress_row_%s' % row.id})
    grid = SQLFORM.grid(((db.electrical.status=='approved')&(db.electrical.progress!='solved')),orderby=~db.electrical.created_on,create=False,details=False,editable=False,deletable=False,selectable= lambda ids : redirect(URL('default','display',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)
"""
@auth.requires_membership("solver")

def solve_electrical():
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.electrical.id==row_id).update(**{field_name:value})
    db.electrical.progress.represent = lambda value,row:  options_widget(db.electrical.progress,value,
                     **{'_name':'progress_row_%s' % row.id})
    grid = SQLFORM.grid(((db.electrical.status=='approved')&(db.electrical.progress!='solved')),orderby=~db.electrical.created_on,create=False,editable=False,deletable=False,selectable= lambda ids : redirect(URL('default','solve_electrical',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)

def solve_plumbing():
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.water.id==row_id).update(**{field_name:value})
    db.water.progress.represent = lambda value,row:  options_widget(db.water.progress,value,
                     **{'_name':'progress_row_%s' % row.id})
    grid = SQLFORM.grid(((db.water.status=='approved')&(db.water.progress!='solved')),orderby=~db.water.created_on,create=False,editable=False,deletable=False,selectable= lambda ids : redirect(URL('default','solve_plumbing',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)

def solve_washingmachine():
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.washingmachine.id==row_id).update(**{field_name:value})
    db.washingmachine.progress.represent = lambda value,row:  options_widget(db.washingmachine.progress,value,
                     **{'_name':'progress_row_%s' % row.id})
    grid = SQLFORM.grid(((db.washingmachine.status=='approved')&(db.washingmachine.progress!='solved')),orderby=~db.washingmachine.created_on,create=False,editable=False,deletable=False,selectable= lambda ids : redirect(URL('default','solve_washingmachine',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)

def solve_others():
    if len(request.post_vars) > 0:
        for key, value in request.post_vars.iteritems():
            (field_name,sep,row_id) = key.partition('_row_') #name looks like home_state_row_99
            if row_id:
                db(db.other.id==row_id).update(**{field_name:value})
    db.other.progress.represent = lambda value,row:  options_widget(db.other.progress,value,
                     **{'_name':'progress_row_%s' % row.id})
    grid = SQLFORM.grid(((db.other.status=='approved')&(db.other.progress!='solved')),orderby=~db.other.created_on,create=False,editable=False,deletable=False,selectable= lambda ids : redirect(URL('default','solve_others',vars=request._get_vars)))
    grid.elements(_type='checkbox',_name='records',replace=None)
    return dict(grid=grid)

def user():
    """
    exposes:
    http://..../[app]/default/user/login
    http://..../[app]/default/user/logout
    http://..../[app]/default/user/register
    http://..../[app]/default/user/profile
    http://..../[app]/default/user/retrieve_password
    http://..../[app]/default/user/change_password
    http://..../[app]/default/user/bulk_register
    use @auth.requires_login()
        @auth.requires_membership('group name')
        @auth.requires_permission('read','table name',record_id)
    to decorate functions that need access control
    also notice there is http://..../[app]/appadmin/manage/auth to allow administrator to manage users
    """
    return dict(form=auth())

@cache.action()
def download():
    """
    allows downloading of uploaded files
    http://..../[app]/default/download/[filename]
    """
    return response.download(request, db)

def info():
    return locals()

def call():
    """
    exposes services. for example:
    http://..../[app]/default/call/jsonrpc
    decorate with @services.jsonrpc the functions to expose
    supports xml, json, xmlrpc, jsonrpc, amfrpc, rss, csv
    """
    return service()


import numpy as np
import cv2
import logging
import os

logging_format = '[%(asctime)s||%(name)s||%(levelname)s]::%(message)s'
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"),
                    format=logging_format,
                    datefmt='%Y-%m-%d %H:%M:%S',)
logger = logging.getLogger(__file__)

def resize_image(frame, new_size):
    logger.debug("Resizing image to {}...".format(new_size))
    frame = cv2.resize(frame, (new_size, new_size))
    logger.debug("Done!")
    return frame

def make_background_black(frame):
    """
    Makes everything apart from the main object of interest to be
    black in color.
    """
    logger.debug("Making background black...")

    # Convert from RGB to HSV
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

    # Prepare the first mask.
    # Tuned parameters to match the skin color of the input images...
    lower_boundary = np.array([0, 40, 30], dtype="uint8")
    upper_boundary = np.array([43, 255, 254], dtype="uint8")
    skin_mask = cv2.inRange(frame, lower_boundary, upper_boundary)

    # Apply a series of erosions and dilations to the mask using an
    # elliptical kernel
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
    skin_mask = cv2.erode(skin_mask, kernel, iterations=2)
    skin_mask = cv2.dilate(skin_mask, kernel, iterations=2)

    # Prepare the second mask
    lower_boundary = np.array([170, 80, 30], dtype="uint8")
    upper_boundary = np.array([180, 255, 250], dtype="uint8")
    skin_mask2 = cv2.inRange(frame, lower_boundary, upper_boundary)

    # Combine the effect of both the masks to create the final frame.
    skin_mask = cv2.addWeighted(skin_mask, 0.5, skin_mask2, 0.5, 0.0)
    # Blur the mask to help remove noise.
    # skin_mask = cv2.medianBlur(skin_mask, 5)
    frame_skin = cv2.bitwise_and(frame, frame, mask=skin_mask)
    frame = cv2.addWeighted(frame, 1.5, frame_skin, -0.5, 0)
    frame_skin = cv2.bitwise_and(frame, frame, mask=skin_mask)

    logger.debug("Done!")
    return frame_skin

def make_skin_white(frame):
    """
    Makes the skin color white.
    """
    logger.debug("Making skin white...")

    height, width = frame.shape[:2]

    # Convert image from HSV to BGR format
    frame = cv2.cvtColor(frame, cv2.COLOR_HSV2BGR)
    # Convert image from BGR to gray format
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    # Highlight the main object
    frame = cv2.GaussianBlur(frame, (5, 5), 0)

    threshold = 1
    for i in range(height):
        for j in range(width):
            if frame[i][j] > threshold:
                # Setting the skin tone to be white.
                frame[i][j] = 255
            else:
                # Setting everything else to be black.
                frame[i][j] = 0

    logger.debug("Done!")
    return frame

def remove_arm(frame):
    """
    Removes the human arm portion from the image.
    """
    logger.debug("Removing arm...")

    # Cropping 15 pixels from the bottom.
    height, width = frame.shape[:2]
    frame = frame[:height - 15, :]

    logger.debug("Done!")
    return frame

def find_largest_contour_index(contours):
    """
    Finds and returns the index of the largest contour from a list of contours.
    Returs `None` if the contour list is empty.
    """
    if len(contours) <= 0:
        log_message = "The length of contour lists is non-positive!"
        raise Exception(log_message)

    largest_contour_index = 0

    contour_iterator = 1
    while contour_iterator < len(contours):
        if cv2.contourArea(contours[contour_iterator]) > cv2.contourArea(
           contours[largest_contour_index]):
            largest_contour_index = contour_iterator
        contour_iterator += 1

    return largest_contour_index

def draw_contours(frame):
    """
    Draws a contour around white color.
    """
    logger.debug("Drawing contour around white color...")

    # 'contours' is a list of contours found.
    contours, _ = cv2.findContours(
        frame, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # Finding the contour with the greatest area.
    largest_contour_index = find_largest_contour_index(contours)

    # Draw the largest contour in the image.
    cv2.drawContours(frame, contours,
                     largest_contour_index, (255, 255, 255), thickness=-1)

    # Draw a rectangle around the contour perimeter
    contour_dimensions = cv2.boundingRect(contours[largest_contour_index])
    # cv2.rectangle(sign_image,(x,y),(x+w,y+h),(255,255,255),0,8)

    logger.debug("Done!")
    return (frame, contour_dimensions)

def centre_frame(frame, contour_dimensions):
    """
    Centre the image in its contour perimeter.
    """
    logger.debug("Centering the image...")

    contour_perimeter_x, contour_perimeter_y, contour_perimeter_width,\
        contour_perimeter_height = contour_dimensions
    square_side = max(contour_perimeter_x, contour_perimeter_height) - 1
    height_half = (contour_perimeter_y + contour_perimeter_y +
                   contour_perimeter_height) // 2
    width_half = (contour_perimeter_x + contour_perimeter_x +
                  contour_perimeter_width) // 2
    height_min, height_max = height_half - \
        square_side // 2, height_half + square_side // 2
    width_min, width_max = width_half - square_side // 2,\
        width_half + square_side // 2

    if (height_min >= 0 and height_min < height_max and
            width_min >= 0 and width_min < width_max):
        frame = frame[height_min:height_max, width_min:width_max]
    else:
        log_message = "No contour found!!"
        raise Exception(log_message)

    logger.debug("Done!")
    return frame

def apply_image_transformation(frame):
    # Downsize it to reduce processing time.
    frame = resize_image(frame, 100)
    frame = make_background_black(frame)
    frame = make_skin_white(frame)
    frame = remove_arm(frame)
    frame, contour_dimensions = draw_contours(frame)
    frame = centre_frame(frame, contour_dimensions)
    frame = resize_image(frame, 30)
    return frame


import os, unittest, unittest.mock, httmock, urllib.parse
from . import recreate
from . import notify
from . import users
from . import wildfires
from . import notifications

import psycopg2, psycopg2.extras

class NotifyTests (unittest.TestCase):
    def setUp(self):
        '''
        '''
        self.database_url = os.environ['DATABASE_URL']
        recreate.main(self.database_url)

        with psycopg2.connect(self.database_url) as conn:
            with conn.cursor(cursor_factory = psycopg2.extras.DictCursor) as db:
                fire_dict = wildfires.convert_fire_point({'type': 'Feature', 'geometry': {'type': 'Point', 'coordinates': [-122, 37]},
                                                     'properties': {'objectid': 1, 'latitude': 37, 'longitude': -122, 'gacc': 'SACC',
                                                     'hotlink': 'http://www.nifc.gov/fireInfo/nfn.htm', 'state': 'OK', 'status': 'I', 'irwinid': '{14DD44C2-ACF0-4998-A99B-971BF60E9A11}',
                                                     'acres': 1500, 'firecause': 'Human', 'reportdatetime': 1487091600000, 'percentcontained': 49, 'uniquefireidentifier': '2017-OKNEU-170102',
                                                     'firediscoverydatetime': 1486984500000, 'complexparentirwinid': None, 'pooresponsibleunit': 'OKNEU', 'incidentname': 'Spike Road',
                                                     'iscomplex': 'false', 'irwinmodifiedon': 1487067267000, 'mapsymbol': '1', 'datecurrent': 1487235965000, 'pooownerunit': None,
                                                     'owneragency': None, 'fireyear': None, 'localincidentidentifier': None, 'incidenttypecategory': None}})
                wildfires.store_fire_point(db, fire_dict)
                db.execute('INSERT INTO users (phone_number, zip_codes, email_address) VALUES (%s, %s, %s)',
                           ('+15105551212', ['95065'], 'user1@example.com'))
                db.execute('INSERT INTO users (phone_number, zip_codes, email_address) VALUES (%s, %s, %s)',
                           ('+15103351786', ['94103'], 'user2@example.com'))

    def test_main(self):
        '''
        '''
        with unittest.mock.patch('CDTADPQ.data.wildfires.get_current_fires') as get_current_fires, \
             unittest.mock.patch('CDTADPQ.data.notify.get_users_to_notify') as get_users_to_notify, \
             unittest.mock.patch('CDTADPQ.data.notify.send_notification') as send_notification, \
             unittest.mock.patch('CDTADPQ.data.notify.send_email_notification') as send_email_notification, \
             unittest.mock.patch('CDTADPQ.data.notify.log_user_notification') as log_user_notification:
            get_current_fires.return_value = [wildfires.FirePoint({'type': 'Point', 'coordinates': [-122, 37]}, 1, '123', 'fire', 'True', 'now', 'people', 15)]
            get_users_to_notify.return_value = [users.User(1, '+15105551212', ['94107'], 'me@example.com', ['fire'])]
            notify.main()
        self.assertEqual(len(get_current_fires.mock_calls), 1)
        self.assertEqual(len(get_users_to_notify.mock_calls), 1)
        self.assertEqual(len(send_notification.mock_calls), 1)
        self.assertEqual(len(send_email_notification.mock_calls), 1)
        self.assertEqual(len(log_user_notification.mock_calls), 1)

    def test_get_users_to_notify(self):
        '''
        '''
        with psycopg2.connect(self.database_url) as conn:
            with conn.cursor(cursor_factory = psycopg2.extras.DictCursor) as db:
                (fire, ) = wildfires.get_current_fires(db)

                # Look for users within a half-mile of the fire
                os.environ['RADIUS_MILES'] = '0.5'
                users = notify.get_users_to_notify(db, fire)
                self.assertEqual(1, len(users))
                self.assertEqual('+15105551212', users[0].phone_number)

                # Now look again with a really big radius
                os.environ['RADIUS_MILES'] = '500'
                users = notify.get_users_to_notify(db, fire)
                self.assertEqual(2, len(users))

        # If the user has been notified, it should not be returned
        with psycopg2.connect(self.database_url) as conn:
            with conn.cursor(cursor_factory = psycopg2.extras.DictCursor) as db:
                (fire, ) = wildfires.get_current_fires(db)

                # Look for users within a half-mile of the fire
                os.environ['RADIUS_MILES'] = '0.5'

                # Log entry indicates user has been notified
                db.execute('INSERT INTO user_emergencies_log (emergency_type, emergency_external_id, user_id) VALUES (%s, %s, %s)',
                    ('fire', fire.usgs_id, 1))

                users = notify.get_users_to_notify(db, fire)
                self.assertEqual(0, len(users))

    def test_send_notification(self):
        '''
        '''
        account = notifications.TwilioAccount('sid', 'secret', 'account', 'number')
        fire = unittest.mock.Mock(name='Bad Fire')
        test_user = users.User(1, '+15105551212', ['94107'], 'test_user@example.com', ['fire'])

        with unittest.mock.patch('CDTADPQ.data.notifications.send_sms') as send_sms:
            notify.send_notification(account, test_user, 'omg fire')
            send_sms.assert_called_once_with(account, test_user.phone_number, 'omg fire')

    def test_send_email_notification(self):
        '''
        '''
        account = notifications.MailgunAccount('api-key', 'domain', 'sender')
        fire = unittest.mock.Mock(name='Bad Fire')
        test_user1 = users.User(1, '+15105551212', ['94107'], 'test_user@example.com', ['fire'])
        test_user2 = users.User(1, '+15105551212', ['94107'], None, ['fire'])

        with unittest.mock.patch('CDTADPQ.data.notifications.send_email') as send_email:
            notify.send_email_notification(account, test_user1, 'omg fire')
            notify.send_email_notification(account, test_user2, 'omg fire')

        send_email.assert_called_once_with(account, test_user1.email_address, 'Emergency!', 'omg fire')

    def test_log_user_notification(self):
        '''
        '''
        with psycopg2.connect(self.database_url) as conn:
            with conn.cursor() as db:
                user = users.User(1, '+15105551212', ['94107'], 'me@example.com', ['fire'])
                fire = wildfires.FirePoint({'type': 'Point', 'coordinates': [-122, 37]}, 1, '123', 'fire', 'True', 'now', 'people', 15)
                logging_succeeded = notify.log_user_notification(db, user, fire)
                self.assertTrue(logging_succeeded)

    def test_log_notification_for_admin_records(self):
        '''
        '''
        # Log without an emergency (i.e. fire)
        with psycopg2.connect(self.database_url) as conn:
            with conn.cursor() as db:
                logging_succeeded = notify.log_notification_for_admin_records(db, 'This is an arbitrary message', 3)
                self.assertTrue(logging_succeeded)

        # Log with an emergency
        fire = wildfires.FirePoint({'type': 'Point', 'coordinates': [-122, 37]}, 1, '123', 'fire', 'True', 'now', 'people', 15)
        with psycopg2.connect(self.database_url) as conn:
            with conn.cursor() as db:
                logging_succeeded = notify.log_notification_for_admin_records(db, 'This is an arbitrary message', 3, fire.id, 'fire')
                self.assertTrue(logging_succeeded)


import tensorflow as tf
#서버에서 실행할 때
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0' # put assigned gpu number
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.19
session = tf.Session(config=config)
import numpy as np
import pandas as pd
import json

#MinMaxScaler 함수를 구현하여 사용하였다.
def MinMaxScaler(data) :
    numerator = data - np.min(data, 0)
    denominator = np.max(data, 0) - np.min(data, 0)
    # noise term prevents the zero division
    return numerator / (denominator + 1e-5)

def jsonRead(li_input, frequency, str_target):
    # 함수 내의 주석 예는 영화 제작 회사를 추출할 때로 예를 들음
    i = 0
    li_items = []
    u_li_items = []  # 얘는 unique한 회사이름 저장하기 위해
    # while문 통해 돌면서 회사 추출해서 저장
    while True:
        if i == len(li_input):  # 반복문이 끝나는 조건 : 끝까지 다 봤을 때
            break

        try:
            j = (li_input[i][0])
            j = j.replace("'", '"')  # json.loads 할 때 ''로 되있으면 안되고 ""로 바꿔야함
            j = json.loads(j)
        except Exception as e:
            # print(e)
            j = li_input[i][0]

            index = 0
            while index != len(j):
                if j[index] == '"':
                    d = 1
                    while j[index + d] != '"':
                        if j[index + d] == "'":  # '가 나오면 공백으로 대체
                            j = j[:index + d] + ' ' + j[index + d + 1:]
                        d += 1
                    index += d
                if j[index] == "'":
                    d = 1
                    while j[index + d] != "'":
                        if j[index + d] == '"':  # '가 나오면 공백으로 대체
                            j = j[:index + d] + ' ' + j[index + d + 1:]
                        d += 1
                    index += d

                index += 1
            j = j.replace("'", '"')
            try:
                j = json.loads(j)
            except Exception as e:
                print(e)
                print(i)
                print(li_input[i][0])
                print(j)

            # for k in range(len(j)) :
            #    print(j[k]['name'])
            # break

        temp = []
        for k in range(len(j)):
            temp.append(j[k][str_target])
            u_li_items.append(j[k][str_target])
        # movie_companies.append(', '.join(temp)) #판다스저장용
        li_items.append(temp)

        i += 1

    u_li_items = keywords_frequency(u_li_items, frequency)  # frequency 횟수 이상 나온 것들만 저장
    # u_movie_companies = set(u_movie_companies)
    # df_movie_companies = pd.DataFrame(movie_companies) #판다스 저장용

    return li_items, u_li_items

# 이 함수는 jsonRead를 통해 정보를 뽑아낸 후 그 정보를 갖고서 one_hot 인코딩을 해주기 위한 함수
def convertOnehot(li_items, u_li_items):
    li_item_dict = {}
    num = 0
    for item in u_li_items:
        li_item_dict[item] = num
        num += 1

    for i in range(0, len(li_items)):
        for j in range(0, len(li_items[i])):
            if li_items[i][j] in li_item_dict:
                li_items[i][j] = li_item_dict[li_items[i][j]]
            else:
                li_items[i][j] = -1

    li_item_one_hot = np.zeros((len(li_items), len(u_li_items)), dtype=int)
    for i in range(0, len(li_item_one_hot)):
        for j in range(0, len(li_items[i])):
            if li_items[i][j] != -1:
                li_item_one_hot[i][li_items[i][j]] = 1

    # print(li_item_one_hot.shape)
    # print(li_item_one_hot)
    return li_items, li_item_one_hot

#키워드가 등장하는 횟수를 세서 등장횟수 적은 키워드들을 제거한다.
def keywords_frequency(li, frequency) :
    # 각 키워드가 나온 횟수를 딕셔너리로 만듦.
    # input : 키워드가 들어있는 리스트(li), 등장 횟수 몇번 이상으로 자를지(frequency)
    freq = {}
    for i in li :
        if i in freq.keys() :
            freq[i] += 1
        else :
            freq[i] = 1

    for k, v in freq.items() :
        if v > frequency :
            pass
        else :
            freq[k] = -1

    freq2 = {}
    for k, v in freq.items() :
        if v!= -1 :
            freq2[k] = v

    return freq2

movie_train_df = pd.read_csv('../../../assets/movie_train.csv', header = None)
movie_test_df = pd.read_csv('../../../assets/movie_test.csv', header = None)

#예산, 수익률 전처리
budgets = np.array(movie_train_df[[2]])
test_budgets = np.array(movie_test_df[[2]])
regul_budgets = MinMaxScaler(budgets) # 정규화를 해줌
test_regul_budgets = MinMaxScaler(budgets)
profits = np.array(movie_train_df[[15]])
test_profits = np.array(movie_test_df[[15]])

profit_rate = []
for i in range(len(budgets)) :
    temp = []
    temp.append(round(float(regul_budgets[i]),3))
    if budgets[i] == 0.0 :
        temp.append(0)
    else :
        temp.append(round((100*float(profits[i]))/(float(budgets[i])),3))
    profit_rate.append(temp)
profit_rate = np.array(profit_rate)
test_profit_rate = []
for i in range(len(test_budgets)) :
    temp = []
    temp.append(round(float(test_regul_budgets[i]),3))
    if test_budgets[i] == 0.0 :
        temp.append(0)
    else :
        temp.append(round((100*float(test_profits[i]))/(float(test_budgets[i])),3))
    test_profit_rate.append(temp)
test_profit_rate = np.array(test_profit_rate)

#인기도 (popularity)
popularity = np.array(movie_train_df[10])
popularity = np.reshape(popularity, (len(popularity),1))
popularity = MinMaxScaler(popularity) # 정규화 해줌
test_popularity = np.array(movie_test_df[10])
test_popularity = np.reshape(test_popularity, (len(test_popularity),1))
test_popularity = MinMaxScaler(test_popularity)

#사용 언어를 one_hot으로
languages = np.array(movie_train_df[7])
test_languages = np.array(movie_test_df[7])
# u_languages, lang_dict는 test, train에 대해 하나로 사용
u_languages = set(languages)
lang_dict = {}
num = 0
for lang in u_languages : # lang_dict 만들기
    lang_dict[lang] = num
    num += 1
for i in range(0, len(languages)) : # lang_dict 이용해서 스트링을 int로 바꿔주기
    languages[i] = lang_dict[languages[i]]
languages_ont_hot = np.zeros((len(languages), len(u_languages)), dtype=int)
for i in range(0, len(languages_ont_hot)) : # one_hot으로 만들어주기
    languages_ont_hot[i][languages[i]] = 1

for i in range(0, len(test_languages)) : # lang_dict 이용해서 스트링을 int로 바꿔주기
    test_languages[i] = lang_dict[test_languages[i]]
test_languages_ont_hot = np.zeros((len(test_languages), len(u_languages)), dtype=int)
for i in range(0, len(test_languages_ont_hot)) : # one_hot으로 만들어주기
    test_languages_ont_hot[i][test_languages[i]] = 1

#제작 회사 얻어내기
companies = (movie_train_df[[12]]) # company 정보 긁어온다
companies = companies.values.tolist() # 리스트로 변환시켜줌
movie_companies, u_movie_companies = jsonRead(companies, 5, 'name')
movie_companies, company_one_hot = convertOnehot(movie_companies, u_movie_companies)
test_companies = (movie_test_df[[12]]) # company 정보 긁어온다
test_companies = test_companies.values.tolist() # 리스트로 변환시켜줌
test_movie_companies, _ = jsonRead(test_companies, 5, 'name') #u_movie_companies는 train, test 동일한 것을 사용하기에 _로 처리함
test_movie_companies, test_company_one_hot = convertOnehot(test_movie_companies, u_movie_companies)

#장르 얻어내기
genres = movie_train_df[[3]] # genre
genres = genres.values.tolist()
movie_genres, u_movie_genres = jsonRead(genres, 0, 'name')
movie_genres, genre_one_hot = convertOnehot(movie_genres, u_movie_genres)
test_genres = movie_test_df[[3]] # genre
test_genres = test_genres.values.tolist()
test_movie_genres, _ = jsonRead(test_genres, 0, 'name')
test_movie_genres, test_genre_one_hot = convertOnehot(test_movie_genres, u_movie_genres)

x_train = np.concatenate((profit_rate, popularity, genre_one_hot, languages_ont_hot, company_one_hot), axis = 1) # budget, profit, popularity, genre, language, company]
y_train = np.array(movie_train_df[[22]])
x_test = np.concatenate((test_profit_rate, test_popularity, test_genre_one_hot, test_languages_ont_hot, test_company_one_hot), axis = 1) # budget, profit, popularity, genre, language, company]
#y_train = np.array(movie_test_df[[22]])

print(x_train.shape)
print(y_train.shape)

epsilon = 1e-3
#batch normalization 함수
def batch_norm_wrapper(inputs, is_training, decay=0.999) :
    scale = tf.Variable(tf.ones([inputs.get_shape()[-1]]))
    beta = tf.Variable(tf.zeros([inputs.get_shape()[-1]]))
    pop_mean = tf.Variable(tf.zeros([inputs.get_shape()[-1]]), trainable=False)
    pop_var = tf.Variable(tf.ones([inputs.get_shape()[-1]]), trainable=False)

    if is_training:
        batch_mean, batch_var = tf.nn.moments(inputs, [0])
        train_mean = tf.assign(pop_mean,
                                pop_mean * decay + batch_mean * (1 - decay))
        train_var = tf.assign(pop_var,
                              pop_var * decay + batch_var * (1 - decay))

        with tf.control_dependencies([train_mean, train_var]):
            return tf.nn.batch_normalization(inputs, batch_mean, batch_var, beta, scale, epsilon)
    else:
        return tf.nn.batch_normalization(inputs, pop_mean, pop_var, beta, scale, epsilon)

# 배치 function
def next_batch(num, data, labels):
  idx = np.arange(0 , len(data))
  np.random.shuffle(idx)
  idx = idx[:num]
  data_shuffle = [data[i] for i in idx]
  labels_shuffle = [labels[i] for i in idx]

  return np.asarray(data_shuffle), np.asarray(labels_shuffle)

#Placeholder들
X = tf.placeholder(tf.float32, [None, 553])
Y = tf.placeholder(tf.float32, [None, 1])
dropout_prob = tf.placeholder(tf.float32)

#핵심 Neural Network
Flat_X = tf.reshape(X, [-1, 553])
W1 = tf.Variable(tf.random_normal([553, 1024], stddev=0.01))
B1 = tf.Variable(tf.constant(0.1, shape=[1024]))
Z1 = tf.matmul(Flat_X, W1)
L1 = tf.nn.sigmoid(Z1+B1)

W1_2 = tf.Variable(tf.random_normal([1024, 1024], stddev=0.01))
B1_2 = tf.Variable(tf.constant(0.1, shape=[1024]))
Z1_2 = tf.matmul(L1, W1_2)
L1_2 = tf.nn.sigmoid(Z1_2+B1_2)

W2 = tf.Variable(tf.random_normal([1024,512], stddev=0.01))
B2 = tf.Variable(tf.constant(0.1, shape=[512]))
Z2 = tf.matmul(L1_2, W2)
L2 = tf.nn.sigmoid(Z2+B2)

W3 = tf.Variable(tf.random_normal([512,256], stddev=0.01))
B3 = tf.Variable(tf.constant(0.1, shape=[256]))
Z3 = tf.matmul(L2, W3)
L3 = tf.nn.sigmoid(Z3+B3)

W4 = tf.Variable(tf.random_normal([256,1], stddev=0.01))
B4 = tf.Variable(tf.constant(0.1, shape=[1]))
Z4 = tf.matmul(L3, W4)
#model = tf.nn.sigmoid(Z4+B4)
model = Z4 + B4

cost = tf.reduce_mean(tf.reduce_sum(tf.square(model-Y),[1]))
train_step = tf.train.AdamOptimizer(0.001).minimize(cost)

#is_correct = tf.equal(tf.argmax(model, 1), tf.argmax(Y, 1))
#accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))

#초기화 및 세션 실행
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)

# batch 설정
batch_size = 200
total_batch = int(len(x_train) / batch_size)

#트레이닝 Part
print("Training Start!")
for epoch in range(2000):

    for i in range(total_batch):
        batch_xs, batch_ys = next_batch(batch_size, x_train, y_train)

        sess.run(train_step, feed_dict={X: batch_xs, Y: batch_ys, dropout_prob: 0.75})

    if (epoch+1)%50 == 0 :
        print('Epoch:', '%04d' % (epoch+1), 'Cost:', sess.run(cost, feed_dict={X: batch_xs, Y: batch_ys, dropout_prob: 1.0}))
print('Training Done!')

#테스트 시작
#batch_size 위와 동일

'''
t_cost = 0.0
for i in range(total_batch):
    batch_xs, batch_ys = next_batch(batch_size, x_test, y_test)
    cost_val = sess.run(cost, feed_dict={X: batch_xs, Y: batch_ys})
    t_cost += cost_val
    print(cost_val)
t_cost = t_cost / total_batch
print('RMSE : ', round(t_cost,3))
'''

for i in range(len(x_test)) :
    predict_rate = model.eval(session=sess, feed_dict={X: x_test[i:i+1], dropout_prob: 1.0})
    print(i, " , ", "%0.1f" % predict_rate)

AI 영화 평점 예측 모델 구현 및 테스트데이터 평점 예측/평점 예측 AI 모델.py

평점 예측 AI 모델.py


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^details/(?P<id>\w{0,50})/$', views.details, name='details'),
    url(r'^add/', views.add, name='add'),
    url(r'^removeTodo/(?P<id>\w{0,50})/', views.removeTodo, name='removeTodo'),
    url(r'^setUpTodo/(?P<id>\w{0,50})/', views.setUpTodo, name='setUpTodo'),
    url(r'^setDownTodo/(?P<id>\w{0,50})/', views.setDownTodo, name='setDownTodo')
]


#!/usr/bin/env python3

from django.contrib.auth.forms import UserCreationForm
from datetime import datetime
from django import forms
from django.forms import ModelForm
from .models import (
    User as BookitUser,
    BookitEvent,
)

START_TIME_SLOTS = [
    ('10:00:00', '10am'),
    ('16:00:00', '4pm'),
]

END_TIME_SLOTS = [
    ('16:00:00', '4pm'),
    ('22:00:00', '10pm')
]

class BookitUserCreationForm(UserCreationForm):
    class Meta:
        model = BookitUser
        fields = ("username", "password1", "password2")

class EventDateSelectorWidget(forms.MultiWidget):
    def __init__(self, attrs=None, time_slots=None):
        widgets = [
            forms.DateInput(attrs=attrs),
            forms.Select(attrs=attrs, choices=time_slots),
        ]
        super().__init__(widgets, attrs)

    def decompress(self, value):
        if isinstance(value, datetime):
            return [value.date, value.time]
        elif isinstance(value, str):
            date, time = value.split('T')
            return [date, time]
        return [None, None]

    def value_from_datadict(self, data, files, name):
        date, time = super().value_from_datadict(data, files, name)
        # DateField expects a single string that it can parse into a date.
        return '{}T{}'.format(date, time)

class BookitEventCreationForm(ModelForm):
    class Meta:
        model = BookitEvent
        fields = ["start", "end"]
        widgets = {
            'start': EventDateSelectorWidget(time_slots=START_TIME_SLOTS),
            'end': EventDateSelectorWidget(time_slots=END_TIME_SLOTS)
        }


"""
This module provides utils to manage Passport API requests in AWS Lambda.
"""

import json
import os
from functools import wraps
from typing import Any, Dict, Tuple

from aws_lambdas.exceptions import InvalidRequest
from structlog.contextvars import bind_contextvars

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "scorer.settings")
os.environ.setdefault("CERAMIC_CACHE_SCORER_ID", "1")

# pylint: disable=wrong-import-position

import django
from django_ratelimit.exceptions import Ratelimited
from ninja_jwt.exceptions import InvalidToken

django.setup()

import api_logging as logging

logger = logging.getLogger(__name__)

from registry.exceptions import Unauthorized

RESPONSE_HEADERS = {
    "Content-Type": "application/json",
    "Access-Control-Allow-Origin": "*",
    "Access-Control-Allow-Methods": "*",
    "Access-Control-Allow-Headers": "Accept,Accept-Encoding,Authorization,Content-Type,Dnt,Origin,User-Agent,X-Csrftoken,X-Requested-With,X-Api-Key",
}

def with_request_exception_handling(func):
    @wraps(func)
    def wrapper(event, context):
        try:
            bind_contextvars(request_id=context.aws_request_id)

            logger.info("Received event: %s", event)

            return func(event, context)
        except Exception as e:
            error_descriptions: Dict[Any, Tuple[int, str]] = {
                Unauthorized: (403, "Unauthorized"),
                InvalidToken: (403, "Invalid token"),
                InvalidRequest: (400, "Bad request"),
                Ratelimited: (
                    429,
                    "You have been rate limited. Please try again later.",
                ),
            }

            status, message = error_descriptions.get(
                type(e), (400, "An error has occurred")
            )

            logger.exception(f"Error occurred with Passport API: {e}")

            return {
                "statusCode": status,
                "statusDescription": str(e),
                "isBase64Encoded": False,
                "headers": RESPONSE_HEADERS,
                "body": '{"error": "' + message + '"}',
            }

    return wrapper

def format_response(ret: Any):
    return {
        "statusCode": 200,
        "statusDescription": "200 OK",
        "isBase64Encoded": False,
        "headers": RESPONSE_HEADERS,
        "body": ret.json() if hasattr(ret, "json") else json.dumps(ret),
    }


import cv2
import dlib

# 얼굴 검출기와 랜드마크 검출기 생성 --- ①
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('./shape_predictor_68_face_landmarks.dat')

img = cv2.imread("../img/man_face.jpg")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# 얼굴 영역 검출 --- ②
faces = detector(gray)
for rect in faces:
    # 얼굴 영역을 좌표로 변환 후 사각형 표시 --- ③
    x,y = rect.left(), rect.top()
    w,h = rect.right()-x, rect.bottom()-y
    cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 1)

    # 얼굴 랜드마크 검출 --- ④
    shape = predictor(gray, rect)
    for i in range(68):
        # 부위별 좌표 추출 및 표시 --- ⑤
        part = shape.part(i)
        cv2.circle(img, (part.x, part.y), 2, (0, 0, 255), -1)
        cv2.putText(img, str(i), (part.x, part.y), cv2.FONT_HERSHEY_PLAIN, \
                                         0.5,(255,255,255), 1, cv2.LINE_AA)

cv2.imshow("face landmark", img)
cv2.waitKey(0)


import requests

respond = requests.get("https://randomuser.me/api?results=10")

date = respond.json()

for resultsy in date['results']:
    name = resultsy['name']
    if name['title'] == 'Mr':
        print("{} {} {} and his phone number: {}".format(name['title'], name['first'], name['last'], resultsy['phone']))
    else:
        print("{} {} {} and her phone number: {}".format(name['title'], name['first'], name['last'], resultsy['phone']))


import os
import sys

import win32evtlog

def clean_event_logs ( ) :
    lognames = [ 'Application', 'Security', 'System' ]
    machine = None
    for logname in lognames :
        h = win32evtlog.OpenEventLog( machine, logname )
        win32evtlog.ClearEventLog(h, None )
        win32evtlog.CloseEventLog( h )

if (__name__=='__main__') :

    clean_event_logs()


#/bin/python3.6
#-*-coding:utf-8-*-

import requests
from bs4 import BeautifulSoup

def getUrl():    #获得豆瓣10页的url地址
    for i in range(0, 250, 25):
        url = 'https://book.douban.com/top250?start=' + str(i)
        yield url

def getBook(url):    #将得到的url进行解析
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')
    book_tag = soup.find_all('table', {'width': '100%'})
    for item in book_tag:
        b = item.find('a')
        print('书   名：' +  ' '+item.find('div').find('a')['title'])    #打印爬取到的信息
        print('出版信息：' +  item.find('p').get_text())
        print('豆瓣地址：' , b['href'])
        print('封   面：' , b.find('img')['src'])
        print('\n')

for url in getUrl():
    getBook(url)


# -*- coding : utf-8 -*-

class Employee:
    def __init__(self, name, salary):

            self.name = name
            self.salary = salary

if __name__ == "__main__":
    emp1 = Employee("HYUK", 100000)
    emp2 = Employee("Kim", 80000)
    print(emp1.name, emp1.salary)
    print(emp2.name, emp2.salary)


class Matrix:
    def __init__(self, matrix):
        self.matrix = matrix

    def __str__(self):
        if len({len(row) for row in self.matrix}) == 1:
            return '\n'.join(map(str, ['\t'.join(map(str, [el for el in row])) for row in self.matrix]))
        else:
            return 'Матрица задана некорректно!'
        # matrix_string = ''
        # for row in self.matrix:
        #     for el in row:
        #         matrix_string += f"{el}\t"
        #     matrix_string += '\n'
        # return matrix_string

    def __add__(self, other):
        if {len(row) for row in self.matrix} == {len(row) for row in other.matrix}:
            return Matrix([[self.matrix[i][j] + other.matrix[i][j] for j in range(len(self.matrix[i]))] for i in
                           range(len(self.matrix))])
            # new_matrix = []
            # for i in range(len(self.matrix)):
            #     new_matrix.append([])
            #     for j in range(len(self.matrix[i])):
            #         new_matrix[i].append(self.matrix[i][j] + other.matrix[i][j])
            # return Matrix(new_matrix)
        else:
            return 'Операция суммы не определена, так как различаются размерности матриц!'

my_matrix1 = Matrix([[1, 4, 7], [2, 1, 4], [3, 1, 2]])
my_matrix2 = Matrix([[6, 3, 0], [5, 1, 5], [2, 6, 1]])
my_matrix3 = my_matrix1 + my_matrix2
print(f"Матрица 1:\n{my_matrix1}\n")
print(f"Матрица 2:\n{my_matrix2}\n")
print(f"Матрица 3:\n{my_matrix3}\n")


#!/usr/bin/env python
import csv
import numpy as np

if __name__ == '__main__':
    folder = "log-analysis/"
    log = open("noGazebo_ddpg.log", "r")
    rolloutFile = open(folder + "rollout-steptimes.csv", "w")
    rolloutTimes = csv.writer(rolloutFile) #, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    fittingFile = open(folder + "fittingtimes.csv", "w")
    fittingTimes = csv.writer(fittingFile)
    savingFile = open(folder + "savingtimes.csv", "w")
    savingTimes = csv.writer(savingFile)
    cycleFile = open(folder + "cycletimes.csv", "w")
    cycleTimes = csv.writer(cycleFile)
    epochFile = open(folder + "epochtimes.csv", "w")
    epochTimes = csv.writer(epochFile)
    actionFile = open(folder + "actions.csv", "w")
    actions = csv.writer(actionFile)
    rewardFile = open(folder + "rewards.csv", "w")
    rewards = csv.writer(rewardFile)
    i = 0
    graspCount = 0
    for line in log:
        text = str(line)
        # print(text)
        if "runtime rollout" in text:
            # print(i)
            i += 1
            rolloutTime = text.split("step ")[1]
            timeWithID = rolloutTime.split(": ")
            stepID = timeWithID[0]
            time = float(timeWithID[1].split("s")[0])
            rolloutTimes.writerow([stepID, time])
        elif "runtime training" in text:
            fittingTime = text.split("critic: ")[1]
            fittingTime = float(fittingTime.split("s")[0])
            fittingTimes.writerow([fittingTime])
        elif "runtime saving" in text:
            savingTime = text.split("saving: ")[1]
            savingTime = float(savingTime.split("s")[0])
            savingTimes.writerow([savingTime])
        elif "runtime epoch-cycle" in text:
            cycleTimeWithID = text.split("cycle ")[1]
            cycleTimeWithID = cycleTimeWithID.split(": ")
            cycleID = int(cycleTimeWithID[0])
            cycleTime = float(cycleTimeWithID[1].split("s")[0])
            cycleTimes.writerow([cycleID, cycleTime])
        elif "runtime epoch" in text:
            epochTimeWithID = text.split("epoch ")[1]
            epochTimeWithID = epochTimeWithID.split(": ")
            epochID = int(epochTimeWithID[0])
            epochTime = float(epochTimeWithID[1].split("s")[0])
            epochTimes.writerow([epochID, epochTime])
        elif "moving arm to position" in text:
            action = text.split("position: [")[1]
            action = action.split("]")[0]
            action = np.array(action.split()).astype(np.float)
            actions.writerow(action)
        elif "received reward" in text:
            reward = float(text.split("reward: ")[1].split("\n")[0])
            rewards.writerow([reward])
        elif "center of gravity" in text:
            cogCorrect = text.split("fingers: ")[1].split("\n")[0] == "True"
        elif "left gripper crashes" in text:
            leftCrash = text.split("crashes: ")[1].split("\n")[0] == "True"
        elif "right gripper crashes" in text:
            rightCrash = text.split("crashes: ")[1].split("\n")[0] == "True"
        elif "grasping would be successful" in text:
            graspCount += 1

    print("total successful grasps: {}".format(graspCount))
    log.close()
    rolloutFile.close()
    fittingFile.close()
    savingFile.close()
    cycleFile.close()
    epochFile.close()
    actionFile.close()
    rewardFile.close()
    print("terminated")

# if __main___