# -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Instituto de Desarrollo Urbano (<http://www.idu.gov.co>). All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, osv
import gitlab
from openerp import SUPERUSER_ID
from datetime import datetime
from datetime import timedelta

def get_connection_gitlab(param_model, cr):
    token = param_model.get_param(cr, SUPERUSER_ID, 'gitlab.token', default=False)
    host = param_model.get_param(cr, SUPERUSER_ID, 'gitlab.host', default=False)
    c = gitlab.Gitlab(host, token)
    return c

class res_users(osv.osv):
    _inherit = "res.users"
    _name = "res.users"

    _columns = {
        'gitlab_id': fields.integer('GitLab ID'),
    }

class gitlab_issue(osv.osv):
    _inherit = [
        'mail.thread',
    ]

    def _get_url(self, cr, uid, ids, field, args, context=None):
        res = {}
        records = self.browse(cr, uid, ids, context=context)
        for record in records:
            res[record.id] = "{0}/issues/{1}".format(record.project_id.url, record.gitlab_idd)
        return res

    def _get_color(self, cr, uid, ids, field, args, context=None):
        res = {}
        records = self.browse(cr, uid, ids, context=context)
        for record in records:
            color = 0
            if record.state == 'closed':
                color = 1
            elif record.state == 'reopened':
                color = 2
            res[record.id] = color
        return res

    _name = "gitlab.issue"
    _order = "sequence, name, id"
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1, track_visibility='onchange',),
        'sequence': fields.integer('Sequence', help="Gives the sequence order when displaying a list."),
        'due_date': fields.datetime('Due Date', track_visibility='onchange',),
        'start_date': fields.datetime('Start Date', track_visibility='onchange',),
        'finish_date': fields.datetime('Finish Date', track_visibility='onchange',),
        'state': fields.selection(
            [('opened','Open'), ('closed','Closed'), ('reopened','Re-Opened')],
            'Estado',
            required=True,
            track_visibility='onchange',
        ),
        'gitlab_id': fields.integer('GitLab ID', track_visibility='onchange',),
        'gitlab_idd': fields.integer('GitLab idd', track_visibility='onchange',),
        'stage_id': fields.many2one('gitlab.stage', 'Stage', track_visibility='onchange',),
        'milestone_id': fields.many2one('gitlab.milestone', 'Milestone', track_visibility='onchange',),
        'project_id': fields.many2one('gitlab.project', 'Project', track_visibility='onchange',),
        'label_ids': fields.many2many('gitlab.label', 'gitlab_issue_label', 'issue_id', 'label_ids', 'Labels'),
        'user_id': fields.many2one('res.users', 'User', track_visibility='onchange',),
        'task_id': fields.many2one('project.task', 'Task', track_visibility='onchange',),
        'url': fields.function(
            _get_url,
            type="char",
            string='GitLab URL',
            store=False,
        ),
        'kanban_color': fields.function(
            _get_color,
            type="integer",
            string='Kanban Color',
            store=True,
        ),
    }

    def kanban_columns(self, cr, uid, ids, domain, read_group_order=None, access_rights_uid=None, context=None):
        stage_model = self.pool.get('gitlab.stage')
        column_ids = stage_model.search(cr, uid, [], context=context)
        columns = stage_model.name_get(cr, uid, column_ids, context=context)
        return columns, {}

    _group_by_full = {
        'stage_id': kanban_columns,
    }

    def set_due_date(self, cr, uid, ids, *args):
        """Set due date tomorrow
        """
        tomorrow = datetime.now() + timedelta(days=1)
        return self.write(cr, uid, ids, {'due_date': tomorrow.strftime("%Y-%m-%d %H:%M:%S")})

    def write(self, cr, uid, ids, vals, context=None):
        if len(vals) == 1 and (vals.get('stage_id') or vals.get('user_id')):
            gitlab_conn = get_connection_gitlab(self.pool.get('ir.config_parameter'), cr)
            if vals.get('stage_id'):
                stage = self.pool.get('gitlab.stage').browse(cr, uid, vals.get('stage_id'), context=context)
                status = stage.set_issue_status
                for issue in self.browse(cr, uid, ids, context=context):
                    labels = 'col:' + stage.name
                    for label in issue.label_ids:
                        labels = labels + ',' + label.name
                    res = gitlab_conn.editissue(issue.project_id.gitlab_id, issue.gitlab_id, labels = labels, state_event = status)
                    if res:
                        vals['state'] = res['state']
                        if status == 'close' and not issue.finish_date:
                            vals['finish_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                        if status == 'reopen' and not issue.start_date:
                            vals['start_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                        if status == 'reopen':
                            vals['finish_date'] = False

            if vals.get('user_id'):
                user = self.pool.get('res.users').browse(cr, uid, vals.get('user_id'), context=context)
                for issue in self.browse(cr, uid, ids, context=context):
                    res = gitlab_conn.editissue(issue.project_id.gitlab_id, issue.gitlab_id, assignee_id = user.gitlab_id)
        return super(gitlab_issue, self).write(cr, uid, ids, vals, context)

class gitlab_stage(osv.osv):
    _name = "gitlab.stage"
    _order = 'sequence ASC, name ASC'
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1),
        'sequence': fields.integer('Sequence', help="Gives the sequence order when displaying a list."),
        'issue_ids': fields.one2many('gitlab.issue', 'stage_id', 'Issues'),
        'set_issue_status': fields.selection(
            [('close','Close'), ('reopen','Re-Open')],
            'Issue set status to',
        ),
    }

class gitlab_project(osv.osv):
    _name = "gitlab.project"
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1),
        'namespace': fields.char('Namespace', size=255, required=True, select=1),
        'url': fields.char('url', size=255),
        'gitlab_id': fields.integer('GitLab ID'),
        'issue_ids': fields.one2many(
            'gitlab.issue',
            'project_id',
            'Issues',
       ),
    }

class gitlab_milestone(osv.osv):
    _name = "gitlab.milestone"
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1),
        'gitlab_id': fields.integer('GitLab ID'),
        'gitlab_idd': fields.integer('GitLab idd'),
        'issue_ids': fields.one2many(
            'gitlab.issue',
            'milestone_id',
            'Issues',
       ),
    }

class gitlab_label(osv.osv):
    _name = "gitlab.label"
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1),
        'issue_ids': fields.many2many('gitlab.issue', 'gitlab_issue_label', 'label_ids','issue_id','Issues'),
    }

"""
@startuml
Title gitlab integración openerp
class gitlab.issue
class gitlab.milestone
class gitlab.project

class project.task
class project.project

class res.user

gitlab.issue "*" -up- res.user
gitlab.issue "*" -- project.task
gitlab.milestone -- "*" gitlab.issue
gitlab.project -- "*" gitlab.issue
gitlab.issue "*" -- "*" gitlab.label

@enduml

"""


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
from framework.utils.log import *

def XETRaise(e):
    """
    XETRaise will logging the exception error information，and then raise the exception
    """
    depth = 0
    callstackinfo = ""
    while True:
        try:
            frame = sys._getframe(depth)
            code = frame.f_code
            callstackinfo = callstackinfo + "\nframe:" + str(depth) + " \"" + code.co_filename + "\" Line:" + str(
                frame.f_back.f_lineno)
            depth = depth + 1
        except:
            Logger.CL.error("Raise exception:" + str(e) + "\n" + "Call trace:" + callstackinfo)
            raise e


import csv
import numpy as np
import itertools
import operator
import json
from multiprocessing import Process
import datetime

with open('/Users/nishant/Downloads/mea_ml_train.csv', 'r') as f:
    reader = csv.reader(f)
    mea_fml = list(reader)

## Can add as many parameters as you want to
param_1 = []
param_2 = []
param_3 = []
param_4 = []
# param_5 = []
# param_6 = []
# param_7 = []
# param_8 = []
# param_9 = []
# param_10 = []
target = []

for each in mea_fml[1:]:
    param_1.append(each[1])
    param_2.append(each[2])
    param_3.append(each[3])
    param_4.append(each[4])
    # param_5.append(each[5])
    # param_6.append(each[6])
    # param_7.append(each[7])
    # param_8.append(each[8])
    # param_9.append(each[9])
    # param_10.append(each[10])
    target.append(each[0])

param_list = [param_1, param_2, param_3, param_4]

comparator_dict = {}
measurements_match = {}

for i in range(3, 5):
    comb_list = list(itertools.combinations(param_list, i))
    for each in comb_list:
        measurements_match_list = []
        key = ""
        list_1 = []
        index_list = []
        for element in each:
            index_list.append(param_list.index(element))
            list_1.append(element)
        list_1.append(target)
        length = len(list_1)

        final_arr = np.array(list_1)
        final_arr_trans = np.transpose(final_arr)
        counter = 0
        counter_not = 0
        index_list_new = []

        for index, each in enumerate(final_arr_trans):
            for element in each:
                if float(element) < 10 or float(element) > 60:
                    index_list_new.append(index)
        final_arr_trans_clean = np.delete(final_arr_trans, index_list_new, 0)
        # print(final_arr_trans_clean)

        ## Clustering of profiles with same input set and a tolerance of 1 in output
        for i, each in enumerate(final_arr_trans_clean):
            if list(each) not in measurements_match_list:
                measurements_match_list.append(list(each))
            for element in final_arr_trans_clean[i + 1:, ...]:
                if (list(each)[0:-1] == list(element)[0:-1]) and (
                        abs(float(list(each)[-1]) - float(list(element)[-1])) < 1):
                    if list(element) not in measurements_match_list:
                        measurements_match_list.append(list(element))
                    counter += 1
                elif (list(each)[0:-1] == list(element)[0:-1]) and (
                        abs(float(list(each)[-1]) - float(list(element)[-1])) > 1):
                    counter_not += 1
                else:
                    pass

        # for i, each in enumerate(final_arr_trans_clean):
        #     for element in final_arr_trans_clean[i+1:,...]:
        #         if list(each)[1:] == list(element)[1:] and list(each)[0] != list(element)[0]:
        #             counter_not += 1

        # counter_row = 0
        # for j in range(0,length-1):
        #     if (each[j] == element[j]):
        #         counter_row += 1
        # if counter_row == length-1 and each[-1] != element[-1]:
        #     counter_not += 1

        ## User profile list for different combination of input parameters
        for each in index_list:
            key = key + str(each) + "-"
            comparator_dict[key] = [counter, counter_not]
            measurements_match[key] = measurements_match_list

sorted_comparator_dict = sorted(comparator_dict.items(), key=lambda t: t[1][0])
# print(sorted_comparator_dict)

with open("match.json", 'w') as outfile:
    json.dump(measurements_match, outfile)


import scrapy
from locations.items import GeojsonPointItem
import json
import re

regex_times = r"\d{1,2}:\d{1,2}\s?[Aa]?[Mm]?[Pp]?[Mm]?\s?-\s?\d{1,2}:\d{1,2}" \
              r"\s?[Pp]?[Mm]?[Aa]?[Mm]?"
regex_am = r"\s?[Aa][Mm]"
regex_pm = r"\s?[Pp][Mm]"

#  Covers United States, Canada, UK, Puerto Rico, Bahamas with 500 mile radius
lats = ['32.806671', '31.054487',  '39.059811', '46.921925', '45.694454',
        '38.039119', '44.045876', '32.593106', '33.596319', '47.398349',
        '24.44715', '18.229351', '19.725342', '64.014496', '51.563412',
        '52.48278', '55.27911529201562', '55.17886766328199',
        '63.15435519659188', '52.96187505907603']

lons = ['-86.791130', '-97.563461', '-105.311104', '-110.454353', '-93.900192',
        '-87.618638', '-72.710686', '-82.342529', '-113.334961', '-121.289062',
        '-78.00293', '-65.830078', '-155.610352', '-153.28125' '-86.923828',
        '-65.126953', '-103.974609375', '-120.76171875', '-136.142578125',
        '-0.17578125']

class RegisSpider(scrapy.Spider):
    name = 'regis'
    item_attributes = { 'brand': "Regis Salon" }
    download_delay = 0
    allowed_domains = ['www.regissalons.com']
    start_urls = ['https://www.regissalons.com']

    def convert_hours(self, hours):
        if not hours:
            return ''
        for i in range(len(hours)):
            converted_times = ''
            if 'Closed' not in hours[i]:
                from_hr, to_hr = [hr.strip() for hr in hours[i].split('-')]
                if re.search(regex_am, from_hr):
                    from_hr = re.sub(regex_am, '', from_hr)
                    hour_min = from_hr.split(':')
                    if len(hour_min[0]) < 2:
                        hour_min[0] = hour_min[0].zfill(2)
                    converted_times += (":".join(hour_min)) + ' - '
                else:
                    from_hr = re.sub(regex_pm, '', from_hr)
                    hour_min = from_hr.split(':')
                    if int(hour_min[0]) < 12:
                        hour_min[0] = str(12 + int(hour_min[0]))
                    converted_times += (":".join(hour_min)) + ' - '

                if re.search(regex_am, to_hr):
                    to_hr = re.sub(regex_am, '', to_hr)
                    hour_min = to_hr.split(':')
                    if len(hour_min[0]) < 2:
                        hour_min[0] = hour_min[0].zfill(2)
                    if int(hour_min[0]) == 12:
                        hour_min[0] = '00'
                    converted_times += (":".join(hour_min))
                else:
                    to_hr = re.sub(regex_pm, '', to_hr)
                    hour_min = to_hr.split(':')
                    if int(hour_min[0]) < 12:
                        hour_min[0] = str(12 + int(hour_min[0]))
                    converted_times += (":".join(hour_min))
            else:
                converted_times += "off"
            hours[i] = converted_times

        days = ['Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su']
        hours = ''.join(' {} {} '.format(*t) for t in zip(days, hours))
        return hours

    def parse_store(self, response):
        results = json.loads(response.body_as_unicode())
        for i in results:
            store_id = i['id']
            name = i['address']
            website = i['permalink'].replace('\\', '')
            street = i['address2']
            city = i['city']
            state = i['state']
            postcode = i['zip']
            addr_full = "{} {}, {} {}".format(street, city, state, postcode)
            country = i['country']
            lat = i['lat']
            lon = i['lng']
            hours = re.findall(regex_times, i['hours'])
            hours = self.convert_hours(hours)

            yield GeojsonPointItem(
                ref=store_id,
                name=name,
                website=website,
                street=street,
                city=city,
                state=state,
                postcode=postcode,
                country=country,
                addr_full=addr_full,
                lat=lat,
                lon=lon,
                opening_hours=hours
            )

    def parse(self, response):
        for x, y in zip(lats, lons):
            base_url = 'https://www.regissalons.com/wp-admin/admin-ajax.php?action=store_search&max_results=100&search_radius=500&'
            base_url += "lat={}&lng={}".format(x, y)
            yield scrapy.Request(base_url, callback=self.parse_store)


# -*- coding: utf-8 -*-

import xmltodict

from django.core.urlresolvers import reverse

from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase

class FeedTest(ModuleStoreTestCase):
    def setUp(self):
        super(FeedTest, self).setUp()
        self.unpublished_course = CourseFactory(org='fun', number='001', display_name=u"unpublished", ispublic=False)
        self.published_course = CourseFactory(org='fun', number='002', display_name=u"published", ispublic=True)
        self.url = reverse('fun-courses-feed')

    def test_feed(self):
        response = self.client.get(self.url)
        self.assertEqual(200, response.status_code)
        data = xmltodict.parse(response.content)
        self.assertEqual(u"Fun latest published courses", data['rss']['channel']['title'])
        self.assertEqual(1, len(data['rss']['channel']['item']))
        self.assertEqual(u"published", data['rss']['channel']['item'][0]['title'])


from tkinter import Tk, FALSE, Button
import tkinter
from tkinter import *
from tkinter.messagebox import *
from Table_py import *
from PIL import Image, ImageTk #pip install pillow
from time import sleep

# Arseny: too many global variables
# Arseny: you will got problem on debug
# Arseny: pack your variables in one class or object for state
form1 = tkinter.Tk()
form1.title('form1')
form1.resizable(width=FALSE, height=FALSE)
form1.geometry('1300x662+100+100')
image = Image.open("image//suit.jpg")
suit = ImageTk.PhotoImage(image)

# Arseny: buf1, buf2, buf3 what is this?
# Arseny: the names of variables don't mean anything
# Arseny: I can't reach their purpose looking on the name. It's wrong
buf1 = StringVar()
buf2 = StringVar()
buf3 = StringVar()
buf1.set('YOUR NAME')
user_name=server_port = tkinter.Entry(form1,textvariable=buf1)
user_name.place(relx=0.20, rely=0.05, relwidth=0.07, relheight=0.03)
buf2.set('IP')
user_ip=server_port = tkinter.Entry(form1,textvariable=buf2)
user_ip.place(relx=0.30, rely=0.05, relwidth=0.06, relheight=0.03)
buf3.set('PORT')
server_port = tkinter.Entry(form1,textvariable=buf3)
server_port.place(relx=0.40, rely=0.05, relwidth=0.04, relheight=0.03)
user_connect = Button(text='connect', command=connect_click)
user_connect.place(relx=0.45, rely=0.05, relwidth=0.04, relheight=0.03)

def add_card_on_center(master, delta):
        bank = Button(text='bank of casino')
        bank.place(relx=0.3, rely=0.25, relwidth=0.09, relheight=0.03)
        card_server = Button(text='card')
        card_server.place(relx=delta+0.02, rely=0.31, relwidth=0.04, relheight=0.13)
        cards_center.append(card_server)
        return cards_center

def place_for_user(user, x, y):
    #suit = ImageTk.PhotoImage('lena.ppm')
    bank = Button(text='bank of '+user)
    bank.place(relx=x, rely=y, relwidth=0.09, relheight=0.03)

    card1 = Button(text='card1', image=suit)
    card1.place(relx=x, rely=y+0.04, relwidth=0.04, relheight=0.13)

    card2 = Button(text='card2', image=suit)
    card2.place(relx=x+0.05, rely=y+0.04, relwidth=0.04, relheight=0.13)

    bet_bank = Button(text='bet', state='disable')
    bet_bank.place(relx=x+0.11, rely=y+0.02, relwidth=0.04, relheight=0.03)

    raise_bet = Button(text='raise bet', command=raise_bet_click, state='disable')
    raise_bet.place(relx=x+0.11, rely=y+0.06, relwidth=0.04, relheight=0.03)

    pass_button = Button(text='pass', command=pass_button_click, state='disable')
    pass_button.place(relx=x+0.11, rely=y+0.1, relwidth=0.04, relheight=0.03)

    check = Button(text='check', command=check_click, state='disable')
    check.place(relx=x+0.11, rely=y+0.14, relwidth=0.04, relheight=0.03)

    user = Button(text='user '+user)
    user.place(relx=x, rely=y+0.19, relwidth=0.09, relheight=0.03)

    # Arseny: why do you use `dict` for scructs in your code?
    # Arseny: you should use object for this
    return {'bank': bank, 'raise_bet': raise_bet, 'bet': bet_bank, 'card1': card1,
            'card2': card2, 'check': check, 'user': user}

# Arseny: you have section global variables on the top of code
# Arseny: and then new section there. Thats hard to read and debug
cards_center = []
croupier = {'bank': 0, 'cards': cards_center}
player_places = []
size = 0.25
delta_center = 0
cards_center = add_card_on_center(form1, size+delta_center)

position = 3
shift = 0

x_default = 0.02
y_default = 0.75
user_name = {'1': 'player 1', '2': 'user 2', '3': 'player 3', '4': 'player 4', '5': 'player 5', '6': 'player 6', '7':
                    'player 7' }

player_places.append(place_for_user(user_name.get('1'), x_default, y_default-0.6))
player_places.append(place_for_user(user_name.get('2'), x_default, y_default-0.3))
player_places.append(place_for_user(user_name.get('3'), x_default, y_default))
player_places.append(place_for_user(user_name.get('4'), x_default+0.3, y_default))
player_places.append(place_for_user(user_name.get('5'), x_default+0.6, y_default))
player_places.append(place_for_user(user_name.get('6'), x_default+0.6, y_default-0.3))
player_places.append(place_for_user(user_name.get('7'), x_default+0.6, y_default-0.6))

#example for add some cards for the second player
# this wil be dict, where number of range of card will be map to face's card

# Arseny: bad names again. Why not `suite_bking`? Why `suit3`?
image = Image.open("image//king_black.jpg")
suit3 = ImageTk.PhotoImage(image)
image = Image.open("image//five_red.jpg")
suit4 = ImageTk.PhotoImage(image)

player_places[1].get('card1').config(image=suit3)
player_places[1].get('card2').config(image=suit4)
card = (croupier.get('cards'))[0]
card.config(image=suit3)

form1.mainloop()

#case with add new card in dynamic
#this code will be work only with new thred
# will be tinking
sleep(0.01)
cards_center = add_card_on_center(form1, size+0.02)
card = (croupier.get('cards'))[1]
card.config(image=suit3)
print('ok')
form1.mainloop()


from myoperator import BlockOperator

class BestInformativeHit(BlockOperator):
        """
        Uses best-hit method to predict DE and GO.

        Select first hit with noninformative words from filtered hitlist.
        Select first hit with GO annotations from filtered hitlist.

        Creates summary table columns 'qpid', 'bits', 'desc' (copied from selected row of data)
        Creates goclass table columns 'qpid','bits','ontology','goid','desc' (copied from selected row of data)

        Inputs: data columns 'status', 'bits', 'qpid', 'desc', 'spid'
        """
        def __init__(self, glob):
                self.glob=glob
                [self.data,self.summary_data,self.goclass_data]=glob.use_sheets(["data","summary","goclass"])
                [self.qpid_col1,self.bits_col1,self.desc_col1,self.status_col1,self.spid_col1,self.ff_col1,self.goclass_col]=self.data.use_columns(['qpid','bits','desc','status','spid','FF','GOclass'])
                self.summary_data.use_columns(['qpid','bits','desc'])
                self.goclass_data.use_columns(['qpid','bits','ontology','goid','desc'])
                glob.use_online_dictionaries(['GOIDELIC'])
                [self.b,self.c,self.d]=glob.use_operators(['Filter','FF','GOrimpsu'])
                # parameter nicknames
                self.FFCUTOFF=glob.param['PANZ_FFCUTOFF']

        def process(self,block):
                # get query
                try:
                        qpid=block[0][self.qpid_col1]
                except:
                        return
                # preprocess
                for row in block:
                        self.c.process(row) # calculate form factor
                        self.d.process(row) # associate GO terms
                self.b.process(block) # set status(qcov,scov,pide|MAXHITS)
                bestbits="0.0"
                bestdesc='No informative hits'
                # output only one row per block
                for row in block:
                        if row[self.status_col1]=="False": continue
                        if row[self.spid_col1]==qpid : continue
                        if float(row[self.ff_col1])<self.FFCUTOFF: continue
                        # first inforative hit
                        bestbits=row[self.bits_col1]
                        bestdesc=row[self.desc_col1]
                        break
                datarow=[qpid,bestbits,bestdesc]
                self.summary_data.append_row(datarow)
                # lift first GO annotation
                bestgo=""
                bestgobits="0.0"
                bestdesc=""
                for row in block:
                        if row[self.status_col1]=="False": continue
                        if row[self.spid_col1]==qpid : continue
                        if row[self.goclass_col]!="":
                                bestgo=row[self.goclass_col]
                                bestgobits=row[self.bits_col1]
                                break
                # one goid per row
                for goid in bestgo.split(' '):
                        if not goid in self.glob.godesc: continue
                        datarow=[qpid,bestgobits,self.glob.ontology[goid],goid,self.glob.godesc[goid]]
                        self.goclass_data.append_row(datarow)


import functools
from typing import Callable

def check_permission(user: str) -> Callable:
    def check(func) -> Callable:
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            try:
                if user in user_permissions:
                    result = func(*args, **kwargs)
                    return result
                else:
                    raise PermissionError
            except PermissionError:
                print('{}: У пользователя недостаточно прав, чтобы выполнить функцию {}'.format(
                    PermissionError.__name__,
                    func.__name__))

        return wrapper

    return check

user_permissions = ['admin']

@check_permission('admin')
def delete_site():
    print('Удаляем сайт')

@check_permission('user_1')
def add_comment():
    print('Добавляем комментарий')

delete_site()
add_comment()


import pytest
import numpy.testing as nptest
import pandas.util.testing as pdtest
from wqio.tests import helpers

import numpy
import pandas
from matplotlib import pyplot

from wqio import hydro

BASELINE_IMAGES = '_baseline_images/hydro_tests'

class fakeStormSublcass(hydro.Storm):
    is_subclassed = True

def setup_storms(filename, baseflow=None):
    storm_file = helpers.test_data_path(filename)
    orig_record = pandas.read_csv(
        storm_file, index_col='date', parse_dates=True
    ).resample('5T').asfreq().fillna(0)

    baseflowcol = None
    if baseflow is not None:
        orig_record = orig_record.assign(baseflow=orig_record['influent'] < baseflow)
        baseflowcol = 'baseflow'

    parsed = hydro.parse_storm_events(orig_record, 3, 5,
                                      precipcol='rain',
                                      inflowcol='influent',
                                      baseflowcol=baseflowcol,
                                      debug=True)
    return parsed

@pytest.fixture
def storms_simple():
    return setup_storms('teststorm_simple.csv')

@pytest.fixture
def storms_firstobs():
    return setup_storms('teststorm_firstobs.csv')

@pytest.fixture
def storms_singular():
    return setup_storms('teststorm_singular.csv')

@pytest.fixture
def storms_with_baseflowcol():
    return setup_storms('teststorm_simple.csv', baseflow=2)

@pytest.mark.parametrize(('wetcol', 'expected'), [('A', 1), ('B', 0)])
def test_wet_first_row(wetcol, expected):
    df = pandas.DataFrame({
        'A': [True, True, False],
        'B': [False, False, True],
        'Z': [0, 0, 0],
    }).pipe(hydro._wet_first_row, wetcol, 'Z')

    assert df.iloc[0].loc['Z'] == expected

def test_wet_window_diff():
    is_wet = pandas.Series([0, 1, 1, 1, 0, 0, 1, 0, 0, 0])
    result = hydro._wet_window_diff(is_wet, 3)
    expected = pandas.Series([numpy.nan, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0])
    pdtest.assert_series_equal(result, expected)

@pytest.mark.parametrize('parsed', [
    storms_simple(),
    storms_firstobs(),
    storms_singular(),
    storms_with_baseflowcol()
])
def test_number_of_storms(parsed):
    last_storm = parsed['storm'].max()
    assert last_storm == 5

@pytest.mark.parametrize(('parsed', 'sn', 'idx', 'known'), [
    (storms_simple(), 1, 0, '2013-05-18 13:40:00'),
    (storms_simple(), 1, -1, '2013-05-19 01:45:00'),
    (storms_simple(), 2, 0, '2013-05-19 06:10'),
    (storms_simple(), 2, -1, '2013-05-19 11:35'),
    (storms_with_baseflowcol(), 1, 0, '2013-05-18 14:05:00'),
    (storms_with_baseflowcol(), 1, -1, '2013-05-19 01:40:00'),
    (storms_with_baseflowcol(), 2, 0, '2013-05-19 06:35'),
    (storms_with_baseflowcol(), 2, -1, '2013-05-19 11:30'),
    (storms_singular(), 1, 0, '2013-05-18 13:40:00'),
    (storms_singular(), 1, -1, '2013-05-19 01:45:00'),
    (storms_singular(), 2, 0, '2013-05-19 07:00'),
    (storms_singular(), 2, -1, '2013-05-19 07:05'),
    (storms_firstobs(), 1, 0, '2013-05-18 12:05'),
    (storms_firstobs(), 1, -1, '2013-05-19 01:45:00'),
    (storms_firstobs(), 2, 0, '2013-05-19 06:10'),
    (storms_firstobs(), 2, -1, '2013-05-19 11:35'),
])
def test_storm_start_end(parsed, sn, idx, known):
    storm = parsed[parsed['storm'] == sn]
    assert storm.index[idx] == pandas.Timestamp(known)

@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_IMAGES, tolerance=15)
def test_HydroRecord_histogram():
    stormfile = helpers.test_data_path('teststorm_simple.csv')
    orig_record = pandas.read_csv(
        stormfile, index_col='date', parse_dates=True
    ).resample('5T').asfreq().fillna(0)
    hr = hydro.HydroRecord(
        orig_record, precipcol='rain', inflowcol='influent',
        outflowcol=None, outputfreqMinutes=5, minprecip=1.5,
        intereventHours=3
    )

    fig = hr.histogram('Total Precip Depth', [4, 6, 8, 10])
    return fig

class base_HydroRecordMixin(object):
    known_storm_stats_columns = [
        'Storm Number', 'Antecedent Days', 'Season', 'Start Date', 'End Date',
        'Duration Hours', 'Peak Precip Intensity', 'Total Precip Depth',
        'Total Inflow Volume', 'Peak Inflow', 'Total Outflow Volume',
        'Peak Outflow', 'Peak Lag Hours', 'Centroid Lag Hours'
    ]

    def teardown(self):
        pyplot.close('all')

    def test_attributes(self):
        assert hasattr(self.hr, '_raw_data')
        assert isinstance(self.hr._raw_data, pandas.DataFrame)

        assert hasattr(self.hr, 'data')
        assert isinstance(self.hr.data, pandas.DataFrame)
        assert isinstance(self.hr.data.index, pandas.DatetimeIndex)
        for col in self.known_std_columns:
            assert col in self.hr.data.columns.tolist()

        assert hasattr(self.hr, 'all_storms')
        assert isinstance(self.hr.all_storms, dict)

        assert hasattr(self.hr, 'storms')
        assert isinstance(self.hr.storms, dict)

        assert hasattr(self.hr, 'storm_stats')
        assert isinstance(self.hr.storm_stats, pandas.DataFrame)

    def test_check_type_and_columns(self):
        assert isinstance(self.hr.data, pandas.DataFrame)

    def test_check_nan_col(self):
        assert numpy.all(numpy.isnan(self.hr.data['outflow']))

    def test_number_storms(self):
        last_storm = self.hr.data['storm'].max()
        assert last_storm == self.known_number_of_storms

    def test_first_storm_start(self):
        storm1 = self.hr.data[self.hr.data['storm'] == 1]
        assert storm1.index[0] == self.known_storm1_start

    def test_first_storm_end(self):
        storm1 = self.hr.data[self.hr.data['storm'] == 1]
        assert storm1.index[-1] == self.known_storm1_end

    def test_second_storm_start(self):
        storm2 = self.hr.data[self.hr.data['storm'] == 2]
        assert storm2.index[0] == self.known_storm2_start

    def test_second_storm_end(self):
        storm2 = self.hr.data[self.hr.data['storm'] == 2]
        assert storm2.index[-1] == self.known_storm2_end

    def test_interevent_stuff(self):
        assert hasattr(self.hr, 'intereventPeriods')
        assert hasattr(self.hr, 'intereventHours')

        assert self.hr.intereventPeriods == self.known_ie_periods
        assert self.hr.intereventHours == self.known_ie_hours

    def test_storm_stat_columns(self):
        assert self.known_storm_stats_columns == self.hr.storm_stats.columns.tolist()

class Test_HydroRecord_Simple(base_HydroRecordMixin):
    def setup(self):
        self.known_ie_hours = 3
        self.known_ie_periods = 36
        self.known_number_of_storms = 5
        self.known_number_of_bigstorms = 2
        self.known_storm1_start = pandas.Timestamp('2013-05-18 13:40:00')
        self.known_storm1_end = pandas.Timestamp('2013-05-19 01:45:00')
        self.known_storm2_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_storm2_end = pandas.Timestamp('2013-05-19 11:35')
        self.storm_file = helpers.test_data_path('teststorm_simple.csv')

        self.known_std_columns = ['rain', 'influent', 'outflow', 'storm']
        self.known_stats_subset = pandas.DataFrame({
            'Storm Number': [1, 5],
            'Antecedent Days': [numpy.nan, 0.708333],
            'Peak Precip Intensity': [1.200, 1.200],
            'Total Precip Depth': [2.76, 4.14],
        })

        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(
            self.orig_record, precipcol='rain', inflowcol='influent',
            outflowcol=None, outputfreqMinutes=5, minprecip=1.5,
            intereventHours=self.known_ie_hours
        )

        self.storm_date = pandas.Timestamp('2013-05-19 11:11')
        self.gap_date = pandas.Timestamp('2013-05-19 14:42')
        self.known_storm = 2

    def test_storm_attr_lengths(self):
        assert len(self.hr.storms) == self.known_number_of_bigstorms
        assert len(self.hr.all_storms) == self.known_number_of_storms

    def test_storm_stats(self):

        cols = [
            'Storm Number',
            'Antecedent Days',
            'Peak Precip Intensity',
            'Total Precip Depth',
        ]

        pdtest.assert_frame_equal(self.hr.storm_stats[cols], self.known_stats_subset[cols])

    def test_getStormFromTimestamp_basic(self):
        sn, storm = self.hr.getStormFromTimestamp(self.storm_date)
        assert sn == self.known_storm
        assert storm is None

    def test_getStormFromTimestamp_string(self):
        datestring = '{}'.format(self.storm_date)
        sn, storm = self.hr.getStormFromTimestamp(datestring)
        assert sn == self.known_storm
        assert storm is None

    def test_getStormFromTimestamp_datetime(self):
        pydt = self.storm_date.to_pydatetime()
        sn, storm = self.hr.getStormFromTimestamp(pydt)
        assert sn == self.known_storm
        assert storm is None

    def test_getStormFromTimestamp_small(self):
        sn, storm = self.hr.getStormFromTimestamp(self.storm_date, smallstorms=True)
        assert sn == self.known_storm
        assert isinstance(storm, hydro.Storm)

    def test_getStormFromTimestame_big(self):
        sn, storm = self.hr.getStormFromTimestamp(self.known_storm1_start)
        assert sn == 1
        assert isinstance(storm, hydro.Storm)

    def test_getStormFromTimestamp_lookback_6(self):
        sn, storm = self.hr.getStormFromTimestamp(self.gap_date, lookback_hours=6)
        assert sn == self.known_storm

    def test_getStormFromTimestamp_lookback_2(self):
        sn, storm = self.hr.getStormFromTimestamp(self.gap_date, lookback_hours=2)
        assert sn is None

    def test_getStormFromTimestamp_bad_date(self):
        with helpers.raises(ValueError):
            self.hr.getStormFromTimestamp('junk')

    def test_getStormFromTimestamp_bad_lookback(self):
        with helpers.raises(ValueError):
            sn = self.hr.getStormFromTimestamp(self.gap_date, lookback_hours=-3)

class Test_HydroRecord_Singular(base_HydroRecordMixin):
    def setup(self):
        self.known_ie_hours = 3
        self.known_ie_periods = 36
        self.known_number_of_storms = 5
        self.known_storm1_start = pandas.Timestamp('2013-05-18 13:40:00')
        self.known_storm1_end = pandas.Timestamp('2013-05-19 01:45:00')
        self.known_storm2_start = pandas.Timestamp('2013-05-19 07:00')
        self.known_storm2_end = pandas.Timestamp('2013-05-19 07:05')
        self.storm_file = helpers.test_data_path('teststorm_singular.csv')
        self.known_std_columns = ['rain', 'influent', 'outflow', 'storm']
        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(
            self.orig_record, precipcol='rain', inflowcol='influent',
            outflowcol=None, outputfreqMinutes=5,
            intereventHours=self.known_ie_hours
        )

class Test_HydroRecord_FirstObservation(base_HydroRecordMixin):
    def setup(self):
        self.known_ie_hours = 3
        self.known_ie_periods = 36
        self.known_number_of_storms = 5
        self.known_storm1_start = pandas.Timestamp('2013-05-18 12:05')
        self.known_storm1_end = pandas.Timestamp('2013-05-19 01:45:00')
        self.known_storm2_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_storm2_end = pandas.Timestamp('2013-05-19 11:35')
        self.storm_file = helpers.test_data_path('teststorm_firstobs.csv')
        self.known_std_columns = ['rain', 'influent', 'outflow', 'storm']
        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(
            self.orig_record, precipcol='rain', inflowcol='influent',
            outflowcol=None, outputfreqMinutes=5,
            intereventHours=self.known_ie_hours
        )

class TestHydroRecord_diffStormClass(base_HydroRecordMixin):
    def setup(self):
        self.known_ie_hours = 3
        self.known_ie_periods = 36
        self.known_number_of_storms = 5
        self.known_storm1_start = pandas.Timestamp('2013-05-18 13:40:00')
        self.known_storm1_end = pandas.Timestamp('2013-05-19 01:45:00')
        self.known_storm2_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_storm2_end = pandas.Timestamp('2013-05-19 11:35')
        self.storm_file = helpers.test_data_path('teststorm_simple.csv')

        self.known_std_columns = ['rain', 'influent', 'outflow', 'storm']
        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(
            self.orig_record, precipcol='rain', inflowcol='influent',
            outflowcol=None, outputfreqMinutes=5, minprecip=1.5,
            stormclass=fakeStormSublcass,
            intereventHours=self.known_ie_hours
        )

        self.storm_date = pandas.Timestamp('2013-05-19 11:11')
        self.gap_date = pandas.Timestamp('2013-05-19 14:42')
        self.known_storm = 2
        self.known_stats_subset = pandas.DataFrame({
            'Storm Number': [1, 5],
            'Antecedent Days': [-2.409722, 0.708333],
            'Peak Precip Intensity': [1.200, 1.200],
            'Total Precip Depth': [2.76, 4.14],
        })

    def test_subclassed_storms(self):
        for sn in self.hr.all_storms:
            assert self.hr.all_storms[sn].is_subclassed

class Test_Storm(object):
    def setup(self):
        # path stuff
        self.storm_file = helpers.test_data_path('teststorm_simple.csv')
        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(self.orig_record,
                                    precipcol='rain',
                                    inflowcol='influent',
                                    outflowcol='effluent',
                                    outputfreqMinutes=5,
                                    intereventHours=2)
        self.storm = hydro.Storm(self.hr.data, 2,
                                 precipcol=self.hr.precipcol,
                                 inflowcol=self.hr.inflowcol,
                                 outflowcol=self.hr.outflowcol,
                                 freqMinutes=self.hr.outputfreq.n)

        self.known_columns = ['rain', 'influent', 'effluent', 'baseflow', 'storm']
        self.known_index_type = pandas.DatetimeIndex
        self.known_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_end = pandas.Timestamp('2013-05-19 11:55')
        self.known_season = 'spring'
        self.known_precip_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_precip_end = pandas.Timestamp('2013-05-19 09:10')
        self.known_inflow_start = pandas.Timestamp('2013-05-19 06:25')
        self.known_inflow_end = pandas.Timestamp('2013-05-19 11:30')
        self.known_outflow_start = pandas.Timestamp('2013-05-19 06:50')
        self.known_outflow_end = pandas.Timestamp('2013-05-19 11:50')
        self.known_duration_hours = 5.75
        self.known_antecedent_period_days = 0.17708333
        self.known_peak_precip_intensity = 1.32
        self.known_peak_inflow = 123.0
        self.known_peak_outflow = 41.0
        self.known_peak_precip_intensity_time = pandas.Timestamp('2013-05-19 08:00')
        self.known_peak_inflow_time = pandas.Timestamp('2013-05-19 08:35')
        self.known_peak_outflow_time = pandas.Timestamp('2013-05-19 08:45')
        self.known_centroid_precip = pandas.Timestamp('2013-05-19 07:41:58.705023')
        self.known_centroid_inflow = pandas.Timestamp('2013-05-19 08:44:31.252553')
        self.known_centroid_outflow = pandas.Timestamp('2013-05-19 08:59:51.675231')
        self.known_total_precip_depth = 1.39
        self.known_total_inflow_volume = 876600
        self.known_total_outflow_volume = 291900
        self.known_peak_lag_time = 0.1666666667
        self.known_centroid_lag_time = 0.255672966

    def teardown(self):
        pyplot.close('all')
        pass

    def test_columns(self):
        assert sorted(self.known_columns) == sorted(self.storm.data.columns.tolist())

    def test_index_type(self):
        assert isinstance(self.storm.data.index, pandas.DatetimeIndex)

    def test_start(self):
        assert hasattr(self.storm, 'start')
        assert self.storm.start == self.known_start

    def test_end(self):
        assert hasattr(self.storm, 'end')
        assert self.storm.end == self.known_end

    def test_season(self):
        assert hasattr(self.storm, 'season')
        assert self.storm.season == self.known_season

    def test_season_setter(self):
        self.storm.season = 'spring'
        assert self.storm.season == 'spring'

    def test_precip_start(self):
        assert hasattr(self.storm, 'precip_start')
        assert self.storm.precip_start == self.known_precip_start

    def test_precip_end(self):
        assert hasattr(self.storm, 'precip_end')
        assert self.storm.precip_end == self.known_precip_end

    def test_inflow_start(self):
        assert hasattr(self.storm, 'inflow_start')
        assert self.storm.inflow_start == self.known_inflow_start

    def test_inflow_end(self):
        assert hasattr(self.storm, 'inflow_end')
        assert self.storm.inflow_end == self.known_inflow_end

    def test_outflow_start(self):
        assert hasattr(self.storm, 'outflow_start')
        assert self.storm.outflow_start == self.known_outflow_start

    def test_outflow_end(self):
        assert hasattr(self.storm, 'outflow_end')
        assert self.storm.outflow_end == self.known_outflow_end

    def test_duration_hours(self):
        assert hasattr(self.storm, 'duration_hours')
        assert abs(self.storm.duration_hours - self.known_duration_hours) < 0.0001

    def test_antecedent_period_days(self):
        assert hasattr(self.storm, 'antecedent_period_days')
        assert abs(self.storm.antecedent_period_days - self.known_antecedent_period_days) < 0.0001

    def test_peak_precip_intensity(self):
        assert hasattr(self.storm, 'peak_precip_intensity')
        assert abs(self.storm.peak_precip_intensity - self.known_peak_precip_intensity) < 0.0001

    def test_peak_inflow(self):
        assert hasattr(self.storm, 'peak_inflow')
        assert abs(self.storm.peak_inflow - self.known_peak_inflow) < 0.0001

    def test_peak_outflow(self):
        assert hasattr(self.storm, 'peak_outflow')
        assert abs(self.storm.peak_outflow - self.known_peak_outflow) < 0.0001

    def test_peak_precip_intensity_time(self):
        assert hasattr(self.storm, 'peak_precip_intensity_time')
        assert self.storm.peak_precip_intensity_time == self.known_peak_precip_intensity_time

    def test_peak_inflow_time(self):
        assert hasattr(self.storm, 'peak_inflow_time')
        assert self.storm.peak_inflow_time == self.known_peak_inflow_time

    def test_peak_outflow_time(self):
        assert hasattr(self.storm, 'peak_outflow_time')
        assert self.storm.peak_outflow_time == self.known_peak_outflow_time

    def test_centroid_precip(self):
        assert hasattr(self.storm, 'centroid_precip_time')
        assert self.storm.centroid_precip_time.strftime('%x %X') == self.known_centroid_precip.strftime('%x %X')

    def test_centroid_inflow(self):
        assert hasattr(self.storm, 'centroid_inflow_time')
        assert self.storm.centroid_inflow_time.strftime('%x %X') == self.known_centroid_inflow.strftime('%x %X')

    def test_centroid_outflow(self):
        assert hasattr(self.storm, 'centroid_outflow_time')
        assert self.storm.centroid_outflow_time.strftime('%x %X') == self.known_centroid_outflow.strftime('%x %X')

    def test_total_precip_depth(self):
        assert hasattr(self.storm, 'total_precip_depth')
        assert abs(self.storm.total_precip_depth - self.known_total_precip_depth) < 0.0001

    def test_total_inflow_volume(self):
        assert hasattr(self.storm, 'total_inflow_volume')
        assert abs(self.storm.total_inflow_volume - self.known_total_inflow_volume) < 0.0001

    def test_total_outflow_volume(self):
        assert hasattr(self.storm, 'total_outflow_volume')
        assert abs(self.storm.total_outflow_volume - self.known_total_outflow_volume) < 0.0001

    def test_peak_lag_hours(self):
        assert hasattr(self.storm, 'peak_lag_hours')
        assert abs(self.storm.peak_lag_hours - self.known_peak_lag_time) < 0.0001

    def test_centroid_lag_hours(self):
        assert hasattr(self.storm, 'centroid_lag_hours')
        assert abs(self.storm.centroid_lag_hours - self.known_centroid_lag_time) < 0.0001

    def test_summary_dict(self):
        assert hasattr(self.storm, 'summary_dict')
        assert isinstance(self.storm.summary_dict, dict)
        known_keys = [
            'Storm Number',
            'Antecedent Days',
            'Start Date',
            'End Date',
            'Duration Hours',
            'Peak Precip Intensity',
            'Total Precip Depth',
            'Total Inflow Volume',
            'Peak Inflow',
            'Total Outflow Volume',
            'Peak Outflow',
            'Peak Lag Hours',
            'Centroid Lag Hours',
            'Season'
        ]
        keys = list(self.storm.summary_dict.keys())
        assert sorted(keys) == sorted(known_keys)

    def test_is_small(self):
        assert self.storm.is_small(minprecip=5.0)
        assert not self.storm.is_small(minprecip=1.0)

def setup_storm():
    pyplot.rcdefaults()
    storm_file = helpers.test_data_path('teststorm_simple.csv')
    orig_record = (
        pandas.read_csv(storm_file, index_col='date', parse_dates=True)
              .resample('5T')
              .asfreq()
              .fillna(0)
    )

    hr = hydro.HydroRecord(
        orig_record, precipcol='rain', inflowcol='influent',
        outflowcol=None, outputfreqMinutes=5, minprecip=0,
        intereventHours=3
    )

    return hr.storms[2]

@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_IMAGES, tolerance=15)
def test_plot_storm_summary():
    storm = setup_storm()

    def doplot(storm):
        labels = {
            storm.inflowcol: 'Effluent (l/s)',
            storm.precipcol: 'Precip Depth (mm)'

        }
        fig, artists, labels = storm.summaryPlot(outflow=False, serieslabels=labels)
        return fig

    fig = doplot(storm)
    return fig

@pytest.fixture
def drainage_area():
    return hydro.DrainageArea(100, 75, 10)

def test_da_total_area(drainage_area):
    assert drainage_area.total_area == 100

def test_da_imp_area(drainage_area):
    assert drainage_area.imp_area == 75

def test_da_bmp_area(drainage_area):
    assert drainage_area.bmp_area == 10

@pytest.mark.parametrize(('conversion', 'factor', 'expected'), [
    (1.0, 1.0, 82.50),
    (1.0, 0.9, 75.25),
    (2.0, 1.0, 165.00),
    (2.0, 0.9, 150.50)
])
def test_simple_method(drainage_area, conversion, factor, expected):
    depth = 1
    result = drainage_area.simple_method(depth, annual_factor=factor,
                                         volume_conversion=conversion)
    area = (drainage_area.total_area + drainage_area.bmp_area)
    storm_volume = depth * conversion * factor * area
    assert abs(expected - result) < 0.001
    assert storm_volume > result


from hearts.envs import TestCase
from agent import RandomAgent

if __name__ == '__main__':
    # create a test case
    test_case = TestCase(test_case_name='random_agent_performance')

    # set test case attributes
    test_case.set_match_num(100)  # 1 match = 4 games = 16 deals = 208 rounds
    test_case.set_random_seed(1234)

    # add 4 random agents
    agent_list = [RandomAgent(player_name=f'random_agent_{i}') for i in range(4)]

    test_case.set_agent(agent_list)

    test_case.evaluate()


# Keep this file separate

# https://apps.twitter.com/
# Create new App and get the four strings

def oauth():
    consumer_key_i = input("\nType or paste your consumer key: ")
    consumer_secret_i = input("Type or paste your consumer secret key: ")
    token_key_i = input("Type or paste your token key: ")
    token_secret_i = input("Type or paste your token secret key: ")
    return {"consumer_key": consumer_key_i,
                "consumer_secret": consumer_secret_i,
                "token_key": token_key_i,
                "token_secret": token_secret_i}


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import flags
import absl.logging as _logging  # pylint: disable=unused-import

import tensorflow as tf
try:
  import google3.experimental.users.zihangd.lm.data_utils as data_utils
  import google3.experimental.users.zihangd.lm.input_function as input_function
  import google3.experimental.users.zihangd.lm.tokenization as tokenization
except ImportError as e:
  import data_utils
  import input_function
  import tokenization

flags.DEFINE_string("record_info_dir", default="./proc_data/example",
                    help="Path to local directory containing *.tfrecord.")
flags.DEFINE_string("tokenizer_type", "sent_piece",
                    help="Type of the tokenizer.")
flags.DEFINE_string("tokenizer_paths", "",
                    help="Comma separated string.")
flags.DEFINE_integer("num_core_per_host", default=16,
                     help="number of cores per host.")
flags.DEFINE_integer("bsz_per_host", default=32, help="batch size per host.")
flags.DEFINE_integer("seq_len", default=512,
                     help="tgt len; 0 for not using it")
flags.DEFINE_bool("lower_case", default=False,
                  help="use lower_case inputs or not.")
flags.DEFINE_string("split", default="train",
                    help="Data split.")
flags.DEFINE_integer("num_passes", default=None,
                     help="Num of passes to use.")

FLAGS = flags.FLAGS

def main(unused_argv):
  del unused_argv  # Unused

  tokenizer = tokenization.get_tokenizer(
      tokenizer_type=FLAGS.tokenizer_type,
      paths=FLAGS.tokenizer_paths.split(","),
      do_lower_case=FLAGS.lower_case)
  data_utils.setup_special_ids(tokenizer)

  input_fn, record_info_dict = input_function.get_input_fn(
      tfrecord_dir=FLAGS.record_info_dir,
      split=FLAGS.split,
      bsz_per_host=FLAGS.bsz_per_host,
      seq_len=FLAGS.seq_len,
      num_hosts=1,
      num_core_per_host=FLAGS.num_core_per_host,
      lower_case=FLAGS.lower_case,
      num_passes=FLAGS.num_passes,
  )
  tf.logging.info(record_info_dict)

  bsz_per_core = FLAGS.bsz_per_host // FLAGS.num_core_per_host
  params = {
      "batch_size": bsz_per_core
  }

  dataset = input_fn(params)
  example = dataset.make_one_shot_iterator().get_next()

  with tf.Session() as sess:
    for i in range(10):
      example_np = sess.run(example)
      if i == 0:
        for key in example_np.keys():
          print(key, example_np[key].shape, len(example_np[key].shape))
          print(example_np[key][0].tolist())
          print(tokenizer.convert_ids_to_text(example_np[key][0].tolist()))
        print("=" * 80)

if __name__ == "__main__":
  tf.app.run()


#!/usr/bin/env python3

"""
Module for matrix_shape.
"""

def matrix_shape(matrix):
    """Finds the shape of matrix."""
    dims = [len(matrix)]
    while type(matrix[0]) != int:
        dims.append(len(matrix[0]))
        matrix = matrix[0]
    return dims


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /users/payno/.local/share/virtualenvs/est_venv/lib/python3.7/site-packages/orangecontrib/est/widgets/pymca/ft.py
# Compiled at: 2020-03-05 02:52:24
# Size of source mod 2**32: 9429 bytes
__authors__ = [
 'H. Payno']
__license__ = 'MIT'
__date__ = '06/07/2019'
import functools, logging
from Orange.widgets import gui
from Orange.widgets.settings import Setting
from Orange.widgets.widget import OWWidget
from Orange.widgets.widget import Input, Output
import Orange.data
import PyMca5.PyMcaGui.physics.xas.XASFourierTransformParameters as XASFourierTransformParameters
from silx.gui import qt
from silx.gui.plot import LegendSelector
import est.core.process.pymca.ft
from orangecontrib.est.process import _ProcessForOrangeMixIn, ProcessRunnable
from est.core.types import XASObject
from est.gui.XasObjectViewer import XasObjectViewer, ViewType
from est.gui.XasObjectViewer import _normalized_exafs, _ft_window_plot
from est.gui.XasObjectViewer import _ft_intensity_plot, _ft_imaginary_plot
from orangecontrib.est.progress import QProgress
from orangecontrib.est.utils import Converter
from ..container import _ParameterWindowContainer
_logger = logging.getLogger(__file__)

class FTWindow(qt.QMainWindow):

    def __init__(self, parent=None):
        qt.QMainWindow.__init__(self, parent)
        self.xasObjViewer = XasObjectViewer(mapKeys=['Mu'], spectrumPlots=('FTWindow',
                                                                           'FTIntensity'))
        self.xasObjViewer._spectrumViews[0]._plot.getXAxis().setLabel('K')
        self.xasObjViewer._spectrumViews[1]._plot.getXAxis().setLabel('R (Angstrom)')
        self.xasObjViewer._spectrumViews[1]._plot.getYAxis().setLabel('Arbitrary Units')
        self.setCentralWidget(self.xasObjViewer)
        self._pymcaWindow = _ParameterWindowContainer(parent=self, parametersWindow=XASFourierTransformParameters)
        dockWidget = qt.QDockWidget(parent=self)
        dockWidget.setWidget(self._pymcaWindow)
        self.addDockWidget(qt.Qt.RightDockWidgetArea, dockWidget)
        dockWidget.setAllowedAreas(qt.Qt.RightDockWidgetArea | qt.Qt.LeftDockWidgetArea)
        dockWidget.setFeatures(qt.QDockWidget.NoDockWidgetFeatures)
        self.legendDockWidget1 = LegendSelector.LegendsDockWidget(parent=self, plot=(self.xasObjViewer._spectrumViews[0]._plot))
        self.legendDockWidget1.setAllowedAreas(qt.Qt.RightDockWidgetArea | qt.Qt.LeftDockWidgetArea)
        self.legendDockWidget1.setFeatures(qt.QDockWidget.NoDockWidgetFeatures)
        self.addDockWidget(qt.Qt.RightDockWidgetArea, self.legendDockWidget1)
        self.legendDockWidget2 = LegendSelector.LegendsDockWidget(parent=self, plot=(self.xasObjViewer._spectrumViews[1]._plot))
        self.legendDockWidget2.setAllowedAreas(qt.Qt.RightDockWidgetArea | qt.Qt.LeftDockWidgetArea)
        self.legendDockWidget2.setFeatures(qt.QDockWidget.NoDockWidgetFeatures)
        self.addDockWidget(qt.Qt.RightDockWidgetArea, self.legendDockWidget2)
        self.addDockWidget(qt.Qt.RightDockWidgetArea, self.xasObjViewer._mapView.keySelectionDocker)
        for ope in (_normalized_exafs, _ft_window_plot):
            self.xasObjViewer._spectrumViews[0].addCurveOperation(ope)

        for ope in (_ft_intensity_plot, _ft_imaginary_plot):
            self.xasObjViewer._spectrumViews[1].addCurveOperation(ope)

        self.setWindowFlags(qt.Qt.Widget)
        self.xasObjViewer.viewTypeChanged.connect(self._updateLegendView)
        self._updateLegendView()

    def getNCurves(self):
        return len(self.plot.getAllCurves())

    def _updateLegendView(self):
        index, viewType = self.xasObjViewer.getViewType()
        self.legendDockWidget1.setVisible(viewType is ViewType.spectrum and index == 0)
        self.legendDockWidget2.setVisible(viewType is ViewType.spectrum and index == 1)
        self.xasObjViewer._mapView.keySelectionDocker.setVisible(viewType is ViewType.map)

class FTOW(_ProcessForOrangeMixIn, OWWidget):
    __doc__ = '\n    Widget used for signal extraction\n    '
    name = 'fourier transform'
    id = 'orange.widgets.pymca.xas.ft'
    description = 'Progress fourier transform'
    icon = 'icons/ft.png'
    priority = 4
    category = 'esrfWidgets'
    keywords = ['spectroscopy', 'signal', 'fourier', 'transform', 'fourier transform']
    want_main_area = True
    resizing_enabled = True
    process_function = est.core.process.pymca.ft.PyMca_ft
    _pymcaSettings = Setting(dict())

    class Inputs:
        xas_obj = Input('xas_obj', XASObject, default=True)
        data_table = Input('Data', Orange.data.Table)

    class Outputs:
        res_xas_obj = Output('xas_obj', XASObject)

    def __init__(self):
        super().__init__()
        self._latest_xas_obj = None
        self._window = FTWindow(parent=self)
        layout = gui.vBox(self.mainArea, 'fourier transform').layout()
        layout.addWidget(self._window)
        self._progress = gui.ProgressBar(self, 100)
        if self._pymcaSettings != dict():
            self._window._pymcaWindow.setParameters(self._pymcaSettings)
        self._window._pymcaWindow.sigChanged.connect(self._updateProcess)

    def _updateProcess(self, *arv, **kwargs):
        self._update_settings()
        if self._latest_xas_obj:
            self.process(xas_obj=(self._latest_xas_obj))

    @Inputs.data_table
    def processFrmDataTable(self, data_table):
        if data_table is None:
            return
        self.process(Converter.toXASObject(data_table=data_table))

    @Inputs.xas_obj
    def process(self, xas_obj):
        if xas_obj is None:
            return
        if not self._canProcess():
            _logger.warning('There is some processing on going already, willnot process the new dataset')
        self._latest_xas_obj = xas_obj.copy(create_h5_file=True)
        self._startProcess()
        process_obj = QPyMca_ft()
        process_obj._advancement.sigProgress.connect(self._setProgressValue)
        process_obj.setProperties({'_pymcaSettings': self._window._pymcaWindow.getParameters()})
        thread = self.getProcessingThread()
        thread.init(process_obj=process_obj, xas_obj=(self._latest_xas_obj))
        self._callback_finish = functools.partial(self._endProcess, self._latest_xas_obj)
        thread.finished.connect(self._callback_finish)
        thread.start(priority=(qt.QThread.LowPriority))

    def _update_settings(self):
        self._pymcaSettings = self._window._pymcaWindow.getParameters()

    def _setProgressValue(self, value):
        self._progress.widget.progressBarSet(value)

class QPyMca_ft(est.core.process.pymca.ft.PyMca_ft):
    __doc__ = '\n    Normalization able to give advancement using qt.Signal and QThreadPool\n    '

    def __init__(self):
        est.core.process.pymca.ft.PyMca_ft.__init__(self)
        self._advancement = QProgress('normalization')

    def _pool_process(self, xas_obj):
        self.pool = qt.QThreadPool()
        self.pool.setMaxThreadCount(5)
        for spectrum in xas_obj.spectra:
            runnable = ProcessRunnable(fct=(est.core.process.pymca.ft.process_spectr_ft), spectrum=spectrum,
              configuration=(xas_obj.configuration),
              callback=(self._advancement.increaseAdvancement))
            self.pool.start(runnable)

        self.pool.waitForDone()


from django.db import models
from legend.models import Parameter, Attribute
import string
# Create your models here.
class OptionForAPI(models.Model):

    parameter = models.ForeignKey(
        Parameter,
        on_delete=models.DO_NOTHING,
        verbose_name="Параметр запроса",
        )

    value = models.CharField(
        verbose_name="Значение параметра",
        help_text="Цифры,знаки",
        max_length=20,
        blank=False,
    )

    def generate_string(self):
        return f'{self.parameter.string}={self.value}'

    class Meta:
        verbose_name = "Пара параметр-значение"
        verbose_name_plural = "Пары параметр-значение"

    def __str__(self):
        return f'{self.parameter.group}| {self.parameter} = {self.value}'

class RequestForApi(models.Model):

    def __init__(self,*args,**kwargs):
        super().__init__(*args,**kwargs)
        #подчистка строки адреса от пробелов при инициализации
        self.addres=''.join(filter(lambda x: x in string.printable and x!=' ', self.addres))

    addres = models.CharField(
        verbose_name="Адрес для API запросов",
        blank=False,
        max_length=150,
    )

    parameters = models.ManyToManyField(
        to = OptionForAPI,
        verbose_name="Параметры",
        blank=True,
    )

    description = models.TextField(
        verbose_name="Пояснение",
        blank=True,
    )

    """def save(self,*args,**kwargs):
        addres = ''.join(filter(lambda x: x in string.printable and x!=' ',self.addres))
        super().save(*args,**kwargs)"""

    def generate_string(self):
        par_list = list(i.generate_string() for i in tuple(self.parameters.all()))
        par_string = '&'.join(par_list)
        return ''.join(filter(lambda x: x in string.printable and x!=' ', f'{self.addres}?{par_string}'))

    def __str__(self):
        return f'id {self.id}| {self.description[:60]}'

    class Meta:
        verbose_name = "Запрос для внешней БД"
        verbose_name_plural = "Запросы для внешней БД"

class CompilationR(models.Model):

    initial_request = models.ForeignKey(
        'RequestForApi',
        verbose_name="Главный запрос",
        on_delete=models.DO_NOTHING
        )

    get_page = models.ForeignKey(
        Parameter,
        verbose_name="Запрос страницы",
        on_delete=models.DO_NOTHING,
        related_name='get_page',
        blank=True,
        default='',
        null=True,
        )

    get_code = models.ForeignKey(
        Parameter,
        verbose_name="Запрос кода",
        on_delete=models.DO_NOTHING,
        related_name='get_code',
        blank=True,
        default='',
        null=True,
        )

    get_data = models.ForeignKey(
        Attribute,
        verbose_name="Данные со страницы",
        on_delete=models.DO_NOTHING,
        related_name='get_data',
        blank=True,
        default=None,
        null=True,
        )

    get_page_count = models.ForeignKey(
        Attribute,
        verbose_name="Общее количество страниц в теле ответа",
        on_delete=models.DO_NOTHING,
        related_name='get_page_count',
        blank=True,
        default='',
        null=True,
        )

    get_record_count = models.ForeignKey(
        Attribute,
        verbose_name="Общее количество записей в теле ответа",
        on_delete=models.DO_NOTHING,
        related_name='get_record_count',
        blank=True,
        default='',
        null=True,
        )

    get_current_page_number = models.ForeignKey(
        Attribute,
        verbose_name="Текущий номер страницы в теле ответа",
        on_delete=models.DO_NOTHING,
        related_name='get_current_page_number',
        blank=True,
        null=True,
        )

    class Meta:
        verbose_name = "Сборник запросов для скачивания БД"
        verbose_name_plural = "Сборники запросов для скачивания БД"

    def __str__(self):
        return f'id {self.id}| {self.initial_request.description[:60]}'


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import time
import hashlib
import numpy as np
import glob
from hyper_params import HyperParamsBase, ParamGenerator

def is_cc(): # check server is compute canada
    hostname = os.uname()[1]
    is_cc = 'gra' in hostname or 'cedar' in hostname

    return is_cc

class HyperParams(HyperParamsBase):
    def __init__(self):
        super(HyperParams, self).__init__()
        self.script_name = 'imagenet_train.py'
        self.sub_dir = 'test' # results are dumped in res_root_dir/sub_dir/...
        if is_cc():
            res_dir = '/scratch/trulls/yuki.ono/results'
        else:
            res_dir = os.getenv("PROJ_RES_DIR")
            if res_dir is None or res_dir == "":
                res_dir = os.path.join(os.getcwd(), 'results')
        self.res_root_dir = os.path.join(res_dir, 'practice/CFNet')

        ######################
        # Parameter catalogs
        ######################

        # Train
        self.max_itr = 1000000
        self.batch_size = 32
        self.optim_method = 'adam'
        self.lr = '1e-3'
        self.lr_decay = False
        self.weight_decay = 0
        # Dataset
        if is_cc():
            dataset_dir = '/scratch/trulls/yuki.ono/datasets'
        else:
            dataset_dir = '/cvlabdata1/home/ono/datasets'
        self.imagenet_dir = os.path.join(dataset_dir, 'imagenet')

        # Network
        self.model = 'custom_vgg' # custom_vgg | custom_alexnet

ROOT_JOB = 'jobs'
TODO_DIR = '{}/todo'.format(ROOT_JOB)
QUEUE_DIR = '{}/queue'.format(ROOT_JOB)
DONE_DIR = '{}/done'.format(ROOT_JOB)
FAIL_DIR = '{}/fail'.format(ROOT_JOB)

def check_job_pool():
    if not os.path.exists(TODO_DIR):
        os.makedirs(TODO_DIR)
    if not os.path.exists(QUEUE_DIR):
        os.makedirs(QUEUE_DIR)
    if not os.path.exists(DONE_DIR):
        os.makedirs(DONE_DIR)
    if not os.path.exists(FAIL_DIR):
        os.makedirs(FAIL_DIR)

def get_command(params, log_dir):

    CMD = params.script_name
    if not is_cc():
        CMD += ' --clear_logs'
    # Train
    CMD += ' --log_dir={}'.format(log_dir)
    CMD += ' --max_itr={}'.format(params.max_itr)
    CMD += ' --batch_size={}'.format(params.batch_size)
    CMD += ' --optim_method={}'.format(params.optim_method)
    CMD += ' --lr={}'.format(params.lr)
    CMD += ' --lr_decay={}'.format(params.lr_decay)
    CMD += ' --weight_decay={}'.format(params.weight_decay)

    # Dataset
    CMD += ' --imagenet_dir={}'.format(params.imagenet_dir)

    # Network
    CMD += ' --model={}'.format(params.model)

    return CMD

def get_log_dir(params):

    lr_str = 'lr-dcy' if params.lr_decay else 'lr-{}'.format(params.lr)

    log_dir = os.path.join(params.res_root_dir,
                params.sub_dir,
                'bs-{}'.format(params.batch_size),
                '{}-{}'.format(params.optim_method, lr_str),
                'wd-{}'.format(params.weight_decay),
                '{}'.format(params.model),
        )

    return log_dir

def write_shell_script(command, memo=None, params=None, log_dir=None):
    hash_str = hashlib.sha256(command.encode('utf-8')).hexdigest()[:16]
    job_name = 'job-{}.sh'.format(hash_str)
    job_file = os.path.join(TODO_DIR, job_name)

    PRESET = 'OMP_NUM_THREAD=4 python'

    with open(job_file, 'w') as f:
        f.write("#!/bin/bash\n")
        f.write('# [TIME] {}\n'.format(time.asctime()))
        if memo is not None and len(memo) > 0:
            f.write('# [MEMO] {}\n'.format(memo))
        if params is not None and len(params) > 0:
            f.write('# [PARAM] {}\n'.format(params))
        if log_dir is not None and len(log_dir) > 0:
            f.write('# [LOGDIR] {}\n'.format(log_dir))

        if is_cc():
            f.write("#SBATCH --account=def-kyi\n")
            f.write("#SBATCH --output=/scratch/trulls/yuki.ono/outputs/%x-%j.out\n")
            f.write("source /scratch/trulls/venvs/tflift/bin/activate\n")
            f.write("cd {}\n".format(os.getcwd()))
        f.write("CMD='{} {}'\n".format(PRESET, command))
        f.write('echo $CMD\n'.format(PRESET, command))
        f.write('eval $CMD\n'.format(PRESET, command))
    print('Create {}'.format(job_file))

if __name__ == '__main__':

    from utils.argparse_utils import *
    parser = get_parser()
    parser.add_argument('--dry_run', action='store_const',
                        const=True, default=False,
                        help="Just print the result")
    config, unparsed = get_config(parser)
    if len(unparsed) > 0:
        raise ValueError('Warning: miss identify argument ?? unparsed={}\n'.format(unparsed))

    exist_strategy = '' # skip, remove, recreate, overwrite
    memo = None
    pg = ParamGenerator()

    # Basic test
    memo = 'Imagenet training'
    pg.add_params('sub_dir', '180715-imagenet')
    # pg.add_params('optim_method', ['adam'])
    # pg.add_params('lr', ['1e-3'])
    # # pg.add_params('weight_decay', ['0', '5e-4'])
    # pg.add_params('model', ['custom_vgg', 'custom_alexnet'])

    pg.add_params('optim_method', ['momentum'])
    pg.add_params('lr_decay', [True])
    pg.add_params('weight_decay', ['5e-4'])
    pg.add_params('model', ['custom_vgg', 'custom_alexnet'])

    all_params = pg.generate(base_params=HyperParams())

    if not config.dry_run:
        check_job_pool()

    num_new_jobs = 0

    log_dir_set = set()

    for idx, params in enumerate(all_params):

        log_dir = get_log_dir(params)
        log_dir_set.add(log_dir)
        # print(os.path.join(params.res_root_dir, log_dir))
        if os.path.exists(os.path.join(params.res_root_dir, log_dir)):
            print('logdir {} has already existed ! strategy={}'.format(log_dir, exist_strategy))
            if exist_strategy == 'skip':
                continue
        cmd = get_command(params, log_dir)
        print(idx, log_dir)
        if not config.dry_run:
            write_shell_script(cmd, memo=memo, params=params.param_str, log_dir=log_dir)
        num_new_jobs += 1

    print('Add {} {}/{}'.format(num_new_jobs, ROOT_JOB, len(all_params)))
    total_num_jobs = len(glob.glob(os.path.join(TODO_DIR, 'job*sh')))
    if config.dry_run:
        total_num_jobs += num_new_jobs
    print('Total #jobs todo = {}'.format(total_num_jobs))

    if len(log_dir_set) != len(all_params):
        print('')
        print('[WARNING] log_dir may be overlapped !! expected={} but actual={}'.format(len(all_params), len(log_dir_set)))


from pytube import YouTube

link = input("Paste a youtube video link here: ")

path = input(" Please insert a path to save your video: ")

yt = YouTube(link)

result = {

    "Title": yt.title,
    "Views": yt.views,
    "Length": yt.length,
    "Rating": yt.rating

}

print(result)

ys = yt.streams.get_highest_resolution()

print ("Downloading your file... Please wait!")

ys.download(path)

print("Your file is ready to use! Thanks for using the app! Follow me on my GitHub.")

# A basic YouTube Video Downloader with python!
# My GitHub: https://github.com/Madeline-Lea


import matplotlib.pyplot as plt
import numpy as np
import read_post
import operator

def runningMean(x, N):
    return np.convolve(x, np.ones((N,))/N)[(N-1):][:(1-N)]

def analysis(file_name):
    incr = 1/12  #every five minutes in hour
    midnight1 = 1424246400 #midnight Tuesday Night
    midnight2 = 1424419200 #midnight Thursday Night
    f = open(file_name)
    number_posts = 0
    everything = {}
    subreddit_num_posts = {}
    index = {}
    times = {}
    for i in f:
        post = read_post.read_post(i)
        bt = post['batch_time']
        sub_id = post['subreddit_id']
        sub_name = post['subreddit']
        if bt <= midnight1 or bt >= midnight2:
            continue
        number_posts += 1
        if number_posts % 50000 == 0:
            print(number_posts);
        if sub_id not in times.keys(): times[sub_id] = {}
        if bt not in times[sub_id].keys(): times[sub_id][bt] = 0
        times[sub_id][bt] += 1
        if bt not in everything.keys(): everything[bt] = 0
        everything[bt] += 1

        if sub_id not in index.keys(): index[sub_id] = sub_name
        elif index[sub_id] != sub_name: print("DUPLACTE")
        if sub_id not in subreddit_num_posts.keys(): subreddit_num_posts[sub_id] = 0
        subreddit_num_posts[sub_id] += 1

    #Create Plotable lists
    batchtime = [(i-midnight1)/3600 for i, j in sorted(everything.items())]
    allposts = {}
    for sub_id in times.keys():
        if sub_id not in allposts: allposts[sub_id] = []
        for bt in everything.keys():
            if bt not in times[sub_id].keys():
                allposts[sub_id].append(0)
            else:
                allposts[sub_id].append(times[sub_id][bt])

    #print
    traffic = 0
    for i,j in [(i,j) for i,j in sorted(subreddit_num_posts.items(),key=operator.itemgetter(1), reverse=True)][:100]:
        print(index[i]+' (' + str(j) + ')')
        traffic += j
    print(traffic)

    for i,j in [(i,j) for i,j in sorted(subreddit_num_posts.items(),key=operator.itemgetter(1), reverse=True)][:7]:
        sub_posts = allposts[i]
        wbatchtime = batchtime[1:]
        wposts = [((i-j)/incr)*k for i,j,k in zip(batchtime[1:],batchtime,sub_posts[1:])]

        plt.figure(1)
        plt.plot(wbatchtime[35:-25],runningMean(wposts[10:],51),label=(index[i]+' ('+str(j)+')'))
        #plt.plot(wbatchtime,wposts)
        plt.title("Subreddits")
        plt.xlabel('Hours (5 minute increment)')
        plt.ylabel('Num of Posts')
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.show()
    print(number_posts)
    print()

if __name__ == "__main__":
    analysis("Data/NewCorpus.txt")


'''
시간 제한 3초, 메모리 제한 8 MB
N개의 수가 주어졌을 때, 이를 오름차순으로 정렬하는 프로그램을 작성하시오.

첫째 줄에 수의 개수 N(1 ≤ N ≤ 10,000,000)이 주어진다. 둘째 줄부터 N개의 줄에는 숫자가 주어진다.
이 수는 10,000보다 작거나 같은 자연수이다.

첫째 줄부터 N개의 줄에 오름차순으로 정렬한 결과를 한 줄에 하나씩 출력한다.
'''
import sys
n = int(sys.stdin.readline())   # for문의 range에 sys문장을 그대로 넣으면 안됨.
series = [0] * 10001

for _ in range(n):
    series[int(sys.stdin.readline())] += 1  # input을 저장하지 않고 인덱스로.

for i in range(10001):
    if series[i] != 0:
        for _ in range(series[i]):
            print(i)

'''
sort로 해서 풀면 메모리 초과가 된다.
메모리가 너무 적어서, input을 처음에 다 받아 저장하면 무조건 메모리 초과가 뜬다.
input을 따로 저장하지 않고 해결해야한다고 함.
'''


from bs4 import BeautifulSoup

import requests

def get_proxies():
    proxies = []
    response = requests.get('https://free-proxy-list.net/')
    soup = BeautifulSoup(response.text, "html.parser")

    for tr in soup.find_all('tr'):
        td_list = tr.find_all('td')
        proxy_dict = {}
        if not td_list is None and len(td_list) == 8:
            for n, key in enumerate(['IP', 'Port', 'code', 'country',
                                     'anonymity', 'google', 'https', 'refresh']):
                text = td_list[n].get_text()
                if text == 'yes': text = True
                elif text == 'no': text = False
                proxy_dict[key] = text
            proxies.append(proxy_dict)
    return proxies


# -*- coding: utf-8 -*-

# first neural network with keras tutorial
from numpy import loadtxt
from keras.models import Sequential
from keras.layers import Dense
from keras.models import model_from_json

# load the dataset
dataset = loadtxt('pima-indians-diabetes.csv', delimiter=',')

# split into input (X) and output (y) variables
X = dataset[:,0:8]
y = dataset[:,8]

# define the keras model
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))

# compile the keras model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# fit the keras model on the dataset
model.fit(X, y, epochs=150, batch_size=10)

# evaluate the keras model
_, accuracy = model.evaluate(X, y)
print('Accuracy: %.2f' % (accuracy*100))

# make class predictions with the model
predictions = model.predict_classes(X)
# summarize the first 5 cases
for i in range(5):
	print('%s => %d (expected %d)' % (X[i].tolist(), predictions[i], y[i]))

# serialize model to JSON
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("model.h5")
print("Saved model to disk")

# load json and create model
with open('model.json', 'r') as json_file:
    loaded_model_json = json_file.read()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("model.h5")
print("Loaded model from disk")

# evaluate loaded model on test data
loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
score = loaded_model.evaluate(X, y, verbose=0)
print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1]*100))


from blinker import Namespace
from signals import executionSignals as execution

class Execution():
    def __init__(self):
        self.signals = Namespace()

        self.start = self.signals.signal("start")
        self.start.connect(execution.start)

        self.stop = self.signals.signal("stop")
        self.stop.connect(execution.stop)

        self.pause = self.signals.signal("pause")
        self.pause.connect(execution.pause)

        self.shutdown = self.signals.signal("shutdown")
        self.shutdown.connect(execution.shutdown)

    #Actions
    def post(self, action, **kwargs):
        if action == "start":
            self.start.send(self)
        elif action == "stop":
            self.stop.send(self)
        return {}


# -*- coding: utf-8 -*-
from __future__ import absolute_import

import pytest
from mock import Mock

from elasticapm.utils import compat, stacks
from elasticapm.utils.stacks import get_culprit, get_stack_info
from tests.utils.stacks import get_me_a_test_frame

class Context(object):
    def __init__(self, dict):
        self.dict = dict

    __getitem__ = lambda s, *a: s.dict.__getitem__(*a)
    __setitem__ = lambda s, *a: s.dict.__setitem__(*a)
    iterkeys = lambda s, *a: compat.iterkeys(s.dict, *a)

def test_get_culprit_bad_module():
    culprit = get_culprit([{
        'module': None,
        'function': 'foo',
    }])
    assert culprit == '<unknown>.foo'

    culprit = get_culprit([{
        'module': 'foo',
        'function': None,
    }])
    assert culprit == 'foo.<unknown>'

    culprit = get_culprit([{
    }])
    assert culprit == '<unknown>.<unknown>'

def test_bad_locals_in_frame():
    frame = Mock()
    frame.f_locals = Context({
        'foo': 'bar',
        'biz': 'baz',
    })
    frame.f_lineno = 1
    frame.f_globals = {}
    frame.f_code.co_filename = __file__.replace('.pyc', '.py')
    frame.f_code.co_name = __name__
    frames = [(frame, 1)]
    results = get_stack_info(frames)
    assert len(results) == 1
    result = results[0]
    assert 'vars' in result
    variables = {
        "foo": "bar",
        "biz": "baz",
    }
    assert result['vars'] == variables

def test_traceback_hide(elasticapm_client):
    def get_me_a_filtered_frame(hide=True):
        __traceback_hide__ = True
        if not hide:
            del __traceback_hide__

        return list(stacks.iter_stack_frames())

    # hide frame from `get_me_a_filtered_frame
    frames = list(stacks.get_stack_info(get_me_a_filtered_frame(hide=True)))
    assert frames[0]['function'] == 'test_traceback_hide'

    # don't hide it:
    frames = list(stacks.get_stack_info(get_me_a_filtered_frame(hide=False)))
    assert frames[0]['function'] == 'get_me_a_filtered_frame'

@pytest.mark.parametrize('elasticapm_client', [{
    'include_paths': ('a.b.c', 'c.d'),
    'exclude_paths': ('c',)
}], indirect=True)
def test_library_frames(elasticapm_client):
    include = elasticapm_client.include_paths_re
    exclude = elasticapm_client.exclude_paths_re
    frame1 = Mock(f_globals={'__name__': 'a.b.c'})
    frame2 = Mock(f_globals={'__name__': 'a.b.c.d'})
    frame3 = Mock(f_globals={'__name__': 'c.d'})

    info1 = stacks.get_frame_info(frame1, 1, False, False, include_paths_re=include, exclude_paths_re=exclude)
    info2 = stacks.get_frame_info(frame2, 1, False, False, include_paths_re=include, exclude_paths_re=exclude)
    info3 = stacks.get_frame_info(frame3, 1, False, False, include_paths_re=include, exclude_paths_re=exclude)
    assert not info1['library_frame']
    assert not info2['library_frame']
    assert info3['library_frame']

def test_get_frame_info():
    frame = get_me_a_test_frame()
    frame_info = stacks.get_frame_info(frame, frame.f_lineno,
                                       with_locals=True, with_source_context=True)
    assert frame_info['function'] == 'get_me_a_test_frame'
    assert frame_info['filename'] == 'tests/utils/stacks/__init__.py'
    assert frame_info['module'] == 'tests.utils.stacks'
    assert frame_info['lineno'] == 6
    assert frame_info['context_line'] == '    return inspect.currentframe()'
    assert frame_info['vars'] == {'a_local_var': 42}


import csv
import glob
import os
import subprocess

def check_zipfiles_expected_by_endpoints(path_to_endpoints, path_to_zipfiles):
    link_count = 0

    missing_zip_files = []

    for fn in glob.glob(os.path.join(path_to_endpoints, "*.html")):
        with open(fn, "r") as inpf:
            for l in inpf:
                d = l[0:l.find("\n") - 1]
                if "<a class" in d:
                    fn = (d[d.find("href") + 6:d.find(">") - 1]).replace(
                        "/ukborders/easy_download/prebuilt",
                        path_to_zipfiles
                    )
                    if not os.path.exists(fn):
                        missing_zip_files.append(
                            "{} Does not exist".format(fn)
                        )
                    link_count += 1

    zipfile_count = 0

    for root, folders, files in os.walk(path_to_zipfiles):
        for fn in files:
            if os.path.splitext(fn)[-1] == ".zip":
                zipfile_count += 1

    if zipfile_count == link_count:
        print("Number of zipfiles({}) equals number of links".format(
            str(zipfile_count)))
    else:
        print(
            "Number of zipfiles({}) DOES NOT equal number of links({})".format(
                str(zipfile_count), str(link_count)
            ))

    if len(missing_zip_files) > 0:
        print("The following zipfiles are missing:")
        for i in missing_zip_files:
            print(i)

def count_features(path_to_zipfiles):
    base_path = path_to_zipfiles

    with open("/home/james/Desktop/qa.csv", "w") as outpf:
        my_writer = csv.writer(outpf, delimiter=",", quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
        my_writer.writerow(["dataset", "feature_count"])
        for df in ("csv", "kml", "mapinfo", "shape"):

            if df == "mapinfo":
                ptn = "*.tab"
            elif df == "shape":
                ptn = "*.shp"
            else:
                ptn = ".".join(["*", df])

            p = os.path.join(base_path, df, ptn)

            for fn in glob.glob(p):
                cmd = "ogrinfo -so -al {} | grep ^'Feature Count'".format(fn)
                output = str(subprocess.check_output(cmd, shell=True))
                fc = int((((output.split(":")[-1]).replace("\\n", ""))[:-1]))
                my_writer.writerow([fn, fc])

def do_qa():
    count_features()

    feature_counts = {}

    c = 1

    with open("/home/james/Desktop/qa.csv", "r") as inpf:
        my_reader = csv.reader(inpf)

        for r in my_reader:
            if c > 1:
                fn = r[0]
                fc = r[1]
                format = os.path.splitext(os.path.split(fn)[-1])[1]
                ds = os.path.splitext(os.path.split(fn)[-1])[0]

                if ds not in feature_counts:
                    if not ds.endswith("_ing"):
                        formats = {".csv":None, ".kml":None, ".shp":None, ".tab":None}
                    else:
                        formats = {".shp": None,".tab": None}
                    formats[format] = int(fc)
                    feature_counts[ds] = formats
                else:
                    feature_counts[ds][format] = int(fc)
            c += 1

    for ds in feature_counts:
        s = set()
        feature_counts_for_ds = feature_counts[ds]
        for fmt in feature_counts_for_ds:
            s.add(feature_counts_for_ds[fmt])

        if len(s) != 1:
            print("Problem with {}".format(ds))
            print("\tFeature Count for .csv:{}".format(feature_counts[ds][".csv"]))
            print("\tFeature Count for .kml:{}".format(feature_counts[ds][".kml"]))
            print("\tFeature Count for .shp:{}".format(feature_counts[ds][".shp"]))
            print("\tFeature Count for .tab:{}".format(feature_counts[ds][".tab"]))

def main():
    #do_qa()
    path_to_endpoints = "/home/james/Desktop/endpoints"
    path_to_zipfiles = "/home/james/Desktop/new_ni_data"
    check_zipfiles_expected_by_endpoints(path_to_endpoints, path_to_zipfiles)

if __name__ == "__main__":
    main()


#!/usr/bin/env python

import os, os.path
import argparse

parser = argparse.ArgumentParser(description="Mr. Shrinker")

parser.add_argument("percent",
                    help="Please specify how bad you'd like to shrink your file")

parser.add_argument("old_name", help="Please specify the old address to your file")

parser.add_argument("new_name", nargs='?', help="Please specify the new address to your file")

args = parser.parse_args()

if not os.path.exists(args.old_name):
    print('There is no file to shrink')

if args.percent:
    import subprocess
    number = args.percent
    old = args.old_name
    new = args.new_name
    if os.path.isfile(old) == True:
        if new != None:
            params = ['convert', old, '-resize', args.percent, new]
            params = " ".join(params)
            subprocess.call(params, shell=True)
        else:
            params = ['convert', old, '-resize', args.percent, old]
            params = " ".join(params)
            subprocess.call(params, shell=True)
    if os.path.isdir(old) == True:
        new = None
        for (dirpath, dirnames, filenames) in os.walk(old):
            for filename in filenames:
                if filename.endswith('.jpg') or filename.endswith('.png'):
                    right_file = old + '/' + filename
                    params = ['convert', right_file, '-resize', args.percent, right_file]
                    params = " ".join(params)
                    subprocess.call(params, shell=True)


#%%
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import networkx as nx

#%%
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch_geometric
from torch_geometric.datasets import Planetoid

#%%
Cora = Planetoid(root = "/Users/mineself2016/study/codeSpace/VSCodeWorkSpace/PythonWorkSpace/PyTorch/Pygeometric/datasets", name = "Cora")

#%%
graph = Cora[0]

#%%
graph

#%%
graph.x.shape

#%%
graph

#%%
graph.edge_index

#%%
e = graph.edge_index.numpy().T.tolist()

#%%
class SAGELayer(nn.Module):
    def __init__(self, edge_index, depth, num_neighbors, in_features, out_features, aggr = "mean"):
        super(SAGELayer, self).__init__()

        self.G = nx.Graph()
        self.G.add_edges_from(edge_index)

        self.linear_1 = nn.Linear(2 * in_features, 20)
        self.linear_2 = nn.Linear(20 * 2, out_features)

        self.depth =  depth
        self.num_neighbors = num_neighbors
        self.aggr = aggr

    def forward(self, x):
        h_old = x
        h_new = None

        for d in range(self.depth):
            if d == 0:
                h_new = torch.zeros(2708, 20)
            elif d == 1:
                h_new = torch.zeros(2708, 10)
            else:
                raise Exception("depth out of range")

            for node_v in self.G.nodes:
                neighbors = self.sample(node_v)
                h_neighbor = self.aggregate(neighbors, h_old)

                h_v = torch.cat([h_old[node_v], h_neighbor], dim = 0)

                if d == 0:
                    h_v = self.linear_1(h_v)
                elif d == 1:
                    h_v = self.linear_2(h_v)
                else:
                    raise Exception("depth out of range")

                h_v = F.sigmoid(h_v)
                # print(h_neighbor[h_neighbor != 0])
                h_v = h_v / torch.norm(h_v, p = 2)

                h_new[node_v] = h_v

            h_old = h_new

        return h_new

    def sample(self, node_v):
        neighbors = list(self.G.adj[node_v])

        if len(neighbors) >= self.num_neighbors:
            neighbors = np.random.choice(neighbors, size = self.num_neighbors, replace = False)
        else:
            neighbors = np.random.choice(neighbors, size = self.num_neighbors, replace = True)

        return neighbors

    def aggregate(self, neighbors, x):
        if self.aggr == "mean":
            idx, value = x[neighbors].max(dim = 0)
            return value

        else:
            print("not mean aggregate")
            return None

#%%
class GraphSAGE(nn.Module):
    def __init__(self, edge_index, depth, num_neighbors, in_features, out_features, aggr = "mean"):
        super(GraphSAGE, self).__init__()
        self.Sage_1 = SAGELayer(edge_index, depth = 2, num_neighbors = 4, in_features = 1433, out_features = 10)
        self.linear = nn.Linear(10, 7)

    def forward(self, x):
        x = self.Sage_1(x)
        x = x.relu()
        x = F.dropout(x, p = 0.5, training = self.training)
        x = self.linear(x)

        return x

model = GraphSAGE(e, depth = 2, num_neighbors = 4, in_features = 1433, out_features = 10)
#%%
print(model)

#%%
optimizer = torch.optim.Adam(model.parameters(), lr = 0.2, weight_decay=5e-4)
criterion = torch.nn.CrossEntropyLoss()

#%%
def train():
      model.train()
      optimizer.zero_grad()  # Clear gradients.

      output = model(graph.x)  # Perform a single forward pass.
      loss = criterion(output[graph.train_mask], graph.y[graph.train_mask])  # Compute the loss solely based on the training nodes.
      loss.backward()  # Derive gradients.
      optimizer.step()  # Update parameters based on gradients.
      return loss

def test():
      model.eval()
      output = model(graph.x)
      pred = output.argmax(dim=1)  # Use the class with highest probability.
      test_correct = pred[graph.test_mask] == graph.y[graph.test_mask]  # Check against ground-truth labels.
      test_acc = int(test_correct.sum()) / int(graph.test_mask.sum())  # Derive ratio of correct predictions.
      return test_acc

#%%
for epoch in range(1, 201):
    loss = train()
    print(f'Epoch: {epoch:03d}, Loss: {loss:.4f}')

    if epoch % 20 == 0:
        test_acc = test()
        print(f'Test Accuracy: {test_acc:.4f}')

#%%

#%%
graph.x

#%%
for node in model.G.nodes:
    print(node)

# #%%
# class GCNModel(nn.Module):
#     def __init__(self, in_features, out_features):
#         super(GCNModel, self).__init__()
#         self.gcn_1 = nn.Linear(in_features, out_features)

#     def forward(self, graph, x):
#         x = self.gcn_1(x)

#         return x

# model = GCNModel(10, 2)
# #%%
# output = model(None, torch.ones(10))

# output
# #%%

# %%


from PyQt4.QtCore import *

import time

from _debug_ import *
from _items_ import LItem
from setup import *

if not vars().has_key('ITEMS'): ITEMS = 0

#----------------------------------------------------------------------
class LMyItem(LItem):
    '''
    >>> c = LMyItem()
    >>> c
    <LMyItem:  title='My title' xLabel='X' yLabel='Y'>
    '''

    def __init__(self, parent = None):
        super(LMyItem, self).__init__(parent)
        self.title = "My title"
        self.xLabel = 'X'
        self.yLabel = 'Y'
        self.subplotId = 224

        self._x = [0,1]
        self._y = [0,1]

        self._orderedAttributes = 'subplotId', 'title', 'xLabel', 'yLabel'

    @LInAndOut(DEBUG & ITEMS)
    def plot(self, figure):
        self.axis = figure.add_subplot(self.subplotId)
        #self.axis.xlabel(self.xLabel)
        #self.axis.ylabel(self.yLabel)
        self.axis.plot(self._x, self._y)

#----------------------------------------------------------------------
# MAIN
#----------------------------------------------------------------------
if __name__ == "__main__":
    import doctest
    doctest.testmod()


from core_data_modules.traced_data import TracedData

class CombineRawDatasets(object):
    @staticmethod
    def combine_raw_datasets(user, messages_datasets, surveys_datasets):
        data = []

        for messages_dataset in messages_datasets:
            data.extend(messages_dataset)

        for surveys_dataset in surveys_datasets:
            TracedData.update_iterable(user, "avf_phone_id", data, surveys_dataset, "survey_responses")

        return data