'''
Test Sciris file I/O functions.
'''

import os
import numpy as np
import pylab as pl
import openpyxl
import sciris as sc
import pytest
ut = sc.importbypath(sc.thispath() / 'sc_test_utils.py')

# Define filenames
files = sc.prettyobj()
filedir = sc.thispath() / 'files'
files.excel  = filedir / 'test.xlsx'
files.binary = filedir / 'test.obj'
files.text   = filedir / 'text.txt'
files.zip    = filedir / 'test.zip'
tidyup = True

# Define the test data
nrows = 15
ncols = 3
testdata   = np.zeros((nrows+1, ncols), dtype=object) # Includes header row
testdata[0,:] = ['A', 'B', 'C'] # Create header
testdata[1:,:] = np.random.rand(nrows,ncols) # Create data

def test_spreadsheets():
    '''
    Preserved for completeness, but fairly fragile since relies on not-well-trodden
    Excel libraries.
    '''

    # Test spreadsheet writing, and create the file for later
    formats = {
        'header':{'bold':True, 'bg_color':'#3c7d3e', 'color':'#ffffff'},
        'plain': {},
        'big':   {'bg_color':'#ffcccc'}}
    formatdata = pl.zeros((nrows+1, ncols), dtype=object) # Format data needs to be the same size
    formatdata[1:,:] = 'plain' # Format data
    formatdata[1:,:][testdata[1:,:]>0.7] = 'big' # Find "big" numbers and format them differently
    formatdata[0,:] = 'header' # Format header
    sc.savespreadsheet(filename=files.excel, data=testdata, formats=formats, formatdata=formatdata)

    # Test loading
    sc.heading('Loading spreadsheet')
    data = sc.loadspreadsheet(files.excel)
    print(data)

    excel_path = filedir / 'exampledata.xlsx'
    if os.path.exists(excel_path):
        sc.heading('Reading cells')
        wb = sc.Spreadsheet(filename=excel_path) # Load a sample databook to try pulling cells from
        kw = dict(sheetname='Baseline year population inputs', cells=[[46, 2], [47, 2]])
        celltest_opyxl = wb.readcells(method='openpyxl', **kw, wbargs={'data_only': True}) # Grab cells using openpyxl.  You have to set wbargs={'data_only': True} to pull out cached values instead of formula strings
        celltest_pd    = wb.readcells(method='pandas',   **kw)  # Grab cells using pandas
        print(f'openpyxl output: {celltest_opyxl}')
        print(f'pandas output: {celltest_pd}')
        assert celltest_opyxl == celltest_pd
    else:
        print(f'{excel_path} not found, skipping...')

    sc.heading('Loading a blobject')
    blob = sc.Blobject(files.excel)
    f = blob.tofile()
    wb = openpyxl.load_workbook(f)
    ws = wb.active
    ws['B7'] = 'Hi!     '
    wb.save(f)
    blob.load(f)
    blob.tofile(output=False)
    data = sc.loadspreadsheet(fileobj=blob.bytes)
    print(blob)
    sc.pp(data)

    # Test spreadsheet saving
    sc.heading('Using a Spreadsheet')
    wb = sc.Spreadsheet().new()
    S = sc.Spreadsheet(files.excel)
    S.writecells(cells=['A6','B7','C8','D9'], vals=['This','is','a','test']) # Method 1
    S.writecells(cells=[pl.array([7,1])+i for i in range(4)], vals=['And','so','is','this']) # Method 2
    newdata = (np.random.rand(3,3)*100).round()
    S.writecells(startrow=14, startcol=1, vals=newdata, verbose=True) # Method 3
    S.save()
    data = S.readcells(header=False)
    print(S)
    sc.pp(data)

    if tidyup:
        sc.rmpath(files.excel)
        sc.rmpath('spreadsheet.xlsx')

    return S

def test_load_save():
    '''
    Test file loading and saving
    '''
    o = sc.objdict()

    sc.heading('Save/load')
    sc.save(files.binary, testdata)
    o.obj1 = sc.load(files.binary)
    print(o.obj1)

    sc.heading('Testing other load/save options')
    sc.save(files.binary, testdata, compresslevel=9)
    sc.save(files.binary, testdata, compression='none')
    sc.zsave(files.binary, testdata)
    zobj = sc.load(files.binary, verbose=True)
    assert np.all(o.obj1 == zobj)

    sc.heading('Savetext/loadtext')
    sc.savetext(files.text, testdata)
    sc.savetext(files.text, testdata.tolist())
    o.obj2 = sc.loadtext(files.text)
    sc.loadtext(files.text, splitlines=True)
    print(o.obj2)

    sc.heading('Save/load zip')
    sc.savezip(files.zip, data={'fake_datafile.obj':testdata})
    sc.savezip(files.zip, [files.text, files.binary], basename=True)
    o.zip = sc.loadzip(files.zip) # Load into memory
    sc.unzip(files.zip, outfolder=filedir) # Unpack onto disk

    # Tidy up
    if tidyup:
        sc.blank()
        sc.heading('Tidying up')
        sc.rmpath([files.binary, files.text, files.zip], die=False)

    return o

def test_load_corrupted():
    '''
    Test that loading a corrupted object still works -- specifically, one with
    a no-longer-existant class. See the ``files`` folder for the scripts used
    to create this file.
    '''
    sc.heading('Intentionally loading corrupted file')

    o = sc.objdict()

    class LiveClass():
        def __init__(self, x):
            self.x = x
        def disp(self):
            return f'x is {self.x}'

    dead_path = filedir / 'deadclass.obj'

    print('Loading with no remapping...')
    with pytest.warns(sc.UnpicklingWarning):
        o.obj1 = sc.load(dead_path)
    print(o.obj1)
    print(f'Loading corrupted object succeeded, x={o.obj1.x}')

    print('Loading with remapping...')
    o.obj2 = sc.load(dead_path, remapping={'deadclass.DeadClass': LiveClass})
    o.obj3 = sc.load(dead_path, remapping={('deadclass', 'DeadClass'): LiveClass}, verbose=True)
    for obj in [o.obj2, o.obj3]:
        assert isinstance(obj, LiveClass)
    print(f'Loading remapped object succeeded, {o.obj2.disp()}, object: {o.obj2}')

    print('Loading with error on initialization')
    class DyingClass():
        def __new__(self):
            raise Exception('Intentional exception')
    with pytest.warns(sc.UnpicklingWarning):
        o.obj4 = sc.load(dead_path, remapping={'deadclass.DeadClass': DyingClass}, method='robust', verbose=True)
    with pytest.raises(sc.UnpicklingError):
        sc.load(dead_path, die=True)

    print('Testing Failed directly')
    f = sc.Failed()
    len(f)
    f['foo'] = 3
    assert f['foo'] == 3
    f()
    f.disp()
    f.showfailure()

    return o

def test_fileio():
    '''
    Test other file I/O functions
    '''
    o = sc.objdict()

    # Test thisdir
    sc.heading('Testing thisdir')
    a = sc.thisdir()
    b = sc.thispath()
    assert a == str(b)
    assert 'sciris' in a
    assert 'tests' in a
    sc.thisdir(sc) # Test with a module
    o.thisdir = a

    sc.heading('Get files')
    print(f'Files in "{filedir}":')
    TF = [True,False]
    for tf in TF:
        sc.pp(sc.getfilepaths(folder=filedir, abspath=tf, filesonly=tf, foldersonly=not(tf), nopath=tf, aspath=tf))
    o.filelist = sc.getfilelist(fnmatch='*.py', nopath=True) # Test alias
    assert all(['py' in f for f in o.filelist])
    assert 'test_fileio.py' in o.filelist

    sc.heading('Sanitizing filenames')
    bad = 'Nöt*a   file&name?!.doc'
    good = sc.sanitizefilename(bad)
    assert str(sc.sanitizepath(bad)) == good
    sc.sanitizefilename(bad, strict=True, aspath=None)
    o.sanitized = good

    sc.heading('Testing other')
    path1 = sc.path('/a/folder', 'a_file.txt')
    path2 = sc.path(['/a', 'folder'], 'a_file.txt')
    path3 = sc.path('/a/folder', None, 'a_file.txt')
    assert str(path1) == str(path2) == str(path3) == os.sep.join(['', 'a', 'folder', 'a_file.txt'])
    assert sc.ispath(path1)
    o.thisfile = sc.thisfile(aspath=True)

    return o

def test_json():
    sc.heading('Testing JSON read/write functions')

    not_jsonifiable = sc.Blobject() # Create an object that can't be JSON serialized

    print('Testing jsonifying a NON-jsonifiable object')
    notjson = sc.jsonify(not_jsonifiable, die=False) # Will return a string representation
    sc.sanitizejson(not_jsonifiable, die=True) # Will still not die thanks to jsonpickle

    jsonifiable = sc.objdict().make(keys=['a','b'], vals=np.random.rand(10), coerce='none')
    json_obj = sc.jsonify(jsonifiable)
    json_str = sc.jsonify(jsonifiable, tostring=True, indent=2) # kwargs are passed to json.dumps()
    json = sc.readjson(json_str)
    assert json_obj == json

    print('Not-a-JSON as sanitized object:')
    print(notjson)
    print('JSON as sanitized object:')
    print(json_obj)
    print('JSON as string:')
    print(json_str)

    # Test JSON load/save
    print('Testing JSON load/save...')
    jsonfile = 'test.json'
    testdata = {'key1':np.random.rand(5,5).tolist(), 'key2':['test1', None]}
    sc.savejson(jsonfile, testdata)
    testdata2 = sc.loadjson(jsonfile)
    assert testdata == testdata2

    # Test YAML load/save
    print('Testing YAML load/save...')
    yamlfile = 'test.yaml'
    sc.saveyaml(yamlfile, testdata)
    testdata2 = sc.loadyaml(yamlfile)
    testdata3 = sc.readyaml(sc.loadtext(yamlfile))
    assert testdata == testdata2 == testdata3

    # Tidy up
    sc.rmpath(jsonfile, yamlfile)

    return json_str

def test_jsonpickle():
    sc.heading('Testing jsonpickle functions')

    myobj = sc.prettyobj()
    myobj.a = 3
    myobj.b = ut.MyClass(nan=False, mixed=False, pandas=False) # jsonpickle can't handle mixed data types

    jp = sc.jsonpickle(myobj)
    jps = sc.jsonpickle(myobj, tostring=True)
    myobj2 = sc.jsonunpickle(jp)
    myobj3 = sc.jsonunpickle(jps)

    jpath = 'my-data.json'
    sc.jsonpickle(myobj, jpath)
    myobj4 = sc.jsonunpickle(jpath)

    # Tidy up
    sc.rmpath(jpath)
    assert sc.equal(myobj, myobj2, myobj3, myobj4, leaf=True)

    return jp

def test_load_dump_str():
    sc.heading('Testing load/dump string')
    obj1 = sc.objdict(a=1, b=2)
    string = sc.dumpstr(obj1)
    obj2 = sc.loadstr(string)
    assert obj1 == obj2
    return string

#%% Run as a script
if __name__ == '__main__':
    sc.tic()

    spread  = test_spreadsheets()
    ls      = test_load_save()
    corrupt = test_load_corrupted()
    fileio  = test_fileio()
    json    = test_json()
    jp      = test_jsonpickle()
    string  = test_load_dump_str()

    sc.toc()
    print('Done.')


import speech_recognition as sr

class speech_listener():
    init_rec = None

    def __init__(self) -> None:
        self.init_rec = sr.Recognizer()

    def listen(self) -> str:
        with sr.Microphone() as source:
            print("Let's speak!!")
            audio_data = self.init_rec.record(source, duration=5)
            print("Recognizing your text.............")
            text = self.init_rec.recognize_google(audio_data)
            print(text)
            return text

if __name__ == "__main__":
    s = speech_listener()

    while True:
        print(s.listen())


#!/usr/bin/python3
"""
Prints the first and last name
"""

def say_my_name(first_name, last_name=""):
    """
    Defines a function.
    first_name: Takes the first name as an argument
    last_name: Takes the last name as an argument
    Arguments must be string type
    """
    # Check if thr first name is a string
    if type(first_name) is not str:
        """ Check if the fist name is a string """
        raise TypeError("{:s} must be a string".format("first_name"))
    # Check if the last name is a string
    if type(last_name) is not str:
        """ Check if the last name is a string """
        raise TypeError("{:s} must be a string".format("last_name"))
    # If the first and last name are strings
    else:
        """ Else printsMy name is <first name> <last name> """
        print("My name is {:s} {:s}".format(first_name, last_name))


class Solution:
    def reverse(self, x: int) -> int:
        if x >= 0:temp = str(x)
        else: temp = str(x)[1:]
        answer = temp[::-1]
        if x >= 0: answer = int(answer)
        elif x < 0: answer = -int(answer)

        if (2**31 - 1) >= answer >= -2**31:
            return answer
        else: return 0

#     Time Complexity = O(n)
#     Space Complexity = O(n)


from psycopg_pool import ConnectionPool
import os

def query_wrap_object(template):
  sql = f"""
  (SELECT COALESCE(row_to_json(object_row),'{{}}'::json) FROM (
  {template}
  ) object_row);
  """
  return sql

def query_wrap_array(template):
  sql = f"""
  (SELECT COALESCE(array_to_json(array_agg(row_to_json(array_row))),'[]'::json) FROM (
  {template}
  ) array_row);
  """
  return sql

connection_url = os.getenv("CONNECTION_URL")
pool = ConnectionPool(connection_url)


import datetime
from typing import Dict, List

from pydantic import parse_obj_as
from sqlalchemy.orm import Session
from sqlalchemy.orm.query import Query

from database.get_db_session import get_db
from models.order import Order
from schemas.order import OrderModel

class OrderRepository:
    """Хранилище (слой управления) таблицей с заказами"""

    def __init__(self, db: Session = get_db()):
        self.db = db

    def all(self, q_limit: int = 1000) -> List[Order]:
        """Все записи из БД"""
        query = self.db.query(Order)
        return query.limit(q_limit).all()

    def find(self, order_number: int) -> Order:
        """Поиск по номеру заказа"""
        query = self.db.query(Order)
        return query.filter(Order.order_number == order_number)

    def delete(self, order_number: int) -> None:
        """Удаление по номеру заказа"""
        order_object = self.find(order_number)
        exp = order_object.delete()

    def update(self, order_number: int, update_data: dict):
        """Обновление заказа по его номеру"""
        query = self.find(order_number)
        query.update(update_data)

    def create(self, order: OrderModel, commit=False) -> Order:
        """Создание заказа"""
        db_order = Order(
            table_row=order.table_row,
            order_number=order.order_number,
            usd_price=order.usd_price,
            rub_price=order.rub_price,
            expired_date=order.expired_date,
        )

        self.db.add(db_order)

        if commit:
            self.db.commit()
            self.db.refresh(db_order)

        return db_order

    def create_from_dict(self, order_dict: Dict[int, OrderModel]):
        """Создание заказов из словаря с заказами (pydantic model)"""
        db_data = self.all()

        if not db_data:
            for order in order_dict.values():
                db_order = self.create(order)
            self.db.commit()
        else:
            result = self.compare_data(db_data, order_dict)

    def compare_data(
        self, db_data: List[Order], sheet_data: Dict[int, OrderModel]
    ) -> bool:
        """Сопоставление данных из таблицы с данными из БД
        1. Если запись не изменилась в таблице и есть в БД - удаляем из табличного списка
        2. Если нет, вносим изменения в заказ из бд, потом удаляем
        3. Если номера заказа нет в бд, но есть в таблице - добавляем в бд
        4. Если номера заказа нет в таблице, но есть в бд - удаляем из бд"""

        for record in db_data:
            if record.order_number in sheet_data.keys():
                order_model = parse_obj_as(OrderModel, record)
                sheet_order_model = sheet_data[order_model.order_number]

                order_model.notified = False

                if order_model != sheet_order_model:
                    update_data = sheet_order_model.dict(exclude_unset=True)
                    self.update(record.order_number, update_data)

                    del sheet_data[record.order_number]
                    continue

                del sheet_data[record.order_number]

            else:
                self.delete(record.order_number)

        for v in sheet_data.values():
            self.create(v)

        self.db.commit()

        return True

    def get_outdated_orders(self) -> Query:
        """Получение просроченых поставок"""
        query = self.db.query(Order)
        today = datetime.date.today()
        orders = query.filter(Order.notified == False).filter(
            Order.expired_date < today
        )

        return orders

    def set_notified(self, orders: Query):
        """Установка маркера, что уведомление о просроченной поставке отправлено"""
        orders.update({"notified": True}, synchronize_session=False)

        self.db.commit()


#!/usr/bin/env python
# encoding: utf-8
"""
@author: wushaohong
@time: 2020/8/10 下午3:22
"""
"""273. 整数转换英文表示
将非负整数转换为其对应的英文表示。可以保证给定输入小于 231 - 1 。

示例 1:

输入: 123
输出: "One Hundred Twenty Three"
示例 2:

输入: 12345
输出: "Twelve Thousand Three Hundred Forty Five"
示例 3:

输入: 1234567
输出: "One Million Two Hundred Thirty Four Thousand Five Hundred Sixty Seven"
示例 4:

输入: 1234567891
输出: "One Billion Two Hundred Thirty Four Million Five Hundred Sixty Seven Thousand Eight Hundred Ninety One"""

class Solution:
    def numberToWords(self, num: int) -> str:
        def get(cur):
            s = []
            if cur >= 100:
                cur, last = divmod(cur, 100)
                s += [num2str[cur] + ' ' + 'Hundred']
                cur = last
            if cur >= 20:
                cur, last = divmod(cur, 10)
                s += [num2str[cur * 10]]
                cur = last
            if cur:
                s += [num2str[cur]]
            return ' '.join(s)

        if not num:
            return 'Zero'
        num2str = {1: 'One', 2: 'Two', 3: 'Three', 4: 'Four', 5: 'Five', 6: 'Six', 7: 'Seven', 8: 'Eight', 9: 'Nine',
                   10: 'Ten', 11: 'Eleven', 12: 'Twelve', 13: 'Thirteen', 14: 'Fourteen', 15: 'Fifteen', 16: 'Sixteen',
                   17: 'Seventeen', 18: 'Eighteen', 19: 'Nineteen', 20: 'Twenty', 30: 'Thirty', 40: 'Forty',
                   50: 'Fifty', 60: 'Sixty', 70: 'Seventy',
                   80: 'Eighty', 90: 'Ninety'}
        l = [1000000000, 1000000, 1000, 100]
        title = ['Billion', 'Million', 'Thousand', 'Hundred']
        res = []
        i = 0
        while i < len(l):
            cur, num = divmod(num, l[i])
            temp = get(cur)
            if temp:
                res.append(temp + ' ' + title[i])
            i += 1
        temp = get(num)
        if temp:
            res.append(temp)
        return ' '.join(res)

if __name__ == '__main__':
    sol = Solution()
    print(sol.numberToWords(100) + 'aaa')


from turtle import Turtle
import os
ALIGNMENT = "center"
FONT = ("Courier", 24, "normal")
SCORE_CARD = "score_card.txt"

class ScoreBoard(Turtle):
    def __init__(self):
        super().__init__()
        self.score = 0
        self.high_score = self.get_highscore()
        self.color("white")
        self.penup()
        self.goto(0, 270)
        self.hideturtle()
        self.update_scoreboard()

    def update_scoreboard(self):
        self.clear()
        self.write(f"Score: {self.score} High Score: {self.high_score}",
                   align=ALIGNMENT,
                   font=FONT)

    def store_highscore(self):
        with open(SCORE_CARD, "w") as file:
            file.write(f"{self.high_score}")

    def get_highscore(self):
        if os.path.exists(SCORE_CARD):
            with open(SCORE_CARD) as file:
                content = file.read()
            print(content.strip())
            return int(content.strip())
        else:
            return 0

    def reset(self):
        if self.score > self.high_score:
            self.high_score = self.score
        self.score = 0
        self.update_scoreboard()
        self.store_highscore()

    def increase_score(self):
        self.score += 1
        self.clear()
        self.update_scoreboard()


from ...unrest import UnRest
from .. import idsorted
from ..model import Tree

def test_post_tree_with_pk(client):
    rest = UnRest(client.app, client.session, framework=client.__framework__)
    rest(Tree, methods=['GET', 'POST'])
    code, json = client.fetch(
        '/api/tree/6', method="POST", json={'name': 'mango'}
    )
    assert code == 501

def test_post_tree_with_pk_custom(client):
    rest = UnRest(client.app, client.session, framework=client.__framework__)
    tree = rest(Tree)

    @tree.declare('POST')
    def post(payload, id=None):
        # Make it work anyway
        return tree.post(payload)

    code, json = client.fetch(
        '/api/tree/9', method="POST", json={'name': 'mango'}
    )
    assert code == 200
    assert json['occurences'] == 1
    assert idsorted(json['objects']) == [{'id': 4, 'name': 'mango'}]

    code, json = client.fetch('/api/tree')
    assert json['occurences'] == 4
    assert idsorted(json['objects']) == [
        {'id': 1, 'name': 'pine'},
        {'id': 2, 'name': 'maple'},
        {'id': 3, 'name': 'oak'},
        {'id': 4, 'name': 'mango'},
    ]


import re
d={'I':1,
   'V':5,
   'X':10,
   'L':50,
   'C':100,
   'M':1000}
m1={"January":1,"February":2,"March":3,"April":4,"May":5,"June":6,"July":7,"August":8,
"September":9,"October":10,"November":11,"December":12}

def rep(x):
    x=x.replace('abc','*')
    return x
def rom(x):
    #print(d['M'])
    s=0+d[x[0]]
    for i in range(1,len(x)):
        if d[x[i]]>d[x[i-1]]:
            s=d[x[i]]-s
        else:
            s=s+d[x[i]]
    return s
def date(x):
    x=x.split(',')
    s=x[0].split()
    print("date:"+str(m1[s[0]])+'/'+s[1]+'/'+x[1])
def m(l):
    g=re.findall('[hb][aiu]t',l)
    print(g)

def id(l):
    g=re.findall('[A-Za-z][A-Za-z0-9]*',l)
    print(g)

i=int(input("1:replace abc by * \n2:roman to arabic\n3:date formatting\n4:match pattern\n5:id \n6:not l"))
if i==1:
    x=input('str')
    y=rep(x)
    print(y)
elif i==2:
    x=input("roman num")
    g=rom(x)
    print(g)
elif i==3:
    x=input('date')
    date(x)
elif i==4:
    x=input('sentence')
    m(x)
else:
    x=input('statement')
    id(x)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name='home'),
    path('book-detail/<str:pk>/', views.book_detail, name='book-detail'),
    path('author-detail/<str:pk>/', views.author_detail, name='author-detail'),
    path('store-detail/<str:pk>/', views.store_detail, name='store-detail'),
    path('publisher-detail/<str:pk>/', views.publisher_detail, name='publisher-detail'),
    path('add-book/', views.add_book, name='add-book'),
    path('add-publisher/', views.add_publisher, name='add-publisher'),
    path('add-store/', views.add_store, name='add-store'),
    path('add-author/', views.add_author, name='add-author'),
]


"""
给定两个数组，编写一个函数来计算它们的交集。

示例 1:

输入: nums1 = [1,2,2,1], nums2 = [2,2]
输出: [2,2]
示例 2:

输入: nums1 = [4,9,5], nums2 = [9,4,9,8,4]
输出: [4,9]
"""
from typing import List

class Solution:
    def intersect(self, nums1: List[int], nums2: List[int]) -> List[int]:
        target_list = []
        hash_dict={}
        if len(nums1) < len(nums2):
            min_list= nums1
            larget_list = nums2
        else:
            min_list= nums2
            larget_list = nums1
        for value in min_list:
            if hash_dict.get(value):
                hash_dict[value]+=1
            else:
                hash_dict[value]=1

        for value in larget_list:
            if hash_dict.get(value):
                target_list.append(value)
                hash_dict[value]-=1
                if hash_dict[value]==0:
                    hash_dict.pop(value)
        return target_list

350. 两个数组的交集.py

350. 两个数组的交集.py


from __future__ import unicode_literals
from builtins import zip
from builtins import str
from builtins import object

import json
import textwrap
import cgi
import re
from ..core import component, component_type, get_registered_components
import parameters
from functools import reduce
import os

fields = ['label', 'timestamp', 'reason', 'outcome', 'duration', 'repository',
          'main_file', 'version', 'script_arguments', 'executable',
          'parameters', 'input_data', 'launch_mode', 'output_data',
          'user', 'tags', 'repeats']

@component_type
class Formatter(object):
    required_attributes = ("short", "long")

    def __init__(self, records, project=None, tags=None):
        self.records = records
        self.project = project
        self.tags = None

    def format(self, mode='short'):
        """
        Format a record according to the given mode. ``mode`` may be 'short',
        'long' or 'table'.
        """
        return getattr(self, mode)()

def record2dict(record):
    """Convert a Sumatra record to nested dicts"""
    data = {
        "label": record.label,  # 0.1: 'group'
        "timestamp": record.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
        "reason": record.reason,
        "duration": record.duration,
        "executable": {
            "path": record.executable.path,
            "version": record.executable.version,
            "name": record.executable.name,
            "options": record.executable.options,  # added in 0.3
        },
        "repository": {
            "url": record.repository.url,
            "type": record.repository.__class__.__name__,
            "upstream": record.repository.upstream,  # added in 0.5
        },
        "main_file": record.main_file,
        "version": record.version,
        "parameters": {
            "content": str(record.parameters),
            "type": record.parameters.__class__.__name__,
        },
        "input_data": [{  # changed in 0.4 (previously a list of strings)
            "path": key.path,
            "digest": key.digest,
            "metadata": key.metadata,
            "creation": None if key.creation is None else key.creation.strftime("%Y-%m-%d %H:%M:%S")  # added in 0.7
        } for key in record.input_data],
        "script_arguments": record.script_arguments,  # added in 0.3
        "launch_mode": {
            "type": record.launch_mode.__class__.__name__,
            "parameters": record.launch_mode.__getstate__(),
        },
        "datastore": {
            "type": record.datastore.__class__.__name__,
            "parameters": record.datastore.__getstate__(),
        },
        "input_datastore": {  # added in 0.4
            "type": record.input_datastore.__class__.__name__,
            "parameters": record.input_datastore.__getstate__(),
        },
        "outcome": record.outcome or "",
        "stdout_stderr": record.stdout_stderr,  # added in 0.4
        "output_data": [{  # added in 0.4 (replaced 'data_key', which was a string)
            "path": key.path,
            "digest": key.digest,
            "metadata": key.metadata,
            "creation": None if key.creation is None else key.creation.strftime("%Y-%m-%d %H:%M:%S")  # added in 0.7
        } for key in record.output_data],
        "tags": sorted(list(record.tags)),  # not sure if tags should be PUT, perhaps have separate URL for this?
        "diff": record.diff,
        "user": record.user,  # added in 0.2
        "dependencies": [{
            "path": d.path,
            "version": d.version,
            "name": d.name,
            # "language": d.language,
            "module": d.module,
            "diff": d.diff,
            "source": d.source,  # added in 0.5
        } for d in record.dependencies],
        "platforms": [{
            "system_name": p.system_name,
            "ip_addr": p.ip_addr,
            "architecture_bits": p.architecture_bits,
            "machine": p.machine,
            "architecture_linkage": p.architecture_linkage,
            "version": p.version,
            "release": p.release,
            "network_name": p.network_name,
            "processor": p.processor
        } for p in record.platforms],
        "repeats": record.repeats,  # added in 0.6
    }
    return data

def record2json(record, indent=None):
    """Encode a Sumatra record as JSON."""
    data = record2dict(record)
    return json.dumps(data, indent=indent)

@component
class JSONFormatter(Formatter):
    name = "json"

    def short(self, indent=2):
        return "[" + ",\n".join(record2json(record, indent=indent)
                                for record in self.records) + "]"

    def long(self, indent=2):
        return self.short(indent=indent)

@component
class TextFormatter(Formatter):
    """
    Format the information from a list of Sumatra records as text.
    """
    name = "text"

    def short(self):
        """Return a list of record labels, one per line."""
        return "\n".join(record.label for record in self.records)

    def long(self, text_width=80, left_column_width=17):
        """
        Return detailed information about a list of records, as text with a
        limited column width. Lines that are too long will be wrapped round.
        """
        output = ""
        for record in self.records:
            output += "-" * text_width + "\n"
            left_column = []
            right_column = []
            for field in fields:
                left_column.append("%s%ds" % ("%-", left_column_width) % field.title())
                entry = getattr(record, field)
                if hasattr(entry, "pretty"):
                    new_lines = entry.pretty().split("\n")
                else:
                    if callable(entry):
                        entryStr = str(entry())
                    elif hasattr(entry, "items"):
                        entryStr = ", ".join(["%s=%s" % item for item in entry.items()])
                    elif isinstance(entry, set):
                        entryStr = ", ".join(entry)
                    else:
                        entryStr = str(entry)
                        if field == 'version' and getattr(record, 'diff'):
                            # if there is a diff, append '*' to the version
                            entryStr += "*"
                    entry_lines = entryStr.split("\n")
                    new_lines = []
                    for entry_line in entry_lines:
                        new_lines.extend(
                            textwrap.wrap(entry_line, width=text_width, replace_whitespace=False))
                if len(new_lines) == 0:
                    new_lines = ['']
                right_column.extend(new_lines)
                if len(new_lines) > 1:
                    left_column.extend([' ' * left_column_width] * (len(new_lines) - 1))
                # import pdb; pdb.set_trace()
            for left, right in zip(left_column, right_column):
                output += left + ": " + right + "\n"
        return output

    def table(self):
        """
        Return information about a list of records as text, in a simple
        tabular format.
        """
        tt = TextTable(fields, self.records)
        return str(tt)

    def output_table(self):
        """Return a list of output files, one per line."""
        dt = DataTable(self.records, max_column_width=20, seperator='|')
        return str(dt)

    def parameter_table(self):
        """ Return parameter information about a list of records as text, in a simple tabular format."""
        pt = ParamsTable(self.records, max_column_width=13, seperator='|')
        return str(pt)

class TextTable(object):
    """
    Very primitive implementation of a text table. There are more sophisticated
    implementations around, e.g. http://pypi.python.org/pypi/texttable/0.6.0/
    but for now I'd like to avoid too many dependencies.
    """

    def __init__(self, headers, rows, max_column_width=20):
        self.headers = headers
        self.rows = rows
        self.max_column_width = max_column_width

    def calculate_column_widths(self):
        column_widths = []
        for header in self.headers:
            column_width = max([len(header)] + [len(str(getattr(row, header))) for row in self.rows])
            column_widths.append(min(self.max_column_width, column_width))
        return column_widths

    def __str__(self):
        column_widths = self.calculate_column_widths()
        format = "| " + " | ".join("%%-%ds" % w for w in column_widths) + " |\n"
        assert len(column_widths) == len(self.headers)
        output = format % tuple(h.title() for h in self.headers)
        for row in self.rows:
            output += format % tuple(str(getattr(row, header))[:self.max_column_width] for header in self.headers)
        return output

class DataTable(object):

    def __init__(self, rows, max_column_width=20, seperator='|'):
        self.rows = rows
        self.headers = self.get_headers()
        self.max_column_width = max_column_width
        self.seperator = seperator

    def get_headers(self):
        return ['output_from_record', 'directory', 'filename', 'digest', 'creation', 'size', 'mimetype']

    def calculate_column_widths(self):
        column_widths = []
        for header in self.headers:
            column_val_width = []
            for row in self.rows:
                for output_file in row.output_data:
                    if header == 'output_from_record':
                        column_val_width.append(len(str(row.label)))
                    if header == 'directory':
                        column_val_width.append(len(str(os.path.dirname(output_file.path))))
                    if header == 'filename':
                        column_val_width.append(len(str(os.path.basename(output_file.path))))
                    if header in output_file.__dict__:
                        column_val_width.append(len(str(output_file.__dict__[header])))
                    elif header in output_file.__dict__['metadata']:
                        column_val_width.append(len(str(output_file.__dict__['metadata'][header])))
            column_width = max([len(header)] + column_val_width)
            column_widths.append(min(self.max_column_width, column_width))
        return column_widths

    def __str__(self):
        column_widths = self.calculate_column_widths()
        if self.seperator == '|':
            format = "| " + " | ".join("%%-%ds" % w for w in column_widths) + " |\n"
        else:
            format = self.seperator.join(len(column_widths)*["%s"]) + "\n"
        assert len(column_widths) == len(self.headers)
        output = format % tuple(h[:self.max_column_width] for h in self.headers)
        for row in self.rows:
            for output_file in row.output_data:
                output += format % tuple(
                    [str(row.label[:self.max_column_width])]
                    +[str(os.path.dirname(output_file.path)[:self.max_column_width])]
                    +[str(os.path.basename(output_file.path)[:self.max_column_width])]
                    +[str(getattr(output_file, header))[:self.max_column_width] for header in self.headers[3:5]]
                    +[str(output_file.metadata[header])[:self.max_column_width] for header in self.headers[5:]])
        return output

class ParamsTable(object):

    def __init__(self, rows, max_column_width=20, seperator='|'):
        self.rows = rows
        self.headers = self.get_headers()
        self.max_column_width = max_column_width
        self.seperator = seperator

    def get_headers(self):
        headers = [u'label',u'version',u'main_file']
        param_headers = []
        for row in self.rows:
            parameter_set = row.parameters
            if hasattr(parameter_set, 'as_dict'):
                parameter_set = parameter_set.as_dict()
            parameter_set = parameters.nesteddictflatten(parameter_set)
            for key in parameter_set.keys():
                if key not in param_headers:
                    param_headers.append(key)
        param_headers.sort()
        return headers + param_headers

    def calculate_column_widths(self):
        column_widths = []
        for header in self.headers:
            if header in [u'label',u'version',u'main_file']:
                column_width = max([len(header)] + [len(str(getattr(row, header))) for row in self.rows])
            else:
                column_val_width = []
                for row in self.rows:
                    parameter_set = row.parameters
                    if hasattr(parameter_set, 'as_dict'):
                        parameter_set = parameter_set.as_dict()
                    parameter_set = parameters.nesteddictflatten(parameter_set)
                    column_val_width.append(len(str(parameter_set.get(header))))
                column_width = max([len(header)] + column_val_width)
            column_widths.append(min(self.max_column_width, column_width))
        return column_widths

    def __str__(self):
        column_widths = self.calculate_column_widths()
        if self.seperator == '|':
            format = "| " + " | ".join("%%-%ds" % w for w in column_widths) + " |\n"
        else:
            format = self.seperator.join(len(column_widths)*["%s"]) + "\n"
        assert len(column_widths) == len(self.headers)
        output = format % tuple(h[:self.max_column_width] for h in self.headers)
        for row in self.rows:
            parameter_set = row.parameters
            if hasattr(parameter_set, 'as_dict'):
                parameter_set = parameter_set.as_dict()
            parameter_set = parameters.nesteddictflatten(parameter_set)
            output += format % tuple(
                [str(getattr(row, header))[:self.max_column_width] for header in self.headers[:3]]
               +[str(parameter_set.get(header,''))[:self.max_column_width] for header in self.headers[3:]])
        return output

@component
class ShellFormatter(Formatter):
    """
    Create a shell script that can be used to repeat a series of computations.
    """
    name = "shell"

    def short(self):
        import operator
        output = ("#!/bin/sh\n"
                  "#\n"
                  "# This script was generated by Sumatra (http://neuralensemble.org/sumatra)\n"
                  "# It replays computations from the project '%s'\n" % self.project.name)
        if self.tags:
            output += "# tagged with %s\n" % ",".join(self.tags)
        if self.project.description:
            output += textwrap.TextWrapper(initial_indent="# ", subsequent_indent="# ").fill(self.project.description)
        cleanup = "\n\n# Clean-up temporary files\n"

        output += "\n# Original hardware environment:\n"
        platforms = list(set(reduce(operator.add, [record.platforms for record in self.records])))
        for i, platform in enumerate(platforms):
            output += "#   Machine #%d: %s processor running %s. %s(%s)\n" % (i + 1, platform.machine, platform.version, platform.network_name, platform.ip_addr, )

        output += "\n# Original software environment:\n"
        repositories = list(set(record.repository for record in self.records))
        if len(repositories) > 1:
            raise NotImplementedError
        else:
            repository = repositories[0] # choose arbitrarily, repository is unique anyway
            output += "#   %s repository at %s\n" % (repository.vcs_type, repository.upstream or repository.url)
        dependency_sets = list(set(tuple(sorted(record.dependencies)) for record in self.records))
        for i, dependency_set in enumerate(dependency_sets):
            output += "#   Dependency set #%d: %s\n" % (i + 1, dependency_set)

        current_directory = ''
        current_version = ''
        for record in reversed(self.records):  # oldest first
            output += "\n# " + "-" * 77 + "\n"
            output += "# %s\n" % record.label
            output += "# Originally run on %s by %s\n" % (record.timestamp.strftime("%Y-%m-%d at %H:%M:%S"), record.user)
            if len(platforms) > 1:
                output += "# on machines %s (see above)\n" % ", ".join(str(platforms.index(platform) + 1) for platform in record.platforms)
            if record.reason:
                output += "# for the following reason: '%s'\n" % record.reason
            if record.outcome:
                output += "# with the following outcome: '%s'\n" % record.outcome
            output += "# Duration: %s\n" % human_readable_duration(record.duration)
            output += "# %s version %s\n" % (record.executable.name, record.executable.version)
            if len(dependency_sets) > 1:
                output += "# Dependencies were as in set #%d above\n" % (dependency_sets.index(tuple(sorted(record.dependencies))) + 1,)
            if record.repeats:
                output += "# This was a repeat of a previous computation, #%s\n" % record.repeats
            output += "\n"
            # switch to the correct working directory
            if record.launch_mode.working_directory != current_directory:
                output += "cd %s\n" % record.launch_mode.working_directory
                current_directory = record.launch_mode.working_directory
            # checkout the correct version of the code from the repository
            # then patch if necessary
            if record.version != current_version or record.diff:
                output += "%s %s\n" % (record.repository.use_version_cmd, record.version)
                current_version = record.version
            if record.diff:
                diff_file = "%s.patch" % record.label
                with open(diff_file, 'w') as fp:
                    fp.write(record.diff + '\n')
                output += "%s %s\n" % (record.repository.apply_patch_cmd, diff_file)
                cleanup += "rm -f %s\n" % diff_file
            # handle pre_run
            if hasattr(record.executable, 'pre_run'):
                output += record.executable.prerun + "\n"
            # build main run command
            # can we add assertions about the SHA1 hash of any input files?
            command_line = record.command_line
            if record.parameters:
                parameter_file_basename = record.label.replace("/", "_")
                parameter_file = record.executable.write_parameters(record.parameters, parameter_file_basename)
                command_line = command_line.replace("<parameters>", parameter_file)
                cleanup += "rm -f %s\n" % parameter_file
            output += command_line + "\n"
        output += cleanup
        return output

# could try embedding patches within shell script using heredoc

    def long(self):
        # perhaps have all the comments in the long version but not the short one
        return self.short()

@component
class HTMLFormatter(Formatter):
    """
    Format information about a group of Sumatra records as HTML fragments, to
    be included in a larger document.
    """
    name = "html"

    def short(self):
        """
        Return a list of record labels as an HTML unordered list.
        """
        return "
\n
" + "
\n
".join(record.label for record in self.records) + "
\n
"

    def long(self):
        """
        Return detailed information about a list of records as an HTML
        description list.
        """
        def format_record(record):
            output = "
%s
\n
\n
\n" % record.label
            for field in fields:
                output += "
%s

%s
\n" % (field, cgi.escape(str(getattr(record, field))))
            output += "
\n
"
            return output
        return "
\n" + "\n".join(format_record(record) for record in self.records) + "\n
"

    def table(self):
        """
        Return detailed information about a list of records as an HTML table.
        """
        def format_record(record):
            return "
\n    " + "	\n    ".join(cgi.escape(str(getattr(record, field))) for field in fields) + "    	\n
"
        return "
\n" + \
               "
\n    " + "	\n    ".join(field.title() for field in fields) + "    	\n
\n" + \
               "\n".join(format_record(record) for record in self.records) + \
               "\n
"

@component
class LaTeXFormatter(Formatter):
    name = "latex"
    SUBSTITUTIONS = (
        (re.compile(r'\\'), r'\\textbackslash'),
        (re.compile(r'([{}_#%&$])'), r'\\\1'),
        (re.compile(r'~'), r'\~{}'),
        (re.compile(r'\^'), r'\^{}'),
        (re.compile(r'"'), r"''"),
        (re.compile(r'\.\.\.+'), r'\\ldots'),
        (re.compile(r'\<'), r'\\textless{}'),
        (re.compile(r'\>'), r'\\textgreater{}')
    )

    @staticmethod
    def _escape_tex(value):
        """Inspired by http://flask.pocoo.org/snippets/55/"""
        newval = value
        for pattern, replacement in LaTeXFormatter.SUBSTITUTIONS:
            newval = pattern.sub(replacement, newval)
        newval = newval.replace('/', '/ ')
        return newval

    def short(self):
        return self.long()

    def long(self):
        from os.path import dirname, join
        from jinja2 import Environment, FileSystemLoader
        template_paths = [dirname(__file__)]
        if self.project:
            template_paths.insert(0, join(self.project.path, ".smt"))
        env = Environment(loader=FileSystemLoader(template_paths))
        env.block_start_string = '{%'
        env.block_end_string = '%}'
        env.variable_start_string = '@'
        env.variable_end_string = '@'
        env.filters['human_readable_duration'] = human_readable_duration
        env.filters['escape_tex'] = LaTeXFormatter._escape_tex
        template = env.get_template('latex_template.tex')
        return template.render(records=self.records,
                               project=self.project,
                               paper_size='a4paper')

@component
class TextDiffFormatter(Formatter):
    """
    Format information about the differences between two Sumatra records in
    text format.
    """
    name = "textdiff"

    def __init__(self, diff):
        self.diff = diff

    def short(self):
        """Return a summary of the differences between two records."""
        def yn(x):
            return x and "yes" or "no"
        D = self.diff
        output = textwrap.dedent("""\
            Record 1                : %s
            Record 2                : %s
            Executable differs      : %s
            Code differs            : %s
              Repository differs    : %s
              Main file differs     : %s
              Version differs       : %s
              Non checked-in code   : %s
              Dependencies differ   : %s
            Launch mode differs     : %s
            Input data differ       : %s
            Script arguments differ : %s
            Parameters differ       : %s
            Data differ             : %s""" % (
            D.recordA.label,
            D.recordB.label,
            yn(D.executable_differs),
            yn(D.code_differs),
            yn(D.repository_differs), yn(D.main_file_differs),
            yn(D.version_differs), yn(D.diff_differs),
            yn(D.dependencies_differ),
            yn(D.launch_mode_differs),
            yn(D.input_data_differ),
            yn(D.script_arguments_differ),
            yn(D.parameters_differ),
            yn(D.output_data_differ))
        )
        return output

    def long(self):
        """
        Return a detailed description of the differences between two records.
        """
        output = ''
        if self.diff.executable_differs:
            output += "Executable differences:\n"
            for record in (self.diff.recordA, self.diff.recordB):
                output += "  %s: %s\n" % (record.label, record.executable)
        if self.diff.code_differs:
            output += "Code differences:\n"
            for record in (self.diff.recordA, self.diff.recordB):
                output += "  %s: main file '%s' at version %s in %s\n" % (record.label, record.main_file, record.version, record.repository)
            if self.diff.diff_differs:
                for record in (self.diff.recordA, self.diff.recordB):
                    output += "  %s:\n %s\n" % (record.label, record.diff)
        if self.diff.dependencies_differ:
            diffs = self.diff.dependency_differences
            output += "Dependency differences:\n"
            for name, (depA, depB) in diffs.items():
                if depA and depB:
                    output += "  %s\n" % name
                    output += "    A: version=%s\n" % depA.version
                    output += "       %s\n" % depA.diff.replace("\n", "\n       ")
                    output += "    B: version=%s\n" % depB.version
                    output += "       %s\n" % depA.diff.replace("\n", "\n       ")
                elif depB is None:
                    output += "  %s is a dependency of %s but not of %s\n" % (name, self.diff.recordA.label, self.diff.recordB.label)
                elif depA is None:
                    output += "  %s is a dependency of %s but not of %s\n" % (name, self.diff.recordB.label, self.diff.recordA.label)

        diffs = self.diff.launch_mode_differences
        if diffs:
            output += "Launch mode differences:\n"
            modeA, modeB = diffs
            output += "  %s: %s\n" % (self.diff.recordA.label, modeA)
            output += "  %s: %s\n" % (self.diff.recordB.label, modeB)
        if self.diff.parameters_differ:
            output += "Parameter differences:\n"
            for record, param_diff in zip((self.diff.recordA, self.diff.recordB),
                                           self.diff.parameter_differences):
                output += "  %s:\n    %s\n" % (record.label, param_diff)
        if self.diff.input_data_differ:
            output += "Input data differences:\n"
            for record in (self.diff.recordA, self.diff.recordB):
                output += "  %s: %s\n" % (record.label, record.input_data)
        if self.diff.script_arguments_differ:
            output += "Script argument differences:\n"
            for record in (self.diff.recordA, self.diff.recordB):
                output += "  %s: %s\n" % (record.label, record.script_arguments)
        AnotB, BnotA = self.diff.output_data_differences
        if AnotB or BnotA:
            output += "Output data differences:\n"
            if AnotB:
                output += "  Generated by %s:\n" % self.diff.recordA.label
                for key in AnotB:
                    output += "    %s\n" % key
            if BnotA:
                output += "  Generated by %s:\n" % self.diff.recordB.label
                for key in BnotA:
                    output += "    %s\n" % key
        return output

def get_formatter(format):
    """
    Return a :class:`Formatter` object of the appropriate type. ``format``
    may be 'text, 'html' or 'textdiff'
    """
    return get_registered_components(Formatter)[format]

def get_diff_formatter():
    """
    Return a :class:`DiffFormatter` object of the appropriate type. Only
    text format is currently available.
    """
    return TextDiffFormatter

def _quotient_remainder(dividend, divisor):
    q = dividend // divisor
    r = dividend - q * divisor
    return (q, r)

def human_readable_duration(seconds):
    """
    Coverts seconds to human readable unit

    >>> human_readable_duration(((6 * 60 + 32) * 60 + 12))
    '6h 32m 12.00s'
    >>> human_readable_duration((((8 * 24 + 7) * 60 + 6) * 60 + 5))
    '8d 7h 6m 5.00s'
    >>> human_readable_duration((((8 * 24 + 7) * 60 + 6) * 60))
    '8d 7h 6m'
    >>> human_readable_duration((((8 * 24 + 7) * 60) * 60))
    '8d 7h'
    >>> human_readable_duration((((8 * 24) * 60) * 60))
    '8d'
    >>> human_readable_duration((((8 * 24) * 60) * 60) + 0.12)
    '8d 0.12s'

    """
    if seconds is None:
        return "None"
    else:
        from math import modf
        (fractional_part, integer_part) = modf(seconds)
        (d, rem) = _quotient_remainder(int(integer_part), 60 * 60 * 24)
        (h, rem) = _quotient_remainder(rem, 60 * 60)
        (m, rem) = _quotient_remainder(rem, 60)
        s = rem + fractional_part

        return ' '.join(
            templ.format(val)
            for (val, templ) in [
                (d, '{0}d'),
                (h, '{0}h'),
                (m, '{0}m'),
                (s, '{0:.2f}s'),
            ]
            if val != 0
        )


from flask import Flask, render_template, request
from flask_mysqldb import MySQL

app = Flask(__name__)

#set up DB
mysql = MySQL()

app.config['MYSQL_USER'] = 'user1'
app.config['MYSQL_PASSWORD'] = '0Database!'
app.config['MYSQL_DB'] = 'MoviePal'
app.config['MYSQL_HOST'] = 'localhost'

mysql.init_app(app)

#global for now, change later
movie_ids = []
movie_titles = []

@app.route('/')
def main():
  #store movie_titles and ids somewhere
  global movie_ids, movie_titles
  movies = list(users())
  movie_id_list = [str(i[0]) for i in movies]
  movie_ids = movie_id_list
  movie_title_list = [str(i[1]) for i in movies]
  movie_titles = movie_title_list
  return render_template('index.html', movies = zip(movie_ids,movie_titles))

def users():
  cur = mysql.connection.cursor()
  cur.execute('''SELECT table1.movie_id,title from movie as table1 inner join
                (SELECT movie_id, COUNT(*) AS magnitude  FROM UserRating  GROUP BY movie_id  ORDER BY magnitude DESC LIMIT 10)
                as table2 ON (table1.movie_id = table2.movie_id)''')
  movies = cur.fetchall()
  return movies

@app.route('/recommendations', methods=['POST'])
def recommendations():
  #maps movie_ids to ratings
  ratings = {}
  for id in movie_ids:
    ratings[id] = int(request.form.get(id))

  #Generate recommendations using ratings
  generateRecommendations(ratings)
  return render_template('recommendations.html')

def generateRecommendations(ratings):
  #keep only non-zero entries
  ratings = {k:v for k,v in ratings.items() if v != 0}

if __name__ == "__main__":
  app.run()


import mxnet as mx

eps = 1e-10 + 1e-5
bn_mom = 0.9
fix_gamma = False

def ConvFactory(data, num_filter, kernel, stride=(1,1), pad=(0, 0), name=None, suffix='', attr={}):
    conv = mx.symbol.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, name='conv_%s%s' %(name, suffix))
    bn = mx.symbol.BatchNorm(data=conv, fix_gamma=fix_gamma, eps=eps, momentum=bn_mom, name='bn_%s%s' %(name, suffix))
    act = mx.symbol.Activation(data=bn, act_type='relu', name='relu_%s%s' %(name, suffix), attr=attr)
    return act

def InceptionFactoryA(data, num_1x1, num_3x3red, num_3x3, num_d3x3red, num_d3x3, pool, proj, name):
    # 1x1
    c1x1 = ConvFactory(data=data, num_filter=num_1x1, kernel=(1, 1), name=('%s_1x1' % name))
    # 3x3 reduce + 3x3
    c3x3r = ConvFactory(data=data, num_filter=num_3x3red, kernel=(1, 1), name=('%s_3x3' % name), suffix='_reduce')
    c3x3 = ConvFactory(data=c3x3r, num_filter=num_3x3, kernel=(3, 3), pad=(1, 1), name=('%s_3x3' % name))
    # double 3x3 reduce + double 3x3
    cd3x3r = ConvFactory(data=data, num_filter=num_d3x3red, kernel=(1, 1), name=('%s_double_3x3' % name), suffix='_reduce')
    cd3x3 = ConvFactory(data=cd3x3r, num_filter=num_d3x3, kernel=(3, 3), pad=(1, 1), name=('%s_double_3x3_0' % name))
    cd3x3 = ConvFactory(data=cd3x3, num_filter=num_d3x3, kernel=(3, 3), pad=(1, 1), name=('%s_double_3x3_1' % name))
    # pool + proj
    pooling = mx.symbol.Pooling(data=data, kernel=(3, 3), stride=(1, 1), pad=(1, 1), pool_type=pool, name=('%s_pool_%s_pool' % (pool, name)))
    cproj = ConvFactory(data=pooling, num_filter=proj, kernel=(1, 1), name=('%s_proj' %  name))
    # concat
    concat = mx.symbol.Concat(*[c1x1, c3x3, cd3x3, cproj], name='ch_concat_%s_chconcat' % name)
    return concat

def InceptionFactoryB(data, num_3x3red, num_3x3, num_d3x3red, num_d3x3, name):
    # 3x3 reduce + 3x3
    c3x3r = ConvFactory(data=data, num_filter=num_3x3red, kernel=(1, 1), name=('%s_3x3' % name), suffix='_reduce')
    c3x3 = ConvFactory(data=c3x3r, num_filter=num_3x3, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name=('%s_3x3' % name))
    # double 3x3 reduce + double 3x3
    cd3x3r = ConvFactory(data=data, num_filter=num_d3x3red, kernel=(1, 1),  name=('%s_double_3x3' % name), suffix='_reduce')
    cd3x3 = ConvFactory(data=cd3x3r, num_filter=num_d3x3, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name=('%s_double_3x3_0' % name))
    cd3x3 = ConvFactory(data=cd3x3, num_filter=num_d3x3, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name=('%s_double_3x3_1' % name))
    # pool + proj
    pooling = mx.symbol.Pooling(data=data, kernel=(3, 3), stride=(2, 2), pad=(1, 1), pool_type="max", name=('max_pool_%s_pool' % name))
    # concat
    concat = mx.symbol.Concat(*[c3x3, cd3x3, pooling], name='ch_concat_%s_chconcat' % name)
    return concat

# A Simple Downsampling Factory
def DownsampleFactory(data, ch_3x3, name, attr):
    # conv 3x3
    conv = ConvFactory(data=data, name=name+'_conv',kernel=(3, 3), stride=(2, 2), num_filter=ch_3x3, pad=(1, 1), attr=attr)
    # pool
    pool = mx.symbol.Pooling(data=data, name=name+'_pool',kernel=(3, 3), stride=(2, 2), pad=(1, 1), pool_type='max', attr=attr)
    # concat
    concat = mx.symbol.Concat(*[conv, pool], name=name+'_ch_concat')
    return concat

# A Simple module
def SimpleFactory(data, ch_1x1, ch_3x3, name, attr):
    # 1x1
    conv1x1 = ConvFactory(data=data, name=name+'_1x1', kernel=(1, 1), pad=(0, 0), num_filter=ch_1x1, attr=attr)
    # 3x3
    conv3x3 = ConvFactory(data=data, name=name+'_3x3', kernel=(3, 3), pad=(1, 1), num_filter=ch_3x3, attr=attr)
    #concat
    concat = mx.symbol.Concat(*[conv1x1, conv3x3], name=name+'_ch_concat')
    return concat

def get_symbol(num_classes, image_shape, **kwargs):
    image_shape = [int(l) for l in image_shape.split(',')]
    (nchannel, height, width) = image_shape
    # attr = {'force_mirroring': 'true'}
    attr = {}

    # data
    data = mx.symbol.Variable(name="data")
    if height <= 28:
        # a simper version
        conv1 = ConvFactory(data=data, kernel=(3,3), pad=(1,1), name="1", num_filter=96, attr=attr)
        in3a = SimpleFactory(conv1, 32, 32, 'in3a', attr)
        in3b = SimpleFactory(in3a, 32, 48, 'in3b', attr)
        in3c = DownsampleFactory(in3b, 80, 'in3c', attr)
        in4a = SimpleFactory(in3c, 112, 48, 'in4a', attr)
        in4b = SimpleFactory(in4a, 96, 64, 'in4b', attr)
        in4c = SimpleFactory(in4b, 80, 80, 'in4c', attr)
        in4d = SimpleFactory(in4c, 48, 96, 'in4d', attr)
        in4e = DownsampleFactory(in4d, 96, 'in4e', attr)
        in5a = SimpleFactory(in4e, 176, 160, 'in5a', attr)
        in5b = SimpleFactory(in5a, 176, 160, 'in5b', attr)
        pool = mx.symbol.Pooling(data=in5b, pool_type="avg", kernel=(7,7), name="global_pool", attr=attr)
    else:
        # stage 1
        conv1 = ConvFactory(data=data, num_filter=64, kernel=(7, 7), stride=(2, 2), pad=(3, 3), name='1')
        pool1 = mx.symbol.Pooling(data=conv1, kernel=(3, 3), stride=(2, 2), name='pool_1', pool_type='max')
        # stage 2
        conv2red = ConvFactory(data=pool1, num_filter=64, kernel=(1, 1), stride=(1, 1), name='2_red')
        conv2 = ConvFactory(data=conv2red, num_filter=192, kernel=(3, 3), stride=(1, 1), pad=(1, 1), name='2')
        pool2 = mx.symbol.Pooling(data=conv2, kernel=(3, 3), stride=(2, 2), name='pool_2', pool_type='max')
        # stage 2
        in3a = InceptionFactoryA(pool2, 64, 64, 64, 64, 96, "avg", 32, '3a')
        in3b = InceptionFactoryA(in3a, 64, 64, 96, 64, 96, "avg", 64, '3b')
        in3c = InceptionFactoryB(in3b, 128, 160, 64, 96, '3c')
        # stage 3
        in4a = InceptionFactoryA(in3c, 224, 64, 96, 96, 128, "avg", 128, '4a')
        in4b = InceptionFactoryA(in4a, 192, 96, 128, 96, 128, "avg", 128, '4b')
        in4c = InceptionFactoryA(in4b, 160, 128, 160, 128, 160, "avg", 128, '4c')
        in4d = InceptionFactoryA(in4c, 96, 128, 192, 160, 192, "avg", 128, '4d')
        in4e = InceptionFactoryB(in4d, 128, 192, 192, 256, '4e')
        # stage 4
        in5a = InceptionFactoryA(in4e, 352, 192, 320, 160, 224, "avg", 128, '5a')
        in5b = InceptionFactoryA(in5a, 352, 192, 320, 192, 224, "max", 128, '5b')
        # global avg pooling
        pool = mx.symbol.Pooling(data=in5b, kernel=(7, 7), stride=(1, 1), name="global_pool", pool_type='avg')

    # linear classifier
    flatten = mx.symbol.Flatten(data=pool)
    fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=num_classes)
    softmax = mx.symbol.SoftmaxOutput(data=fc1, name='softmax')
    return softmax


# 문자열 재정렬
String = "AJKDLSI412K4JSJ9D"
sum = 0
result = ''

for s in String:
    if '0' <= s and s <= '9':
        sum += int(s)
    else:
        result += s
result = sorted(result)
print(''.join(result) + str(sum))


import random
def isgreatorequal(a,max):
    if int(str(max)+str(a)) > int(str(a)+str(max)):
        return False
    else:
        return True

def MaxSalary(l):
    out = ''
    while l != []:
        max = 0
        maxindex = 0
        for i in range(len(l)):
            if isgreatorequal(l[i],max):
                max = l[i]
                maxindex = i
        out = out+str(l.pop(maxindex))
    return out

l = [23,39,92]
n = random.randint(1,100)
l = [random.randint(0,1000) for i in range(n)]
print(MaxSalary(l))


# implement based on 'load_nii.py' file
# should prepare the sliced the 2D images in advance

# test if pixel-wise product works for image segmentation

import cv2
import os
import numpy as np

root_path = '/home/zhaoxiang/dataset/abdnomial_images/data_lesion'            # for MSD
# root_path = '/home/zhaoxiang/dataset/abdnomial_images/data_normal'            # for Altas

folders = os.listdir(root_path)
folders.sort()
for folder in folders:
    files_path = os.path.join(root_path, folder, 'raw')
    files_gt_path = os.path.join(root_path, folder, 'gt')
    # read the files in the folder
    files = os.listdir(files_path)
    files.sort()

    for file in files:
        if file.startswith('.'):
            continue

        number = file.split('_')[0]

        # read the raw image
        raw_path = os.path.join(files_path, file)
        raw_img = cv2.imread(raw_path, cv2.IMREAD_GRAYSCALE) # img.dtype('uint8')

        # read ground truth
        gt_name = number + '_gt.png'
        gt_path = os.path.join(files_gt_path, gt_name)
        gt_img = cv2.imread(gt_path, cv2.IMREAD_GRAYSCALE)

        # raw_img histogram equalization
        hist_img = cv2.equalizeHist(raw_img)

        if np.max(gt_img) != 0:
            # generate the mask and produce the liver image
            mask = np.where(gt_img > 0 , 1, 0)
            liver_img = np.multiply(mask, hist_img)

            # save the image
            new_folder = os.path.join(root_path, folder, 'hist_segmentation_flip')
            if not os.path.exists(new_folder):
                os.mkdir(new_folder)

            new_img_name = number + '_hist_segmentation_flip.png'
            new_img_path = os.path.join(new_folder, new_img_name)

            # flip the image (vertically)
            if root_path.endswith('normal'):
                flipped = cv2.flip(liver_img,0)

                #
                cv2.imwrite(new_img_path, flipped)
            elif root_path.endswith('lesion'):
                cv2.imwrite(new_img_path, liver_img)
        else:
            continue
    print(folder, '  done')


def top_k(nums, k):
  list = []
  count_list = []
  result = []
  real = []
  for i in range(len(nums)):
    list.append(nums.count(nums[i]))
  for j in range(len(list)):
    if list[j] not in count_list:
      count_list.append(list[j])
  for n in range(k):
    count_list.sort(reverse=True)
    result.append(count_list[n])
  for m in range(len(result)):
    real.append(nums[list.index(result[m])])
  return real

빈도 높은 수 k개 반환.py


from typing import List, Tuple, Set, Dict
import numpy as np
from environments.n_puzzle import NPuzzle, NPuzzleState

import torch
from torch import nn

from environments.environment_abstract import Environment, State
from utils import env_utils
from utils.misc_utils import evaluate_cost_to_go
from utils.nnet_utils import states_nnet_to_pytorch_input
from utils.misc_utils import flatten

import pickle, random

from to_implement.functions import get_nnet_model, train_nnet, value_iteration
import pdb, traceback, sys

#from torch.utils.tensorboard import SummaryWriter
from matplotlib import pyplot as plt

from numba import jit, njit, vectorize, cuda, uint32, f8, uint8

def main():
    torch.set_num_threads(1)

    #Set up TesnorBoard writer
    #writer = SummaryWriter()

    # get environment
    env: Environment = env_utils.get_environment("puzzle8")

    # get nnet model
    nnet: nn.Module = get_nnet_model()
    device = torch.device('cpu')
    batch_size: int = 100
    num_itrs_per_vi_update: int = 200
    #num_vi_updates: int = 1
    num_vi_updates: int = 50

    # get data
    print("Preparing Data\n")
    data = pickle.load(open("data/data.pkl", "rb"))

    # train with supervised learning
    print("Training DNN\n")
    train_itr: int = 0
    #for vi_update in range(num_vi_updates):
    for vi_update in range(1):
        print("--- Value Iteration Update: %i ---" % vi_update)
        states: List[State] = env.generate_states(batch_size*num_itrs_per_vi_update, (0, 500))
        #states: List[State] = env.generate_states(200, (0, 500))

        states_nnet: np.ndarray = env.state_to_nnet_input(states)

        inputs_tensor = torch.from_numpy(states_nnet).float()
        #writer.add_graph(nnet, inputs_tensor)

        outputs_np = value_iteration(nnet, device, env, states)
        outputs = np.expand_dims(np.array(outputs_np), 1)

        nnet.train()
        train_nnet(nnet, states_nnet, outputs, batch_size, num_itrs_per_vi_update, train_itr)

        nnet.eval()
        evaluate_cost_to_go(nnet, device, env, data["states"], data["output"])

        #pdb.set_trace()

        train_itr = train_itr + num_itrs_per_vi_update

    #writer.close()
    #pdb.set_trace()
    generate_plot(nnet, device, env, data["states"], data["output"])

def generate_plot(nnet: nn.Module(), device, env: Environment, states: List[State], outputs: np.array):
    nnet.eval()

    states_targ_nnet: np.ndarray = env.state_to_nnet_input(states)
    out_nnet = nnet(states_nnet_to_pytorch_input(states_targ_nnet, device).float()).cpu().data.numpy()
    out_nnet, _ = flatten(out_nnet)
    outputs, _ = flatten(outputs)

    out_nnet_array = np.array(out_nnet)
    outputs_array = np.array(outputs)

    zipped_arr = list(zip(out_nnet_array, outputs_array))

    sample_outputs = []
    sample_expected = []
    for a,b in random.sample(zipped_arr, k = 1):
        sample_outputs.append(a)
        sample_expected.append(b)

    h_new: np.ndarray = approx_admissable_conv(env, out_nnet_array, outputs_array, states, sample_outputs, sample_expected)

    before, after = plt.subplots()
    before.scatter(sample_expected, sample_outputs, c = '000000', linewidths = 0.1)
    #plt.plot([0,0],[30,30], c = 'g')
    before.axline([0,0],[30,30], linewidth =3, c = 'g')
    before.ylabel('NNet output')
    before.xlabel('Expected value')
    before.title("Output vs Expected")
    before.show()
    #before.savefig("preconversion.pdf")

    after.scatter(sample_expected, h_new, c = '000000', linewidths = 0.1)
    after.axline([0,0],[30,30], linewidth =3, c = 'g')
    after.ylabel('Converted output')
    after.xlabel('Expected value')
    after.title("Converted Output vs Expected")
    after.show()

def approx_admissable_conv(env: Environment, nnet_output: np.ndarray, optimal_output: np.ndarray, states: List[State], sample_outputs: np.ndarray, sample_expected: np.ndarray) -> np.ndarray:
    admissible_heur: np.ndarray =  np.zeros_like(nnet_output)
    h_new: np.ndarray = np.copy(admissible_heur)
    b = 3.0
    #create user defined set of cutoffs
    cut_offs: np.ndarray = np.array([])
    i = 0
    max_h = max(nnet_output)

    while i < max_h:
        cut_offs = np.append(cut_offs, [i])
        i = i+1
    cut_offs = np.append(cut_offs, max_h)

    o_c_max: np.ndarray = np.zeros_like(cut_offs)
    solved: np.ndarray = np.zeros_like(nnet_output) #0.0 for false, and 1.0 for true

    #while x is an element of the representative set and is_solved == false
    x = 0

    while x < len(nnet_output) or solved[x] == 0.0:

        #adjust inadmissible hueristic
        h_new = adjust_inadmissible_huerisitc(h_new, cut_offs, o_c_max, nnet_output, admissible_heur, b)

        #for x element X do -> h^a(x), is_solved(x) <- A*(x,h',h^a(x)+n)
        eta = 5.0
        for x in range(len(nnet_output)):
            print(x)
            admissible_heur[x], solved[x] = a_star_update(env, states[x], states, h_new, admissible_heur[x]+eta)

        if x % 100 == 0:
            print("Current progress: "+(x/len(solved))*100+"%")

        x += 1

    #adjust h' one last time before returning
    h_new = adjust_inadmissible_huerisitc(h_new, cut_offs, o_c_max, nnet_output, admissible_heur, b)

    return h_new

#@vectorize(["np.ndarray(np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, float)"], target ='cuda')
def adjust_inadmissible_huerisitc(h_new: np.ndarray, cut_offs: np.ndarray, o_c_max: np.ndarray, nnet_output: np.ndarray, admissible_heur: np.ndarray, b: float) -> np.ndarray:
    #get o_c_max
    o_c_max = get_oc_max(o_c_max, cut_offs, nnet_output, admissible_heur)

    for i in range(len(cut_offs)):
        o_c_max[i] = max(o_c_max[i] - b, 0)

    #set h'
    h_new = get_h_new(h_new, o_c_max, cut_offs, nnet_output)

    return h_new

@jit(nopython = True)
def get_oc_max(o_c_max: np.ndarray, cut_offs: np.ndarray, nnet_output: np.ndarray, admissible_heur: np.ndarray) -> np.ndarray:
    for i, c in enumerate(cut_offs):
        for j in range(len(nnet_output)):
            if nnet_output[j] <= c:
                o_c_max[i] = max(o_c_max[i], nnet_output[j]-admissible_heur[j])

    return o_c_max

@jit(nopython = True)
def get_h_new(h_new: np.ndarray, o_c_max: np.ndarray, cut_offs: np.ndarray, nnet_output: np.ndarray) -> np.ndarray:
    for i in range(len(h_new)):
        for j, c in enumerate(cut_offs):
            if nnet_output[i] <= c:
                h_new[i] = nnet_output[i] - o_c_max[j]
                break

    return h_new

def a_star_update(env: Environment, state: State, states: List[State], h_new: np.ndarray, max_step: float) -> Tuple[float, float]:
    #pdb.set_trace()
    update = AStarUpdate(env, state, states, h_new, max_step)
    update.step()
    max_cost_state: State = state

    while len(update.opened) > 0:
        #check if goal state
        if env.is_solved([update.curr_state])[0] == True:
            return update.h_new_dict[states[states.index(update.curr_state)]], 1.0
        #check if over max step
        elif update.closed[update.curr_state] + update.h_new_dict[states[states.index(update.curr_state)]] > max_step:
            return update.h_new_dict[states[states.index(update.curr_state)]], 0.0
        #check if greater than max cost state
        elif (update.closed[update.curr_state] + update.h_new_dict[states[states.index(update.curr_state)]]) > (update.closed[max_cost_state] + update.h_new_dict[states[states.index(max_cost_state)]]):
            max_cost_state = update.curr_state

        else:
            update.step()

    return max_cost_state, 0.0

#@jit(nopython = True)
#every call to this function takes around 1 or 2 full seconds
class AStarUpdate:
    def __init__(self, env: Environment, state: State, states: List[State], h_new: np.ndarray, max_step: float):
        self.env: Environment = env
        self.states: List[State] = states
        self.max_step: float = max_step
        self.solved: bool = False
        self.curr_state: State = state

        #pdb.set_trace()
        self.h_new_dict: Dict[State, float] = {self.states[i]: h_new[i] for i in range(len(states))}
        self.opened: Dict[State, float] = dict()
        self.closed: Dict[State, float] = dict()

        self.opened[state] = 0.0

    def step(self):
        try:
            self.curr_state: State = list(self.opened.keys())[0]
            for s in self.opened:
                if (self.opened[s] + self.h_new_dict[self.states[self.states.index(s)]]) < (self.opened[self.curr_state] + self.h_new_dict[self.states[self.states.index(self.curr_state)]]):
                    self.curr_state = s

            #children to opened
            for child in self.env.expand([self.curr_state])[0][0]:
                #pdb.set_trace()
                if child not in self.closed:
                    self.opened[child] = self.opened[self.curr_state] + 1.0
                else:
                    if (self.opened[self.curr_state] + 1.0) < self.closed[child]:
                        del self.closed[child]
                        self.opened[child] = self.opened[self.curr_state] + 1.0
            #add to closed and delete from opened
            self.closed[self.curr_state] = self.opened[self.curr_state]
            del self.opened[self.curr_state]
        except:
            extype, value, tb = sys.exc_info()
            traceback.print_exc()
            pdb.post_mortem(tb)

    '''
    children = env.expand(states)
    is_solved = env.is_solved(states)
    opened = np.empty(shape=(1,2), dtype = object)
    closed = np.empty(shape=(1,2), dtype = object)

    opened = np.append(opened, [[state, h_new[states.index(state)]]], axis = 0)
    opened = np.delete(opened, 0, 0)

    max_cost_state: float = 0.0
    try:
        while len(opened) > 0:
            self.curr_state = opened[np.argmin(opened[:,1])]

            #check if lowest cost state is goal state
            if is_solved[states.index(self.curr_state[0])] == True:
                return self.curr_state[1], 1.0

            #Check if over max step limit
            if self.curr_state[1] > max_step:
                return self.curr_state[1], 0.0

            #check if greater than max cost state:
            if self.curr_state[1] > max_cost_state:
                max_cost_state = self.curr_state[1]

            #add children of state to opened, alter closed if necessary
            #opened, closed = add_children(children, self.curr_state, states)
            index_of_LCS = states.index(self.curr_state[0])
            for c in range(len(children[0][index_of_LCS])):
            #check if child in closed:
                new_child = [children[0][index_of_LCS][c], self.curr_state[1]+children[1][index_of_LCS][c]]
                if np.isin(new_child[0], closed) == False:
                    opened = np.append(opened, [new_child], axis = 0)
                else:
                    #if lower cost state found add back to open, not sure if theres a built in function to do this
                    for i, x in enumerate(closed):
                        if type(x[0]) != type(None) and new_child[0] == x[0] and new_child[1] < x[1]:
                            opened = np.append(opened, [new_child], axis = 0)
                            closed = np.delete(closed, i, 0)

            #add to closed and delete from opened
            closed = np.append(closed, [self.curr_state], axis = 0)

            if len(closed) > 0 and np.isin(True, np.isin(closed, None)) == True: #really dumb work around, not sure how else to solve it tho
                closed = np.delete(closed, 0, 0)

            for i, x in enumerate(opened):
                if all(x == self.curr_state):
                    opened = np.delete(opened, i, 0)

        return max_cost_state, 0.0
    except:
        extype, value, tb = sys.exc_info()
        traceback.print_exc()
        pdb.post_mortem(tb)

@jit(nopython = True)
def add_children(children: Tuple[List[List[State]], List[List[float]]], self.curr_state: np.ndarray, states:np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
    index_of_LCS = states.index(self.curr_state[0])
    for c in range(len(children[0][index_of_LCS])):
        #check if child in closed:
        new_child = [children[0][index_of_LCS][c], self.curr_state[1]+children[1][index_of_LCS][c]]
        #pdb.set_trace()
        if np.isin(new_child[0], closed) == False:
            opened = np.append(opened, [new_child], axis = 0)
        else:
            #if lower cost state found add back to open, not sure if theres a built in function to do this
            #pdb.set_trace()
            for i, x in enumerate(closed):
                #pdb.set_trace()
                if type(x[0]) != type(None) and new_child[0] == x[0] and new_child[1] < x[1]:
                    opened = np.append(opened, [new_child], axis = 0)
                    closed = np.delete(closed, i, 0)

    return opened, closed
'''

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Sep  3 13:10:08 2019

@author: jdk450
"""

import os
import pandas  as pd
from sklearn import model_selection, linear_model, metrics
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model.logistic import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import classification_report, confusion_matrix
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
os.getcwd()
matplotlib.use("TkAgg")
sns.u
#os.chdir('Python') # this depends on where you placed the dataset

# load the dataset
data = pd.read_csv('SMSSpamCollection.csv')

# create  dataframes using texts and labels
texts = data.iloc[:, 1]
labels = data.iloc[:, 0]

#take a look
texts
labels

# split the dataset into training and validation datasets
X_train, X_test, y_train, y_test = train_test_split(texts,labels, test_size=0.20,random_state=0)#80:20
vectorizer = TfidfVectorizer()
X_train= vectorizer.fit_transform(X_train)
X_test= vectorizer.transform(X_test)
logReg = LogisticRegression()
logReg.fit(X_train, y_train)

#predict
y_predict = logReg.predict(X_test)

#get confusion matrix
cnf_matrix = metrics.confusion_matrix(y_test, y_predict)

#show confusion matrix
print(cnf_matrix)

#Calculate performance measures:
print("Accuracy:", metrics.accuracy_score(y_test, y_predict))

#if you don't include pos_label='sham' you get this error:
#ValueError: pos_label=1 is not a valid label: array(['ham', 'spam'], dtype='<U4')
print("Precision:", metrics.precision_score(y_test, y_predict, pos_label='spam'))
print("Recall:",metrics.recall_score(y_test, y_predict, pos_label = 'spam'))
print(metrics.f1_score(y_test, y_predict, pos_label='spam'))

#from sklearn.metrics import confusion_matrix,precision_recall_curve,auc,roc_auc_score,roc_curve ,recall_score,classification_report
print(metrics.classification_report(y_test, y_predict))
#Create decision tree classifier object.
clf = DecisionTreeClassifier()
# Train Decision Tree Classifier
clf = clf.fit(X_train,y_train)
#Predict the response for test dataset
y_pred = clf.predict(X_test)
cm = confusion_matrix(y_test,y_pred)
print("For decision Tree: ", cm )
print(classification_report(y_test,y_pred))
print("Decision Tree Accuracy Score: ",metrics.accuracy_score(y_test, y_pred))
#Display the heatmap
sns.heatmap(cm)
plt.show()


numbers_dict = {0:"",1:" one ",2:" two ",3:" three ",
                4:" Four ",5:" five ",6:" six ",7:" seven ",
                8:" eight ",9:" nine "}
sindex_dict = {1:" ten ",2:" twenty ",3:" thirteen ",
                4:" Forty ",5:" fifty ",6:" sixty ",7:" seventy ",
                8:" eighty ",9:" ninty "}
place_dict = {-3:"Hundred", -4:"thousand"}

number = int(input("Enter your number : "))
digits = []
while number>0:
    digits.append(number%10)
    number=int(number/10)

digits= digits[::-1]
print(numbers_dict[digits[-4]]+place_dict[-4]+
      numbers_dict[digits[-3]]+place_dict[-3]+
      sindex_dict[digits[-2]]+
      numbers_dict[digits[-1]])


from functools import wraps
try:
    from io import BytesIO
except ImportError:  # pragma:  no cover
    from cStringIO import StringIO as BytesIO

from flask import Blueprint, abort, g, request
from werkzeug.exceptions import InternalServerError
from celery import states

from . import celery
from .utils import url_for

text_types = (str, bytes)
try:
    text_types += (unicode,)
except NameError:
    # no unicode on Python 3
    pass

tasks_bp = Blueprint('tasks', __name__)

@celery.task
def run_flask_request(environ):
    from .wsgi_aux import app

    if '_wsgi.input' in environ:
        environ['wsgi.input'] = BytesIO(environ['_wsgi.input'])

    # Create a request context similar to that of the original request
    # so that the task can have access to flask.g, flask.request, etc.
    with app.request_context(environ):
        # Record the fact that we are running in the Celery worker now
        g.in_celery = True

        # Run the route function and record the response
        try:
            rv = app.full_dispatch_request()
        except:
            # If we are in debug mode we want to see the exception
            # Else, return a 500 error
            if app.debug:
                raise
            rv = app.make_response(InternalServerError())
        return (rv.get_data(), rv.status_code, rv.headers)

def async_task(f):
    """
    This decorator transforms a sync route to asynchronous by running it
    in a background thread.
    """
    @wraps(f)
    def wrapped(*args, **kwargs):
        # If we are already running the request on the celery side, then we
        # just call the wrapped function to allow the request to execute.
        if getattr(g, 'in_celery', False):
            return f(*args, **kwargs)

        # If we are on the Flask side, we need to launch the Celery task,
        # passing the request environment, which will be used to reconstruct
        # the request object. The request body has to be handled as a special
        # case, since WSGI requires it to be provided as a file-like object.
        environ = {k: v for k, v in request.environ.items()
                   if isinstance(v, text_types)}
        if 'wsgi.input' in request.environ:
            environ['_wsgi.input'] = request.get_data()
        t = run_flask_request.apply_async(args=(environ,))

        # Return a 202 response, with a link that the client can use to
        # obtain task status that is based on the Celery task id.
        if t.state == states.PENDING or t.state == states.RECEIVED or \
                t.state == states.STARTED:
            return '', 202, {'Location': url_for('tasks.get_status', id=t.id)}

        # If the task already finished, return its return value as response.
        # This would be the case when CELERY_ALWAYS_EAGER is set to True.
        return t.info
    return wrapped

@tasks_bp.route('/status/<id>', methods=['GET'])
def get_status(id):
    """
    Return status about an asynchronous task. If this request returns a 202
    status code, it means that task hasn't finished yet. Else, the response
    from the task is returned.
    """
    task = run_flask_request.AsyncResult(id)
    if task.state == states.PENDING:
        abort(404)
    if task.state == states.RECEIVED or task.state == states.STARTED:
        return '', 202, {'Location': url_for('tasks.get_status', id=id)}
    return task.info


#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# File   : _lazy_bind.py
# Author : Jiayuan Mao
# Email  : maojiayuan@gmail.com
# Date   : 01/19/2018
#
# This file is part of Jacinle.
# Distributed under terms of the MIT license.

import functools

from .rng import get_default_rng, _rng

__all__ = [
    # Utility functions
    'random_sample',        # Uniformly distributed floats over ``[0, 1)``.
    'bytes',                # Uniformly distributed random bytes.
    'random_integers',      # Uniformly distributed integers in a given range.
    'permutation',          # Randomly permute a sequence / generate a random sequence.
    'shuffle',              # Randomly permute a sequence in place.
    'seed',                 # Seed the random number generator.
    'choice',               # Random sample from 1-D array.

    # Compatibility functions
    'rand',                 # Uniformly distributed values.
    'randn',                # Normally distributed values.
    'randint',              # Uniformly distributed integers in a given range.

    # Univariate distributions
    'beta',                 # Beta distribution over ``[0, 1]``.
    'binomial',             # Binomial distribution.
    'chisquare',            # :math:`\\chi^2` distribution.
    'exponential',          # Exponential distribution.
    'f',                    # F (Fisher-Snedecor) distribution.
    'gamma',                # Gamma distribution.
    'geometric',            # Geometric distribution.
    'gumbel',               # Gumbel distribution.
    'hypergeometric',       # Hypergeometric distribution.
    'laplace',              # Laplace distribution.
    'logistic',             # Logistic distribution.
    'lognormal',            # Log-normal distribution.
    'logseries',            # Logarithmic series distribution.
    'negative_binomial',    # Negative binomial distribution.
    'noncentral_chisquare', # Non-central chi-square distribution.
    'noncentral_f',         # Non-central F distribution.
    'normal',               # Normal / Gaussian distribution.
    'pareto',               # Pareto distribution.
    'poisson',              # Poisson distribution.
    'power',                # Power distribution.
    'rayleigh',             # Rayleigh distribution.
    'triangular',           # Triangular distribution.
    'uniform',              # Uniform distribution.
    'vonmises',             # Von Mises circular distribution.
    'wald',                 # Wald (inverse Gaussian) distribution.
    'weibull',              # Weibull distribution.
    'zipf',                 # Zipf's distribution over ranked data.

    # Multivariate distributions
    'dirichlet',            # Multivariate generalization of Beta distribution.
    'multinomial',          # Multivariate generalization of the binomial distribution.
    'multivariate_normal',  # Multivariate generalization of the normal distribution.

    # Standard distributions
    'standard_cauchy',      # Standard Cauchy-Lorentz distribution.
    'standard_exponential', # Standard exponential distribution.
    'standard_gamma',       # Standard Gamma distribution.
    'standard_normal',      # Standard normal distribution.
    'standard_t',           # Standard Student's t-distribution.

    # Internal functions
    'get_state',            # Get tuple representing internal state of generator.
    'set_state',            # Set state of generator.

    # Customized utility functions
    'choice_list',          # Choose an element from a python list.
    'shuffle_list',         # Shuffle an python list.
    'shuffle_multi',        # Shuffle multiple arrays simultaneously.
]

def _lazy_bind(instance_func):
    func_name = instance_func.__name__

    @functools.wraps(instance_func)
    def wrapped(*args, **kwargs):
        func = getattr(get_default_rng(), func_name)
        return func(*args, **kwargs)

    return wrapped

# Utility functions
random_sample = _lazy_bind(_rng.random_sample)
bytes = _lazy_bind(_rng.bytes)
random_integers = _lazy_bind(_rng.random_integers)
permutation = _lazy_bind(_rng.permutation)
shuffle = _lazy_bind(_rng.shuffle)
seed = _lazy_bind(_rng.seed)
choice = _lazy_bind(_rng.choice)
# Compatibility functions
rand = _lazy_bind(_rng.rand)
randn = _lazy_bind(_rng.randn)
randint = _lazy_bind(_rng.randint)
# Univariate distributions
beta = _lazy_bind(_rng.beta)
binomial = _lazy_bind(_rng.binomial)
chisquare = _lazy_bind(_rng.chisquare)
exponential = _lazy_bind(_rng.exponential)
f = _lazy_bind(_rng.f)
gamma = _lazy_bind(_rng.gamma)
geometric = _lazy_bind(_rng.geometric)
gumbel = _lazy_bind(_rng.gumbel)
hypergeometric = _lazy_bind(_rng.hypergeometric)
laplace = _lazy_bind(_rng.laplace)
logistic = _lazy_bind(_rng.logistic)
lognormal = _lazy_bind(_rng.lognormal)
logseries = _lazy_bind(_rng.logseries)
negative_binomial = _lazy_bind(_rng.negative_binomial)
noncentral_chisquare = _lazy_bind(_rng.noncentral_chisquare)
noncentral_f = _lazy_bind(_rng.noncentral_f)
normal = _lazy_bind(_rng.normal)
pareto = _lazy_bind(_rng.pareto)
poisson = _lazy_bind(_rng.poisson)
power = _lazy_bind(_rng.power)
rayleigh = _lazy_bind(_rng.rayleigh)
triangular = _lazy_bind(_rng.triangular)
uniform = _lazy_bind(_rng.uniform)
vonmises = _lazy_bind(_rng.vonmises)
wald = _lazy_bind(_rng.wald)
weibull = _lazy_bind(_rng.weibull)
zipf = _lazy_bind(_rng.zipf)
# Multivariate distributions
dirichlet = _lazy_bind(_rng.dirichlet)
multinomial = _lazy_bind(_rng.multinomial)
multivariate_normal = _lazy_bind(_rng.multivariate_normal)
# Standard distributions
standard_cauchy = _lazy_bind(_rng.standard_cauchy)
standard_exponential = _lazy_bind(_rng.standard_exponential)
standard_gamma = _lazy_bind(_rng.standard_gamma)
standard_normal = _lazy_bind(_rng.standard_normal)
standard_t = _lazy_bind(_rng.standard_t)
# Internal functions
get_state = _lazy_bind(_rng.get_state)
set_state = _lazy_bind(_rng.set_state)
# Customized utility functions
choice_list = _lazy_bind(_rng.choice_list)
shuffle_list = _lazy_bind(_rng.shuffle_list)
shuffle_multi = _lazy_bind(_rng.shuffle_multi)

# Some aliases:
ranf = random = sample = random_sample
__all__.extend(['ranf', 'random', 'sample'])


"""
        QUBIT SPECTROSCOPY
This sequence involves sending a saturation pulse to the qubit, placing it in a mixed state,
and then measuring the state of the resonator across various qubit drive intermediate dfs.
In order to facilitate the qubit search, the qubit pulse duration and amplitude can be changed manually in the QUA
program directly without having to modify the configuration.

The data is post-processed to determine the qubit resonance frequency, which can then be used to adjust
the qubit intermediate frequency in the configuration under "qubit_IF".

Note that it can happen that the qubit is excited by the image sideband or LO leakage instead of the desired sideband.
This is why calibrating the qubit mixer is highly recommended.

This step can be repeated using the "x180" operation to adjust the pulse parameters (amplitude, duration, frequency)
before performing the next calibration steps.

Prerequisites:
    - Identification of the resonator's resonance frequency when coupled to the qubit in question (referred to as "resonator_spectroscopy_multiplexed").
    - Calibration of the IQ mixer connected to the qubit drive line (whether it's an external mixer or an Octave port).
    - Set the flux bias to the maximum frequency point, labeled as "max_frequency_point", in the configuration.
    - Configuration of the cw pulse amplitude (const_amp) and duration (const_len) to transition the qubit into a mixed state.
    - Specification of the expected qubits T1 in the configuration.

Before proceeding to the next node:
    - Update the qubit frequency, labeled as "qubit_IF_q", in the configuration.
"""

from qm.qua import *
from qm.QuantumMachinesManager import QuantumMachinesManager
from qm import SimulationConfig
import sys
import pathlib
QM_script_root = str(pathlib.Path(__file__).parent.parent.resolve())
sys.path.append(QM_script_root)
from configuration_with_octave import *
from qualang_tools.results import progress_counter, fetching_tool
from qualang_tools.plot import interrupt_on_close
from qualang_tools.loops import from_array
from macros import qua_declaration, multiplexed_readout
from scipy import signal
import matplotlib.pyplot as plt
import warnings

warnings.filterwarnings("ignore")

###################
#   Data Saving   #
###################
from datetime import datetime
import sys
save_data = False  # change to True when you want to save data

if save_data == True:
    save_dir = r"C:\Users\ASUS\SynologyDrive\09 Data\Fridge Data\Qubit\20231011_DR4_5QX\08 AC_Stark_shift"  # change save directory
    # save_name = f"Test"  # change data name
    save_progam_name = sys.argv[0].split('\\')[-1].split('.')[0]  # get the name of current running .py program
    save_time = str(datetime.now().strftime("%Y%m%d-%H%M%S"))
    # save_path = f"{save_dir}\{save_time}_{save_progam_name}_{save_name}"
    save_path = f"{save_dir}\{save_time}_{save_progam_name}"
    # save_path = f"{save_dir}\Test"

###################
# The QUA program #
###################
n_avg = 100000  # The number of averages
# Adjust the pulse duration and amplitude to drive the qubit into a mixed state

saturation_len1 = 2 * u.us  # In ns
saturation_amp1 = 0.001 # 0.005  # pre-factor to the value defined in the config - restricted to [-2; 2)
ro_amp_min = 0
ro_amp_max = 1.2
da = 0.05
amplitudes = np.arange(ro_amp_min, ro_amp_max + da / 2, da)  # The amplitude vector +da/2 to add a_max to the scan

# Qubit detuning sweep with respect to qubit_IF
span = 100 * u.MHz
df = 100 * u.kHz
dfs = np.arange(-span, span , df)

with program() as res_amp_vs_quibt_spec:
    I, I_st, Q, Q_st, n, n_st = qua_declaration(nb_of_qubits=1)
    df = declare(int)  # QUA variable for the readout frequency
    a = declare(fixed)  # QUA variable for sweeping the readout amplitude pre-factor

    # Adjust the flux line biases to check whether you are actually measuring the qubit
    set_dc_offset("q1_z", "single", max_frequency_point1)
    set_dc_offset("q2_z", "single", max_frequency_point2)
    with for_(n, 0, n < n_avg, n + 1):
        with for_(*from_array(df, dfs)):
            # Update the frequency of the two qubit elements
            update_frequency("q2_xy", df + qubit_IF_q2)
            with for_(*from_array(a, amplitudes)):  # QUA for_ loop for sweeping the readout amplitude
                # Play the saturation pulse to put the qubit in a mixed state - Can adjust the amplitude on the fly [-2; 2)
                # qubit 1
                play("stark" * amp(a), "q2_ro", duration=saturation_len1 * u.ns)
                wait(1500 * u.ns, "q2_xy")
                # play("x180", "q2_xy")
                # play("x180", "q2_xy")
                # play("x180", "q2_xy")
                # play("x180", "q2_xy")
                play("x180", "q2_xy")

                align("q2_xy", "q2_ro")
                wait(3000 * u.ns)
                measure(
                    "readout",
                    "rr2",
                    None,
                    dual_demod.full("cos", "out1", "sin", "out2", I[0]),
                    dual_demod.full("minus_sin", "out1", "cos", "out2", Q[0]),
                )
                # Save the 'I' & 'Q' quadratures for rr2 to their respective streams
                save(I[0], I_st[0])
                save(Q[0], Q_st[0])
                # Wait for the qubit to decay to the ground state
                wait(20 * u.us)

        save(n, n_st)

    with stream_processing():
        n_st.save("n")
        # resonator 1
        I_st[0].buffer(len(amplitudes)).buffer(len(dfs)).average().save("I1")
        Q_st[0].buffer(len(amplitudes)).buffer(len(dfs)).average().save("Q1")

#####################################
#  Open Communication with the QOP  #
#####################################
qmm = QuantumMachinesManager(host=qop_ip, cluster_name=cluster_name, octave=octave_config)

###########################
# Run or Simulate Program #
###########################

simulate = False

if simulate:
    # Simulates the QUA program for the specified duration
    simulation_config = SimulationConfig(duration=10_000)  # In clock cycles = 4ns
    job = qmm.simulate(config, res_amp_vs_quibt_spec, simulation_config)
    job.get_simulated_samples().con1.plot()
    plt.show()
else:
    # Open a quantum machine to execute the QUA program
    qm = qmm.open_qm(config)
    # Send the QUA program to the OPX, which compiles and executes it
    job = qm.execute(res_amp_vs_quibt_spec)
    # Prepare the figure for live plotting
    fig = plt.figure()
    interrupt_on_close(fig, job)
    # Tool to easily fetch results from the OPX (results_handle used in it)
    results = fetching_tool(job, ["n", "I1", "Q1"], mode="live")
    # Live plotting
    while results.is_processing():
        # Fetch results
        n, I1, Q1 = results.fetch_all()
        # Progress bar
        progress_counter(n, n_avg, start_time=results.start_time)
        # Data analysis
        S1 = u.demod2volts(I1 + 1j * Q1, readout_len)
        R1 = np.abs(S1)
        phase1 = np.angle(S1)

        # Plot
        plt.suptitle(f"AC Stark Shift \n Number of average = {n_avg}")
        plt.subplot(211)
        plt.cla()
        plt.title("Q2")
        plt.ylabel("qubit IF [MHz]")
        plt.pcolor((amplitudes * readout_amp_q1)*2, (dfs + qubit_IF_q1) / u.MHz, R1)

        plt.subplot(212)
        plt.cla()
        plt.xlabel("Readout power [V^2]")
        plt.ylabel("qubit IF [MHz]")
        plt.pcolor((amplitudes * readout_amp_q1)*2, (dfs + qubit_IF_q1) / u.MHz, signal.detrend(np.unwrap(phase1)))
        plt.pause(0.1)

    ###################
    #  Figure Saving  #
    ###################
    if save_data == True:
        figure = plt.gcf() # get current figure
        figure.set_size_inches(16, 9)
        plt.tight_layout()
        # Save Figure
        plt.savefig(f"{save_path}.png", dpi = 500)

    ###################
    #   .npz Saving   #
    ###################
    if save_data == True:
        # Change what you want to save
        np.savez(save_path,
                F1=qubit_LO_q1+qubit_IF_q1+dfs, R1=R1, P1=phase1, U1=signal.detrend(np.unwrap(phase1)), I1=I1, Q1=Q1,
                )

    # Close the quantum machines at the end in order to put all flux biases to 0 so that the fridge doesn't heat-up
    qm.close()

    plt.show()


"""Реализовать скрипт, в котором должна быть предусмотрена функция расчета
заработной платы сотрудника. В расчете необходимо использовать формулу:
(выработка в часах*ставка в час) + премия. Для выполнения расчета для
конкретных значений необходимо запускать скрипт с параметрами."""

# 1 Вводим нам необходимые нам параметры
def sal(hours, rate, bonus):
    try:
        res = (working_hours * working_rate) + (bonus * working_hours)
        print(f'Сотрудник {name_worker} заработал {res}')
    except TypeError:
        print(f'Error')
        exit()
name_worker = input('Имя сотрудника ')
working_hours = float(input('Выработка в часах '))
working_rate = int(input('Ставка '))
bonus = int(input('Премия '))
sal(working_hours, working_rate, bonus)

# 2 Задаем параметры в Edit Configurations

from sys import argv

name_script, name_worker, working_hours, working_rate, bonus = argv

try:
    name_worker = str(name_worker)
    working_hours = int(working_hours)
    working_rate = int(working_rate)
    bonus = int(bonus)
    res = (working_hours * working_rate) + (bonus * working_hours)
    print(f'Сотрудник {name_worker} заработал {res}')
except ValueError:
    print('Not a number')