# -*- coding: utf-8 -*-
"""
Created on Thu Dec 27 10:36:00 2018

@author: ogianan
"""

import os
import pandas as pd
from string import punctuation

from app import app
from flask import render_template, request, redirect, url_for, send_file, send_from_directory
from flask_wtf import FlaskForm
from wtforms import RadioField
from flask_wtf.file import FileField
from werkzeug.utils import secure_filename

class UploadForm(FlaskForm):
    file = FileField()

# Upload folder where xls files will be saved and retrieved by auto-downloader
UPLOAD_FOLDER = '/home/sysadmin/ProServConvert/files/'
ALLOWED_EXTENSIONS = set(['txt', 'csv'])
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def load_skus(file, subitems, subcons):
    # Create columns order list
    cols = ['Product ID', 'Quantity', 'Unit Price', 'Unit Cost']
    # Read csv file
    soitems = pd.read_csv(file)
    soitems['Product ID'] = soitems['Product ID'][:].str.strip(punctuation).str.lstrip()
    soitems_sub = soitems[soitems['Product ID'].isin(subitems)]
    soitems_sub = soitems_sub.groupby(['Product ID', 'Unit Price', 'Unit Cost'])['Quantity'].sum().reset_index(name = 'Quantity')
    return soitems, soitems_sub[cols]

def create_proserv(file):
    # Create tables template
    # Create sub-items series
    subitems = pd.Series(['PRO-UC-L1', 'PRO-NET-L1', 'PRO-SVR-L1', 'PRO-PMO-L1', 'Risk', 'PRO-SMARTHANDS-L1','PRO-SMARTHANDS'])
    # Create SUBCON variations series
    subcons = pd.Series(['PRO-SMARTHANDS-L1', 'PRO-SMARTHANDS'])
    # Create output template
    template = { 'Product ID' : ['PKG-PRO-PROFSVCS', 'PKG-PRO-PROFSVCS', 'PKG-PRO-PROFSVCS', 'PKG-PRO-PROFSVCS'] , 'Quantity' : [1, 1, 1, 1] , 'Price' : [0, 0, 0, 0] , 'Cost' : [0, 0, 0, 0] , 'Customer Description' : ['Professional Services: 50%', 'Professional Services: 20%', 'Professional Services: 20%', 'Professional Services: 10%'] }
    out_table = pd.DataFrame(template, columns=['Product ID', 'Quantity', 'Unit Price', 'Unit Cost', 'Customer Description'])
    # Create a Pandas Excel writer using XlsxWriter as the engine.
    #new_file = app.config['UPLOAD_FOLDER'] + 'proserv_' + file

    new_file = app.config['UPLOAD_FOLDER'] + 'proserv_converted.xlsx'

    writer = pd.ExcelWriter(new_file, engine='xlsxwriter')

    # Load soitems file
    df, agg_df = load_skus(file, subitems, subcons)

    # Get total proserv price
    t_proserv_price = df[df['Product Class'].str.contains('Bundle')]['Unit Price'].sum()
    # Get total proserv cost
    t_proserv_cost = df[df['Product Class'].str.contains('Bundle')]['Unit Cost'].sum()

    # Split smarthands from sub_agg
    subcons_df = agg_df.loc[agg_df['Product ID'].isin(subcons)]
    subitems_df = agg_df.loc[~agg_df['Product ID'].isin(subcons)]

    # Get total subcons cost
    t_subcons_cost = (subcons_df['Quantity'] * subcons_df['Unit Cost']).sum()

    # Get adjusted proserv cost
    t_proserv_cost = t_proserv_cost - t_subcons_cost

    # Assign values to out_table
    out_table.at[0, 'Unit Price'] = .5 * t_proserv_price
    out_table.at[1, 'Unit Price'] = .2 * t_proserv_price
    out_table.at[2, 'Unit Price'] = .2 * t_proserv_price
    out_table.at[3, 'Unit Price'] = .1 * t_proserv_price
    out_table.at[0, 'Unit Cost'] =  .5 * t_proserv_cost
    out_table.at[1, 'Unit Cost'] = .2 * t_proserv_cost
    out_table.at[2, 'Unit Cost'] = .2 * t_proserv_cost
    out_table.at[3, 'Unit Cost'] = .1 * t_proserv_cost

    # Zero out subcon pricing
    #subcons_df = subcons_df.replace([subcons_df['Unit Price']], 0)
    # the above code quits when df has only one row
    # Disable warnings
    pd.options.mode.chained_assignment = None
    subcons_df['Unit Price'] = 0
    # the above code outputs a warning
    # rename PRO-SMARTHANDS* to SUBCON
    subcons_df['Product ID'] = "SUBCON"
    # the above code outputs a warning

    # Output final table
    final = pd.concat([out_table.loc[:0], subitems_df, out_table.loc[1:], subcons_df]).reset_index(drop=True)
    final_cols = ['Product ID', 'Quantity', 'Unit Price', 'Unit Cost', 'Customer Description']
    final = final[final_cols]

    # Number of rows of original table
    df_rows = df.shape[0]
    final_start_row = df_rows + 3

    # Writing the dataframes into the worksheet
    df.to_excel(writer, sheet_name = 'Sheet1', index = False)
    final.to_excel(writer, sheet_name = 'Sheet1', startrow=final_start_row, index = False)

    # Conditional formatting
    # Get the xlsxwriter workbook and worksheet objects.
    workbook  = writer.book
    worksheet = writer.sheets['Sheet1']
    # Add a format. Light red fill with dark red text.
    red_format = workbook.add_format({'bg_color': '#FFC7CE', 'font_color': '#9C0006'})

    # Apply format to the cell range.
    marker = df['Product ID'].isin(subitems) | df['Product Class'].str.contains('Bundle')
    # Get index of first TRUE value
    f_in = 1 + marker[marker==True].index.tolist()[0]

    # Get index of last TRUE value
    l_in = 1 +  marker[marker==True].index.tolist()[-1]
    #str('L%d:L%d' % (f_in, l_in))

    for row in range (f_in, l_in+1):
    #    print(row)
        worksheet.set_row(row, 15, red_format)

    #print(marker)
    writer.save()
    return

@app.route('/', methods=['GET', 'POST'])
@app.route('/index')
def upload_file():
    form = UploadForm(csrf_enabled=False)
    if form.validate_on_submit():
	    filename = secure_filename(form.file.data.filename)
	    form.file.data.save(filename)
	    create_proserv(filename)
	    return send_from_directory(app.config['UPLOAD_FOLDER'], 'proserv_converted.xlsx', as_attachment=True)
	    return redirect(url_for('upload_file', filename=filename))
    return render_template('index.html', title='ProServ Conversion Tool', form=form)


import numpy as np
from PIL import Image
#from pathlib import Path
from random import randint
#import argparse
from tqdm import trange
import itertools
from scipy.spatial.distance import cdist

class DatasetEdit:

    #slice_size
    #directory
    #filename # = outname
    #best_hamming
    #M
    #N
    #P: n of permutations
    #pos_auto_x
    #pos_auto_y
    #used_pos

    def __init__(self, img_dim, P, slice_size=3):
        self.slice_size=slice_size
        #self.directory=directory

        #img = Image.open(directory+'pic_001.jpg') # access a picture to retrieve size
        self.img_input_nn_dim = img_dim

        if img_dim%slice_size != 0:
            self.img_dim = 224
        else:
            self.img_dim = img_dim

        self.M = self.img_dim//slice_size
        self.N = self.img_dim//slice_size

        self.mappa3=[[0,0],[1,0],[2,0],[0,1],[1,1],[2,1],[0,2],[1,2],[2,2]]
        self.mappa2=[[0,0],[1,0],[0,1],[1,1]]
        self.mappa4=[[0,0],[1,0],[2,0],[3,0],[0,1],[1,1],[2,1],[3,1],[0,2],[1,2],[2,2],[3,2],[0,3],[1,3],[2,3],[3,3]]

        self.pos_auto_x = np.arange(0,self.img_dim,self.M)
        self.pos_auto_y = np.arange(0,self.img_dim,self.N)#[y for y in range(0,img_dim,N)]

        self.diff_pos = slice_size**2 # slice_size = 3 so different positions are 9 (3^2) / 2 so 4 / 4 so 16
        self.P = P

        if slice_size == 4:
            self.four_by_four = True
        else:
            self.four_by_four = False

        self.permutations()

    #used to retrive where to put a sliced image in mosaic.
    def matrixIndeces(self,index,slice_size):
        if slice_size == 2: #2x2
            ind_y = self.mappa2[index][1] # row
            ind_x = self.mappa2[index][0] # column
        elif slice_size == 3: #3x3
            ind_y = self.mappa3[index][1] # row
            ind_x = self.mappa3[index][0] # column
        elif slice_size == 4: #4x4
            ind_y = self.mappa4[index][1] # row
            ind_x = self.mappa4[index][0] # column
        else:
            # default case
            pass

        return ind_x,ind_y

    def permutations(self):
        outname = 'permutations/permutations_hamming_%d'%(self.diff_pos)
        selection = "max"
        # with [0,1,2,3,4,5,6,7,8] - original image
        if self.four_by_four == True:
            P_hat_c1 = np.array(list(itertools.permutations(list(range(8)), 8)))
            P_hat_c2 = np.array(list(itertools.permutations(list(range(8)), 8))) + 8
            P_hat_full = np.concatenate((P_hat_c1, P_hat_c2), axis=1)
        else:
            P_hat_full = np.array(list(itertools.permutations(list(range(self.diff_pos)), self.diff_pos)))
        P_hat = np.delete(P_hat_full,0,0)

        n = P_hat.shape[0]

        for i in range(self.P):
            if i==0:
                j = np.random.randint(n)
                P = np.array(P_hat[j]).reshape([1,-1])
            else:
                P = np.concatenate([P,P_hat[j].reshape([1,-1])],axis=0)

            P_hat = np.delete(P_hat,j,axis=0)
            D = cdist(P,P_hat, metric='hamming').mean(axis=0).flatten()

            if selection=='max':
                j = D.argmax()
            else:
                m = int(D.shape[0]/2)
                S = D.argsort()
                j = S[np.random.randint(m-10,m+10)]

            #if i%30==0:
                #np.save(outname,P)
        #np.save(outname,P)
        self.filename = outname
        self.best_hamming = P
        #print('file created --> '+outname)

    def readHamming(self):

        best_hamming = np.load(self.filename+'.npy')

        return best_hamming #access as matrix

    def edit(self, img):
        """ slice the image in parts
            Args:
                img (Image): opened image file
        """
        if self.img_dim!=self.img_input_nn_dim:
            img = self.resize(img, self.img_dim, self.img_dim)

        # Open images and store them in a list
        #for path in Path(self.directory).rglob('*.jpg'): # errpr: WindowsPath object is not iterable
        #images = [Image.open(x) for x in path]

        #if img.size[0]%self.slice_size!=0 or img.size[0]%self.slice_size!=0:

        #slice in parts and create the new image
        #for img in images:
        k=0 # index for hamming_array
        perm = randint(0, self.P-1) # 1 random best hamming
        hamming_array=self.best_hamming[perm]
        # create a new image with the appropriate height and width
        new_img = Image.new('RGB', (img.size[1], img.size[0]))
        for y in range(0,img.size[0],self.M):
            for x in range(0,img.size[1],self.N):
                tile = img.crop((x,y,x+self.M,y+self.N))

                #place tile
                if k>=self.diff_pos:
                    break
                MatInd = self.matrixIndeces(hamming_array[k],self.slice_size)
                i=int(MatInd[0])
                j=int(MatInd[1])
                new_img.paste(tile, (self.pos_auto_x[i],self.pos_auto_y[j]))
                k=k+1

        # Save the image
        #img_name=path.name.split('.')
        #full_img_name=img_name+"_sliced(%d)"+".jpg"%(i+1) # i is the label (1 to 30)
        #new_img.save(full_img_name)
        if self.img_dim!=self.img_input_nn_dim:
            new_img = self.resize(new_img, self.img_input_nn_dim, self.img_input_nn_dim)
        return new_img, perm+1

    def odd_one_out(self, img1, img2):
        """ switch a random tile in puzzle of the first image
            with a random crop of the second image
            Args:
                img1 (Image): opened image file
                img2 (Image): opened image file
            Out:
                img1 (Image): image with odd tile
                index_odd (int): position of the odd tile
        """

        img1, _ = self.edit(img1)

        # random crop in img2
        x = randint(0, img2.size[0] - self.M)
        y = randint(0, img2.size[1] - self.N)
        odd_tile = img2.crop((x,y,x+self.M,y+self.N))

        # random position of the odd tile in img1
        index_odd = randint(0,8)

        # paste the odd tile in img1
        MatInd = self.matrixIndeces(index_odd,self.slice_size)
        i = int(MatInd[0])
        j = int(MatInd[1])
        img1.paste(odd_tile, (self.pos_auto_x[i],self.pos_auto_y[j]))

        return img1, index_odd+1

    def randomRotation(self, img):
        """ rotate the image of a random multiple of 90 degree
            Args:
                img (Image): opened image file
            Out:
                img (Image): rotated image
                angle (int): angle of rotation
        """

        angle = randint(0, 2)

        if angle == 0:
            new_img = img.transpose(Image.ROTATE_90)
        elif angle == 1:
            new_img = img.transpose(Image.ROTATE_180)
        elif angle == 2:
            new_img = img.transpose(Image.ROTATE_270)

        return new_img, angle+1

    def resize(self, im, size_width, size_height) -> Image:
        new_dim = (size_width, size_height)
        return im.resize(new_dim)


import turtle
import pandas

screen = turtle.Screen()
screen.title("U.S States Game")
image = "blank_states_img.gif"
screen.addshape(image)
turtle.shape(image)

data = pandas.read_csv("50_states.csv")
all__states = data.state.to_list()
guessed_states = []

while len(guessed_states) <= 50:
    answer_state = screen.textinput(title= f"{len(guessed_states)}/50 States Correct",
                                    prompt="What's another state names?:").title()
    if answer_state == "Exit":
        missing_states = [state for state in all__states if state not in guessed_states]
        new_data = pandas.DataFrame(missing_states)
        new_data.to_csv("states_to_learn.csv")
        break
    if answer_state in all__states:
        guessed_states.append(answer_state)
        t = turtle.Turtle()
        t.hideturtle()
        t.penup()
        state_data = data[data.state == answer_state]
        t.goto(int(state_data.x), int(state_data.y))
        t.write(answer_state)

# is_guess_true = True
# filling_list = []
# score = 0
# while is_guess_true:
#     if answer_state in list_of_states and answer_state not in filling_list:
#         single_state_info = data[data.state == f"{answer_state}"]
#         x_cor = single_state_info.x
#         y_cor = single_state_info.y
#         turtle.goto(x_cor, y_cor)
#         filling_list.append(answer_state)
#
#     elif len(filling_list) == 50:
#         print("Congrats! you have correctly guessed all the states.")
#         is_guess_true = False

screen.exitonclick()


"""Geometry visualization module."""
from __future__ import annotations
from functools import singledispatch

import numpy as np
import matplotlib.pyplot as plt

from optim.utils.utils import import_guard

if import_guard():
    from matplotlib.figure import Figure  # noqa: TC002, is guarded
    from optim.geometry import Cube, Geometry  # noqa: TC001, is guarded

@singledispatch
def visualize_geometry(geom: Geometry):
    """Visualize dispatch function for geometries.

    Args:
        geom: Geometry.

    Raises:
        TypeError: Default dispatcher means an unsupported geometry has been used.
    """
    raise TypeError(f"Geometry type {type(geom)} not supported")

@visualize_geometry.register
def _(geom: Cube) -> Figure:
    """Visualize the cube geometry.

    Args:
        geom: Cube geometry.

    Returns:
        The figure.
    """
    fig = plt.figure()
    fig.suptitle("Contact point optimization")
    ax = []
    ax.append(fig.add_subplot(111, projection="3d"))
    dst = np.linalg.norm(geom.plane_offsets[0, 1] - geom.plane_offsets[0, 2])
    ax[0].set_title("cube")
    ax[0].set_box_aspect(aspect=(1, 1, 1))
    ax[0].set_xlim(geom.com[0] + dst * np.array([-2, 2]))
    ax[0].set_ylim(geom.com[1] + dst * np.array([-2, 2]))
    ax[0].set_zlim(geom.com[2] + dst * np.array([-2, 2]))
    ax[0].set_xlabel("x")
    ax[0].set_ylabel("y")
    ax[0].set_zlabel("z")

    for k, offsets in enumerate(geom.plane_offsets):
        for i in range(1, len(offsets)):
            j = 3 if i < 3 else 1
            p0 = offsets[i] + (offsets[j] - offsets[0])
            p1 = offsets[i] + (offsets[j + 1] - offsets[0])
            ax[0].plot3D(*zip(p0, p1), color="k")
        ax[0].scatter(*offsets[0], color=("r", "g", "b", "c", "m", "y")[k])
    # Plot table surface
    xsupport = np.linspace(*(geom.com[0] + dst * np.array([-1, 1])), 2)
    ysupport = np.linspace(*(geom.com[1] + dst * np.array([-1, 1])), 2)
    x, y = np.meshgrid(xsupport, ysupport)
    z = np.ones_like(x) * 0.4
    ax[0].plot_surface(x, y, z, alpha=0.4)
    return fig


#!/usr/bin/python
import unittest
import time
import logging
import os
import threading

import common
import utils_env
import utils_params
import utils_misc

class FakeVm(object):

    def __init__(self, vm_name, params):
        self.name = vm_name
        self.params = params
        self.vm_type = self.params.get('vm_type')
        self.driver_type = self.params.get('driver_type')
        self.instance = ("%s-%s" % (
            time.strftime("%Y%m%d-%H%M%S"),
            utils_misc.generate_random_string(16)))

    def get_params(self):
        return self.params

    def is_alive(self):
        logging.info("Fake VM %s (instance %s)", self.name, self.instance)

class FakeSyncListenServer(object):

    def __init__(self, address='', port=123, tmpdir=None):
        self.instance = ("%s-%s" % (
            time.strftime("%Y%m%d-%H%M%S"),
            utils_misc.generate_random_string(16)))
        self.port = port

    def close(self):
        logging.info("Closing sync server (instance %s)", self.instance)

class TestEnv(unittest.TestCase):

    def setUp(self):
        self.envfilename = "/dev/shm/EnvUnittest" + self.id()

    def tearDown(self):
        if os.path.exists(self.envfilename):
            os.unlink(self.envfilename)

    def test_save(self):
        """
        1) Verify that calling env.save() with no filename where env doesn't
           specify a filename will throw an EnvSaveError.
        2) Register a VM in environment, save env to a file, recover env from
           that file, get the vm and verify that the instance attribute of the
           2 objects is the same.
        3) Register a SyncListenServer and don't save env. Restore env from
           file and try to get the syncserver, verify it doesn't work.
        4) Now save env to a file, restore env from file and verify that
           the syncserver can be found there, and that the sync server
           instance attribute is equal to the initial sync server instance.
        """

        env = utils_env.Env()
        self.assertRaises(utils_env.EnvSaveError, env.save, {})

        params = utils_params.Params({"main_vm": 'rhel7-migration'})
        vm1 = FakeVm(params['main_vm'], params)
        vm1.is_alive()
        env.register_vm(params['main_vm'], vm1)
        env.save(filename=self.envfilename)
        env2 = utils_env.Env(filename=self.envfilename)
        vm2 = env2.get_vm(params['main_vm'])
        vm2.is_alive()
        assert vm1.instance == vm2.instance

        sync1 = FakeSyncListenServer(port=222)
        env.register_syncserver(222, sync1)
        env3 = utils_env.Env(filename=self.envfilename)
        syncnone = env3.get_syncserver(222)
        assert syncnone is None

        env.save(filename=self.envfilename)
        env4 = utils_env.Env(filename=self.envfilename)
        sync2 = env4.get_syncserver(222)
        assert sync2.instance == sync1.instance

    def test_register_vm(self):
        """
        1) Create an env object.
        2) Create a VM and register it from env.
        3) Get the vm back from the env.
        4) Verify that the 2 objects are the same.
        """
        env = utils_env.Env(filename=self.envfilename)
        params = utils_params.Params({"main_vm": 'rhel7-migration'})
        vm1 = FakeVm(params['main_vm'], params)
        vm1.is_alive()
        env.register_vm(params['main_vm'], vm1)
        vm2 = env.get_vm(params['main_vm'])
        vm2.is_alive()
        assert vm1 == vm2

    def test_unregister_vm(self):
        """
        1) Create an env object.
        2) Register 2 vms to the env.
        3) Verify both vms are in the env.
        4) Remove one of those vms.
        5) Verify that the removed vm is no longer in env.
        """
        env = utils_env.Env(filename=self.envfilename)
        params = utils_params.Params({"main_vm": 'rhel7-migration'})
        vm1 = FakeVm(params['main_vm'], params)
        vm1.is_alive()
        vm2 = FakeVm('vm2', params)
        vm2.is_alive()
        env.register_vm(params['main_vm'], vm1)
        env.register_vm('vm2', vm2)
        assert vm1 in env.get_all_vms()
        assert vm2 in env.get_all_vms()
        env.unregister_vm('vm2')
        assert vm1 in env.get_all_vms()
        assert vm2 not in env.get_all_vms()

    def test_get_all_vms(self):
        """
        1) Create an env object.
        2) Create 2 vms and register them in the env.
        3) Create a SyncListenServer and register it in the env.
        4) Verify that the 2 vms are in the output of get_all_vms.
        5) Verify that the sync server is not in the output of get_all_vms.
        """
        env = utils_env.Env(filename=self.envfilename)
        params = utils_params.Params({"main_vm": 'rhel7-migration'})
        vm1 = FakeVm(params['main_vm'], params)
        vm1.is_alive()
        vm2 = FakeVm('vm2', params)
        vm2.is_alive()
        env.register_vm(params['main_vm'], vm1)
        env.register_vm('vm2', vm2)
        sync1 = FakeSyncListenServer(port=333)
        env.register_syncserver(333, sync1)
        assert vm1 in env.get_all_vms()
        assert vm2 in env.get_all_vms()
        assert sync1 not in env.get_all_vms()

    def test_register_syncserver(self):
        """
        1) Create an env file.
        2) Create a SyncListenServer object and register it in the env.
        3) Get that SyncListenServer with get_syncserver.
        4) Verify that both objects are the same.
        """
        env = utils_env.Env(filename=self.envfilename)
        sync1 = FakeSyncListenServer(port=333)
        env.register_syncserver(333, sync1)
        sync2 = env.get_syncserver(333)
        assert sync1 == sync2

    def test_unregister_syncserver(self):
        """
        Unregister a sync server.

        1) Create an env file.
        2) Create and register 2 SyncListenServers in the env.
        3) Get one of the SyncListenServers in the env.
        4) Unregister one of the SyncListenServers.
        5) Verify that the SyncListenServer unregistered can't be retrieved
           anymore with ``get_syncserver()``.

        """
        env = utils_env.Env(filename=self.envfilename)
        sync1 = FakeSyncListenServer(port=333)
        env.register_syncserver(333, sync1)
        sync2 = FakeSyncListenServer(port=444)
        env.register_syncserver(444, sync2)
        sync3 = env.get_syncserver(333)
        assert sync1 == sync3
        env.unregister_syncserver(444)
        sync4 = env.get_syncserver(444)
        assert sync4 is None

    def test_locking(self):
        """
        1) Create an env file.
        2) Create a thread that creates a dict as one of env's elements, and
           keeps updating it, using the env save_lock attribute.
        3) Try to save the environment.
        """
        termination_event = threading.Event()
        env = utils_env.Env(filename=self.envfilename)

        def update_env(env):
            @utils_env.lock_safe
            def _update_env(env, key, value):
                env["changing_dict"][key] = value

            if "changing_dict" not in env:
                env["changing_dict"] = {}
            while True:
                key = "%s" % utils_misc.generate_random_string(length=10)
                value = "%s" % utils_misc.generate_random_string(length=10)
                _update_env(env, key, value)
                if termination_event.isSet():
                    break

        changing_thread = threading.Thread(target=update_env,
                                           args=(env,))
        changing_thread.start()
        time.sleep(0.3)
        try:
            env.save()
        finally:
            termination_event.set()

if __name__ == '__main__':
    unittest.main()


import typer
from .server.serv import main as server
from .converter import stats_to_df

app = typer.Typer()

@app.command()
def main(file: str):
    """Takes path to .prof file and runs the server"""
    df = stats_to_df(file)
    server(df)

if __name__ == "__main__":
    app()


#!/usr/bin/env python
"""
# Author: Xiong Lei
# Created Time : Sun 17 Nov 2019 03:37:47 PM CST

# File Name: setup.py
# Description:

"""
import pathlib
from setuptools import setup, find_packages

with open('requirements.txt') as f:
    requirements = f.read().splitlines()

HERE = pathlib.Path(__file__).parent
README = (HERE / "README.md").read_text()

import codecs
import os.path

def read(rel_path):
    here = os.path.abspath(os.path.dirname(__file__))
    with codecs.open(os.path.join(here, rel_path), 'r') as fp:
        return fp.read()

def get_version(rel_path):
    for line in read(rel_path).splitlines():
        if line.startswith('__version__'):
            delim = '"' if '"' in line else "'"
            return line.split(delim)[1]
    else:
        raise RuntimeError("Unable to find version string.")

setup(name='scalex',
      version=get_version(HERE / "scalex/__init__.py"),
      packages=find_packages(),
      description='Online single-cell data integration through projecting heterogeneous datasets into a common cell-embedding space',
      long_description=README,

      author='Lei Xiong',
      author_email='jsxlei@gmail.com',
      url='https://github.com/jsxlei/scalex',
      scripts=['SCALEX.py'],
      install_requires=requirements,
      python_requires='>3.6.0',
      license='MIT',

      classifiers=[
          'Development Status :: 4 - Beta',
          'Intended Audience :: Science/Research',
          'License :: OSI Approved :: MIT License',
          'Programming Language :: Python :: 3.7',
          'Operating System :: MacOS :: MacOS X',
          'Operating System :: Microsoft :: Windows',
          'Operating System :: POSIX :: Linux',
          'Topic :: Scientific/Engineering :: Bio-Informatics',
     ],
     )


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import importlib
import logging
import os
import sys

from mattapi.api.os_helpers import OSHelper
from mattapi.api.settings import Settings
from mattapi.util.arg_parser import get_core_args
from mattapi.util.path_manager import PathManager

logger = logging.getLogger(__name__)
core_args = get_core_args()

def check_target(target: str = None):
    """Checks if provided target exists."""
    # Currently not in use, but maintaining for possible future use.

    if target is None:
        logger.warning('No target provided.')

    local_target_path = os.path.join(PathManager.get_module_dir(), 'targets')
    package_target_path = os.path.join(PathManager.get_module_dir(), 'mattapi', 'targets')

    if os.path.exists(os.path.join(local_target_path, target)):
        return True
    if os.path.exists(os.path.join(package_target_path, target)):
        return True

    target_list = []
    local_path_exists = os.path.exists(local_target_path)
    package_path_exists = os.path.exists(package_target_path)

    if local_path_exists:
        target_list += scan_dir(local_target_path)

    if package_path_exists:
        target_list += scan_dir(package_target_path)

    if not local_path_exists and not package_path_exists:
        path_warning(local_target_path)
    else:
        logger.critical('Target %s doesn\'t exist.')
        logger.critical('Did you mean to choose one of these instead?')
        for target in target_list:
            logger.critical('\t%s' % target)

    return False

def scan_dir(path):
    target_list = [f.path for f in os.scandir(path) if f.is_dir()]
    target_names = []

    for idx, target in enumerate(target_list):
        if 'pycache' not in target:
            target_names.append(os.path.basename(os.path.normpath(target)))

    return target_names

def get_target(target_name: str):
    logger.info('Desired target: %s' % target_name)
    logger.info('Desired module: targets.%s.main' %  target_name)
    try:
        logger.debug('Attempt to load target from default package root.')
        target = import_package_by_name(target_name, os.path.join(Settings.PACKAGE_ROOT, 'mattapi'))
        return target
    except Exception as e:
        logger.error(e)
        logger.debug('Attempt to load target from code root.')
        target = import_package_by_name(target_name, Settings.CODE_ROOT)
        return target

def import_package_by_name(target_name: str, path: str):
    sys.path.append(path)
    logger.debug('Looking for %s in path %s' % (target_name, path))
    try:
        my_module = importlib.import_module('targets.%s.main' % target_name)
        logger.info('Successful import!')
        try:
            target_plugin = my_module.Target()
            logger.info('Found target named %s' % target_plugin.target_name)
            return target_plugin
        except NameError:
            raise Exception('Target %s not found in path %s' % (target_name, path))
    except ImportError as e:
        if e.name.__contains__('Xlib') and not OSHelper.is_linux():
            pass
        else:
            logger.error('Problems importing module:\n%s' % e)
            raise ImportError

def collect_tests():
    """Collects tests based on include/exclude criteria and selected target."""
    target = core_args.target
    test_list = []

    include = core_args.test
    exclude = core_args.exclude
    if os.path.isfile(include):
        with open(include, 'r') as f:
            for line in f:
                test_list.append(line.rstrip('\n'))
        f.close()
    else:
        tests_dir = os.path.join(PathManager.get_tests_dir(), target)
        if not os.path.exists(tests_dir):
            path_warning(tests_dir)
            return test_list

        logger.debug('Path %s found. Checking content ...', tests_dir)
        for dir_path, sub_dirs, all_files in PathManager.sorted_walk(tests_dir):
            for current_file in all_files:
                directory = '%s%s%s' % (os.sep, core_args.directory, os.sep)
                include_params = [include]
                exclude_params = [exclude]
                if ',' in include:
                    include_params = include.split(',')
                if ',' in exclude:
                    exclude_params = exclude.split(',')
                current_full_path = os.path.join(dir_path, current_file)
                if current_file.endswith('.py') and not current_file.startswith('__'):
                    if include is '' and exclude is '' and directory is '':
                        if not current_full_path in test_list:
                            test_list.append(current_full_path)
                    else:
                        if core_args.directory is '' or directory in current_full_path:
                            for include_param in include_params:
                                if include_param is '' or include_param in current_full_path:
                                    for exclude_param in exclude_params:
                                        if exclude_param is '':
                                            if not current_full_path in test_list:
                                                test_list.append(current_full_path)
                                        else:
                                            if exclude_param not in current_full_path:
                                                if not current_full_path in test_list:
                                                    test_list.append(current_full_path)
        if len(test_list) == 0:
            logger.error('\'%s\' does not contain tests based on your search criteria. Exiting program.' % tests_dir)
        else:
            logger.debug('List of all tests found: [%s]' % ', '.join(map(str, test_list)))

    return test_list

def path_warning(path):
    logger.error('Path not found: %s' % path)
    logger.critical('This can happen when Iris can\'t find your code root.')
    logger.critical('Try setting these environment variables:')
    if OSHelper.is_windows():
        logger.critical('\tsetx IRIS_CODE_ROOT %CD%')
        logger.critical('\tsetx PYTHONPATH %CD%')
        logger.critical('\nYou must restart your terminal for this to take effect.\n')
    else:
        logger.critical('\texport IRIS_CODE_ROOT=$PWD')
        logger.critical('\texport PYTHONPATH=$PWD')


import os
from typing import List

import tensorflow as tf
from tensorflow.python.data import Dataset
from tensorflow.python.data.experimental import group_by_window
from tensorflow.python.keras import Model
from tensorflow.python.keras.callbacks import Callback
from tensorflow.python.keras.metrics import Recall, Metric, CategoricalCrossentropy
import numpy as np
from texttable import Texttable

from socialdetector.dataset.social_images.social_images import SocialImages
from socialdetector.dataset_utils import datasets_concatenate
from socialdetector.ds_split import DsSplit

class Metrics(Callback):

    def __init__(self, val_generator: Dataset, freq=10):
        super().__init__()

        def filter_fn(i, n):
            zeros = [0] * n
            zeros[i] = 1
            return lambda *args: tf.reduce_all(args[1] == zeros)

        label_shape = val_generator.output_shapes[1]
        n = label_shape[0]

        self.freq = freq
        self.max_met = 0

        self.cls = [val_generator.filter(filter_fn(i, n)).batch(128).cache() for i in range(n)]

    def on_epoch_end(self, epoch, logs=None):
        if epoch % self.freq != 1:
            return
        self.model: Model
        # print("L3", len(self.cls3))
        hand = getattr(self.model, "_eval_data_handler", None)
        self.model._eval_data_handler = None
        mets = []
        for i in range(len(self.cls)):
            print("Class %d:" % i)
            mets.append(self.model.evaluate(self.cls[i]))
        self.model._eval_data_handler = hand
        mets = np.mean(np.array(mets), axis=0)
        print("\n\nMetrics: ", mets)
        val = mets[1]
        if val > self.max_met:
            self.max_met = val
            print("NEW_MAX", val)

class MyValidation(Callback):

    def __init__(self, ds, batch_size=128, name='Validation'):

        super().__init__()

        def filter_fn(i, n):
            zeros = [0] * n
            zeros[i] = 1
            return lambda *args: tf.reduce_all(args[1] == zeros)

        label_shape = tf.compat.v1.data.get_output_shapes(ds)[1]
        n = label_shape[0]

        self.max_met = 0
        self.name = name
        self.cls = [ds.filter(filter_fn(i, n)).batch(batch_size).cache() for i in range(n)]
        self.fp = open('log_%s.txt' % name, 'w')

    def on_epoch_end(self, epoch, logs=None):
        if logs is None:
            logs = {}

        hand = getattr(self.model, "_eval_data_handler", None)
        self.model._eval_data_handler = None
        mets = []
        print('\n')
        for i in range(len(self.cls)):
            print("%s %d:" % (self.name, i))
            mets.append(self.model.evaluate(self.cls[i]))
        self.model._eval_data_handler = hand
        mets = np.mean(np.array(mets), axis=0)
        self.model: Model
        assert len(self.model.metrics) == (len(mets))
        for m, val in zip(self.model.metrics, mets):
            m: Metric
            logs['val_' + m.name] = val
        print(logs)
        self.fp.write('%d: %r\n' % (epoch, logs))
        self.fp.flush()
        return logs

def evaluate_ds(model, ds, mapping, img_metrics=(), chunk_metrics=()):
    def take(ds, n):
        pred, true = [], []
        for i in range(n):
            e = next(ds)
            pred.append(e[0])
            true.append(e[1])
        return pred, true

    def map_pred(ds):
        for sample in ds:
            x = sample[0]
            y_true = sample[1]
            y_pred = model.predict(x)
            for e in zip(y_pred, y_true):
                yield e

    def form(arr):
        return list(map('{:.2f}'.format, arr))

    n_lab = tf.compat.v1.data.get_output_shapes(ds)[1][1]
    ds = map_pred(ds)
    cce = CategoricalCrossentropy()

    total = 0
    avgs = []
    for entry in mapping:
        name, chunks = entry
        name = name.decode("utf-8")
        y_pred, y_true = take(ds, chunks)
        total += len(y_pred)
        # print(name, chunks, "LOSS", cce(y_true, y_pred).numpy())
        for metric in chunk_metrics:
            metric.update_state(np.array(y_true), np.array(y_pred))
        y_pred2 = np.mean(y_pred, axis=0)
        y_pred = np.argmax(y_pred, axis=1)
        zeros = [0] * n_lab
        ele, count = np.unique(y_pred, return_counts=True)
        lbm = {i: 0 for i in range(n_lab)}
        lbm = {**lbm, **{ele[i]: count[i] for i in range(len(ele))}}
        counts = [int(lbm[i]) for i in range(n_lab)]
        y_pred = ele[max(range(len(ele)), key=lambda x: count[x])]
        zeros[y_pred] = 1
        y_pred = zeros
        y_true = y_true[0]
        print(counts[tf.argmax(y_true)], '/', sum(counts))
        avgs.append([n / sum(counts) for n in counts])
        if tf.argmax(y_true) != tf.argmax(y_pred):
            print("WRONG: %s" % name, form(y_pred2), " | ", counts, " -> ", form(y_true))
        # print(form(y_pred), form(y_true.numpy()))
        for metric in img_metrics:
            metric.update_state(y_true, y_pred)

    more = 0
    print("AVGSS", np.mean(np.array(avgs), axis=0))
    try:
        while True:
            next(ds)
            more += 1
    except StopIteration:
        if more:
            print("WARNING: more data then expected (%d)" % more)

    print("Evaluated %d/%d chunks" % (total, total + more))

class EvaluateCallback(Callback):

    @staticmethod
    def run_experiment(exp):
        exp.get_datasets()
        e = EvaluateCallback(exp.get_datasets()[2], exp.img_mappings[2], exp.labels_map)
        e.model = exp.model.model
        e.eval()

    @staticmethod
    def run(model, *args, **kwargs):
        cb = EvaluateCallback(*args, **kwargs)
        cb.model = model
        cb.eval()

    def __init__(self, val_generator: Dataset, mapping, labels_map):
        super().__init__()

        inverse_labels = {tuple(v): k for k, v in labels_map.items()}

        def label_of(i, n):
            zeros = [0] * n
            zeros[i] = 1
            return zeros

        def filter_fn(i, n):
            lb = label_of(i, n)
            return lambda *args: tf.reduce_all(args[1] == lb)

        label_shape = tf.compat.v1.data.get_output_shapes(val_generator)[1]
        n = label_shape[0]

        self.cls = [val_generator.filter(filter_fn(i, n)).batch(128).cache() for i in range(n)]
        self.labels = [inverse_labels[tuple(label_of(i, n))] for i in range(n)]
        self.mappings = [mapping[lb] for lb in self.labels]

    def on_epoch_end(self, epoch, logs=None):
        self.eval()

    def eval(self):
        image_wise = Recall()
        chunk_wise = Recall()
        cm = ConfusionMatrix(self.labels)
        chunk_cm = ConfusionMatrix(self.labels)
        for cl_ds, mapping, label in zip(self.cls, self.mappings, self.labels):
            class_wise = Recall()
            class_chunk = Recall()
            evaluate_ds(self.model, cl_ds, mapping, (image_wise, class_wise, cm), (chunk_cm, chunk_wise, class_chunk))
            print("Image class recall %s" % label)
            print(class_wise.result())
            print("Chunk class recall %s" % label)
            print(class_chunk.result())
        print("Image-wise recall")
        print(image_wise.result())
        print("Chunk-wise recall")
        print(chunk_wise.result())
        print("Confusion matrix")
        cm.print_result()
        print("Chunk matrix")
        chunk_cm.print_result()

class ConfusionMatrix(Metric):

    def __init__(self, labels, **kwargs):
        super().__init__(**kwargs)
        self.labels = labels
        self.size = len(labels)
        self.matrix = [[0] * self.size for _ in range(self.size)]

    def update_state_np(self, y_true, y_pred):
        if len(y_true.shape) > 0:
            for t, p in zip(y_true, y_pred):
                self.update_state_np(t, p)
            return
        self.matrix[y_true][y_pred] += 1

    def get_matrix(self):
        return self.matrix

    def update_state(self, y_true, y_pred, sample_weight=None):
        if len(y_true.shape) > 1:
            true_cls = tf.argmax(y_true, axis=1)
            pred_cls = tf.argmax(y_pred, axis=1)
            return tf.numpy_function(self.update_state_np, [true_cls, pred_cls], ())
        true_cls = tf.argmax(y_true)
        pred_cls = tf.argmax(y_pred)
        self.matrix[true_cls][pred_cls] += 1

    def result(self):
        mat = tf.numpy_function(self.get_matrix, (), [tf.int32, tf.int32, tf.int32])
        # mat.set_shape((self.size, self.size))
        """for pred_line in self.matrix:
            tot = sum(pred_line)
            if tot > 0:
                mat.append([x / tot for x in pred_line])
            else:
                mat.append(pred_line)"""
        # tf.print(mat)
        # tf.print(self.matrix)
        return mat

    def print_result(self):
        table = Texttable()
        res = self.result().numpy().reshape((self.size, self.size))
        th = ["", *self.labels]
        table.add_row(th)
        i = 0
        for pred_line in res:
            tot = sum(pred_line)
            if tot > 0:
                pred_line = [x / tot for x in pred_line]
            table.add_row([self.labels[i], *pred_line])
            i += 1

        print(table.draw())

def _all_equals(iterator):
    first = True
    last = None
    for i in iterator:
        if not first and i != last:
            return False
        first = False
        last = i
    return True

def balance_validation(ds):
    print('Balancing validation...')

    def filter_fn(i, n):
        zeros = [0] * n
        zeros[i] = 1
        return lambda *args: tf.reduce_all(args[1] == zeros)

    oss = tf.compat.v1.data.get_output_shapes(ds)
    label_shape = oss[1]
    n = label_shape[0]

    if len(oss) > 2:
        gw = lambda *args: args[2]
        st = lambda f: lambda x, y, w: (x, y, w*f)
    else:
        gw = lambda *args: 1
        st = lambda f: lambda x, y: (x, y, f)

    cls = [ds.filter(filter_fn(i, n)) for i in range(n)]
    weights = [sum(cl.map(gw).as_numpy_iterator()) for cl in cls]
    print('Before balancing', weights)
    if _all_equals([round(w*100) for w in weights]):
        return ds
    dest_w = np.mean(weights)
    fs = [dest_w/w for w in weights]
    fns = [st(f) for f in fs]
    cls = [cl.map(f) for cl, f in zip(cls, fns)]
    weights = [sum(cl.map(lambda *args: args[2]).as_numpy_iterator()) for cl in cls]
    print('After balancing', weights)
    return datasets_concatenate(cls)


"""Catalogs are immutable mappings useful for building indexes.

This module contains definitions of many different catalog types,
all of which implement the interface defined by the Catalog abstract
base class, which is itself implements the mapping protocol.

Rather than constructing Catalog subtypes directly, prefer to use
the CatalogBuilder class which will analyse the contents of the
mapping to find a space and time efficient representation.
"""
from collections import Mapping, Sequence, OrderedDict, Iterable
from fractions import Fraction
from itertools import product

from segpy.sorted_frozen_set import SortedFrozenSet
from segpy.util import (contains_duplicates, measure_stride, make_sorted_distinct_sequence,
                        is_sorted, first)

class CatalogBuilder(object):
    """Use a catalog builder to construct optimised, immutable mappings.

    A CatalogBuilder is useful when, depending on the particular keys and
    values used, a more compact or efficient representation of the mapping
    is possible than, say, a regular dictionary. The CatalogBuilder
    accumulates values and then, once all values have been added, analyzes
    the keys and values to produce a more optimized representation of the
    mapping.
    """

    def __init__(self, mapping=None):
        """Initialize a Catalog Builder.

        Args:
            mapping: An optional mapping (such as a dictionary) of items, or an
                iterable series of 2-tuples containing keys and values.

        Raises:
            TypeError: If mapping is not None and is neither of mapping nor iterable type.
            ValueError: If mapping is an iterable of tuples, and the tuples are not pairs.
        """
        if mapping is None:
            self._catalog = []
        elif isinstance(mapping, Mapping):
            self._catalog = list(mapping.items())
        elif isinstance(mapping, Iterable):
            self._catalog = []
            for pair in mapping:
                if len(pair) != 2:
                    raise ValueError("{!r} is not a pair. Catalogs can only be constructed "
                                     "from iterable series of 2-tuples.")
                self._catalog.append(pair)
        else:
            raise TypeError("Mapping must be either a mapping (e.g. dict), or an iterable "
                            "series of 2-tuples. {!r} does not qualify.")

    def add(self, index, value):
        """Add an item.

        Each index must be unique if create() is to be subsequently
        called successfully, although duplicate index values will be
        accepted by this call without complaint.

        """
        self._catalog.append((index, value))

    def create(self):
        """Create a possibly more optimized representation of the mapping.

        In this worst case, this method returns an object which is
        essentially an immutable dictionary. In the best case, the
        space savings can be vast.

        Returns:
            A mapping, if a unique mapping from indexes to values is
            possible, otherwise None.

        """

        # This method examines the contents of the mapping using
        # various heuristics to come up with a better representation.

        # In-place sort by index
        self._catalog.sort(key=first)

        if contains_duplicates(index for index, value in self._catalog):
            return None

        if all(isinstance(index, Sequence) and (len(index) == 2)
               for index, value in self._catalog):
            return self._create_catalog_2()

        return self._create_catalog_1()

    def _create_catalog_1(self):
        """Create a catalog for one-dimensional integer keys (i.e. scalars)
        """
        if len(self._catalog) <= 1:
            return DictionaryCatalog(self._catalog)

        index_min = self._catalog[0][0]
        index_max = self._catalog[-1][0]
        index_stride = measure_stride(index for index, value in self._catalog)
        assert index_stride != 0

        value_start = self._catalog[0][1]
        value_stop = self._catalog[-1][1]
        value_stride = measure_stride(value for index, value in self._catalog)

        if index_stride is not None and value_stride == 0:
            assert value_start == value_stop
            return RegularConstantCatalog(index_min,
                                          index_max,
                                          index_stride,
                                          value_start)

        if index_stride is None and value_stride == 0:
            assert value_start == value_stop
            return ConstantCatalog(
                    (index for index, value in self._catalog),
                    value_start)

        if index_stride is not None and value_stride is None:
            # Regular index - regular keys and arbitrary values
            return RegularCatalog(index_min,
                                  index_max,
                                  index_stride,
                                  (value for index, value in self._catalog))

        if (index_stride is not None) and (value_stride is not None):
            assert value_stride != 0
            return LinearRegularCatalog(index_min,
                                        index_max,
                                        index_stride,
                                        value_start,
                                        value_stop,
                                        value_stride)

        return DictionaryCatalog(self._catalog)

    def _create_catalog_2(self):
        """Create a catalog for two-dimensional integer keys.

        Each key must be a two-element sequence.
        """
        return (self.make_last_index_varies_quickest_catalog_2d()
             or self.make_first_index_varies_quickest_catalog_2d()
             or self.make_dictionary_catalog_2d())

    def make_last_index_varies_quickest_catalog_2d(self):
        self._catalog.sort(key=_first_then_second_index)
        sorted_sequences = self.make_sorted_ranges()
        if sorted_sequences is None:
            return None
        return LastIndexVariesQuickestCatalog2D(*sorted_sequences)

    def make_first_index_varies_quickest_catalog_2d(self):
        self._catalog.sort(key=_second_then_first_index)
        sorted_sequences = self.make_sorted_ranges()
        if sorted_sequences is None:
            return None
        return FirstIndexVariesQuickestCatalog2D(*sorted_sequences)

    def make_dictionary_catalog_2d(self):
        self._catalog.sort(key=_first_then_second_index)
        i_sorted = make_sorted_distinct_sequence(i for (i, _), _ in self._catalog)
        j_sorted = make_sorted_distinct_sequence(j for (_, j), _ in self._catalog)
        return DictionaryCatalog2D(i_sorted, j_sorted, self._catalog)

    def make_sorted_ranges(self):
        i_sorted = make_sorted_distinct_sequence(i for (i, _), _ in self._catalog)
        j_sorted = make_sorted_distinct_sequence(j for (_, j), _ in self._catalog)
        if len(i_sorted) * len(j_sorted) != len(self._catalog):
            # The do not match so use a dictionary-based mapping
            return None
        vs = [v for (_, _), v in self._catalog]
        # Are the values unique and in ascending or descending order?
        if not (is_sorted(vs, reverse=False, distinct=True) or is_sorted(vs, reverse=True, distinct=True)):
            # The values are not both unique and sorted, so use a dictionary-based mapping
            return None
        v_sorted = make_sorted_distinct_sequence(vs, sense=None)
        i_is_regular = isinstance(i_sorted, range)
        j_is_regular = isinstance(j_sorted, range)
        v_is_regular = isinstance(v_sorted, range)
        if not (i_is_regular and j_is_regular and v_is_regular):
            # The i, j and v values are not regularly spaced, so use a dictionary-based mapping
            return None
        return i_sorted, j_sorted, v_sorted

class Catalog2D(Mapping):
    """An abstract base class for 2D catalogs.
    """

    def __init__(self, i_range, j_range):
        """Initialize a Catalog2D.

        Args:
            i_range: A range which can generate all and only valid i indexes.
            j_range: A range which can generate all and only valid j indexes.

        Raises:
            ValueError: If either i_range or j_range are not totally sorted.
        """
        if not is_sorted(i_range, distinct=True):
            raise ValueError("i indexes must be sorted and unique")
        if not is_sorted(j_range, distinct=True):
            raise ValueError("j indexes must be sorted and unique")
        self._i_range = i_range
        self._j_range = j_range

    @property
    def i_range(self):
        # TODO: This should be renamed, as it's not necessarily a range - just a sorted container (sequence?)
        return self._i_range

    @property
    def j_range(self):
        # TODO: This should be renamed, as it's not necessarily a range - just a sorted container (sequence?)
        return self._j_range

    @property
    def i_min(self):
        """Minimum i value"""
        return self._i_range[0]

    @property
    def i_max(self):
        """Maximum i value"""
        return self._i_range[-1]

    @property
    def j_min(self):
        """Minimum j value"""
        return self._j_range[0]

    @property
    def j_max(self):
        """Maximum j value"""
        return self._j_range[-1]

    def key_min(self):
        """Minimum (i, j) key"""
        return self.i_min, self.j_min

    def key_max(self):
        """Maximum (i, j) key"""
        return self.i_max, self.j_max

    def value_first(self):
        """Minimum value at key_min"""
        return self[self.key_min()]

    def value_last(self):
        """Maximum value at key_max"""
        return self[self.key_max()]

class LastIndexVariesQuickestCatalog2D(Catalog2D):

    def __init__(self, i_range, j_range, v_range):
        """
        Args:
            i_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid i keys.

            j_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid j keys.

            v_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid values.
        """
        num_indices = len(i_range) * len(j_range)
        if num_indices != len(v_range):
            raise ValueError("i_range={} and j_range={} totalling {} indices are incompatible with "
                             "v_range={} with length {}".format(i_range, j_range, num_indices, v_range, len(v_range)))
        super().__init__(i_range, j_range)
        self._v_range = v_range

    @property
    def v_range(self):
        return self._v_range

    def __getitem__(self, key):
        i, j = key
        if not self._contains(i, j):
            raise KeyError("Key {!r} not in {!r}".format(key, self))
        i_index = self.i_range.index(i)
        j_index = self.j_range.index(j)
        v_index = i_index * len(self.j_range) + j_index
        v = self._v_range[v_index]
        return v

    def __contains__(self, key):
        i, j = key
        return self._contains(i, j)

    def _contains(self, i, j):
        return (i in self.i_range) and (j in self.j_range)

    def __len__(self):
        return len(self.i_range) * len(self.j_range)

    def __iter__(self):
        return product(self.i_range, self.j_range)

    def __repr__(self):
        return '{}(i_range={}, j_range={}, v_range={})'.format(
            self.__class__.__name__,
            self.i_range, self.j_range, self._v_range)

    def key(self, value):
        """Given a value, get the corresponding key.

        Args:
            value: The value for which to find the key.

        Returns:
            A 2-tuple containing the (i, j) values corresponding
            to the given value.

        Raises:
            ValueError: If there is no corresponding key.
        """

        try:
            v_index = self._v_range.index(value)
        except ValueError:
            raise ValueError("{!r} is not a value of {!r}".format(value, self))
        i_index, j_index = divmod(v_index, len(self.j_range))
        i = self.i_range[i_index]
        j = self.j_range[j_index]
        return (i, j)

class FirstIndexVariesQuickestCatalog2D(Catalog2D):

    def __init__(self, i_range, j_range, v_range):
        """
        Args:
            i_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid i keys.

            j_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid j keys.

            v_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid values.
        """
        num_indices = len(i_range) * len(j_range)
        if num_indices != len(v_range):
            raise ValueError("i_range={} and j_range={} totalling {} indices are incompatible with "
                             "v_range={} with length {}".format(i_range, j_range, num_indices, v_range, len(v_range)))
        super().__init__(i_range, j_range)
        self._v_range = v_range

    @property
    def v_range(self):
        return self._v_range

    def __getitem__(self, key):
        i, j = key
        if not self._contains(i, j):
            raise KeyError("Key {!r} not in {!r}".format(key, self))
        i_index = self.i_range.index(i)
        j_index = self.j_range.index(j)
        v_index = j_index * len(self.i_range) + i_index
        v = self._v_range[v_index]
        return v

    def __contains__(self, key):
        i, j = key
        return self._contains(i, j)

    def _contains(self, i, j):
        return (i in self.i_range) and (j in self.j_range)

    def __len__(self):
        return len(self.i_range) * len(self.j_range)

    def __iter__(self):
        return ((i, j) for (j, i) in product(self.j_range, self.i_range))

    def __repr__(self):
        return '{}(i_range={}, j_range={}, v_range={})'.format(
            self.__class__.__name__,
            self.i_range, self.j_range, self._v_range)

    def key(self, value):
        """Given a value, get the corresponding key.

        Args:
            value: The value for which to find the key.

        Returns:
            A 2-tuple containing the (i, j) values corresponding
            to the given value.

        Raises:
            ValueError: If there is no corresponding key.
        """

        try:
            v_index = self._v_range.index(value)
        except ValueError:
            raise ValueError("{!r} is not a value of {!r}".format(value, self))
        j_index, i_index = divmod(v_index, len(self.i_range))
        i = self.i_range[i_index]
        j = self.j_range[j_index]
        return (i, j)

class DictionaryCatalog(Mapping):
    """An immutable, ordered, dictionary mapping.
    """

    def __init__(self, items):
        self._items = OrderedDict(items)

    def __getitem__(self, key):
        return self._items[key]

    def __iter__(self):
        return iter(self._items)

    def __len__(self):
        return len(self._items)

    def __contains__(self, item):
        return item in self._items

    def __repr__(self):
        return '{}(items=[<{} items>])'.format(
            self.__class__.__name__, len(self._items))

class DictionaryCatalog2D(Catalog2D):
    """An immutable, ordered, dictionary mapping for 2D keys.
    """

    def __init__(self, i_range, j_range, items):
        """Initialize a DictionaryCatalog2D.

        Args:
            i_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid i values.

            j_range: An ordered collection of evenly-spaced integers
                which can be used to generate valid j values.

            items: Either a) a mapping (dict) where the keys are 2-tuples containing i and j values
                contained within the respective ranges, and the value is an integer, or b) an
                iterable series of tuple structures ((i, j), v).

        Raises:
            TypeError: If items is not a dictionary or iterable of tuples.
            ValueError: If the i and j components of the keys in items are not contained within
                the respective i_range and j_range collections.
        """
        super().__init__(i_range, j_range)
        self._items = OrderedDict()
        if isinstance(items, Mapping):
            mapping = items.items()
        elif isinstance(items, Iterable):
            mapping = items
        else:
            raise TypeError("{} must be constructed from a mapping or an iterable of ((i, j), v) tuples, "
                            "not {}".format(self.__class__.__name__, items))

        for key, value in mapping:
            i, j = key
            if i not in i_range:
                raise ValueError("i={} not in i_range".format(i))
            if j not in j_range:
                raise ValueError("j={} not in j_range".format(j))
            self._items[key] = value

    def __getitem__(self, key):
        return self._items[key]

    def __iter__(self):
        return iter(self._items)

    def __len__(self):
        return len(self._items)

    def __contains__(self, item):
        return item in self._items

    def __repr__(self):
        return '{}(i_range={}, j_range={}, items=[<{} items>])'.format(
            self.__class__.__name__,
            self.i_range, self.j_range,
            len(self._items))

class RegularConstantCatalog(Mapping):
    """Mapping with keys ordered with regular spacing along the number line.

    The values associated with the keys are constant.
    """

    def __init__(self, key_min, key_max, key_stride, value):
        """Initialize a RegularConstantCatalog.

        The catalog is initialized by a description of how the keys
        are distributed along the number line, and a value which
        corresponds with all keys.

        Args:
            key_min: The minimum key.
            key_max: The maximum key.
            key_stride: The difference between successive keys.
            value: A value associated with all keys.
        """
        if key_min > key_max:
            raise ValueError("key_min {} is greater than key_max {}, but key_min must be equal or less-than key_max."
                             .format(key_min, key_max))

        key_range = key_max - key_min
        if key_range % key_stride != 0:
            raise ValueError("RegularIndex key range {!r} is not "
                             "a multiple of stride {!r}".format(
                                 key_stride, key_range))

        self._key_min = key_min
        self._key_max = key_max
        self._key_stride = key_stride
        self._value = value

    def __getitem__(self, key):
        if key not in self:
            raise KeyError("{!r} does not contain key {!r}".format(self, key))
        return self._value

    def __len__(self):
        return 1 + (self._key_max - self._key_min) // self._key_stride

    def __contains__(self, key):
        return (self._key_min <= key <= self._key_max) and \
               ((key - self._key_min) % self._key_stride == 0)

    def __iter__(self):
        return iter(range(self._key_min,
                          self._key_max + 1,
                          self._key_stride))

    def __repr__(self):
        return '{}(key_min={}, key_max={}, key_stride={}, value={})'.format(
            self.__class__.__name__,
            self._key_min,
            self._key_max,
            self._key_stride,
            self._value)

class ConstantCatalog(Mapping):
    """Mapping with arbitrary keys and a single constant value.
    """

    def __init__(self, keys, value):
        """Initialize a ConstantCatalog.

        The catalog is initialized by a description with an iterable series of
        keys and a constant value to be associated with all the keys.

        Args:
            keys: An iterable series of keys.
            value: A value associated with all keys.
        """
        self._keys = SortedFrozenSet(keys)
        self._value = value

    def __getitem__(self, key):
        if key not in self:
            raise KeyError("{!r} does not contain key {!r}".format(self, key))
        return self._value

    def __len__(self):
        return len(self._keys)

    def __contains__(self, key):
        return key in self._keys

    def __iter__(self):
        return iter(self._keys)

    def __repr__(self):
        return '{}(keys=[{} items], value={})'.format(
            self.__class__.__name__,
            len(self._keys),
            self._value)

class RegularCatalog(Mapping):
    """Mapping with keys ordered with regular spacing along the number line.

    The values associated with the keys are arbitrary.
    """

    def __init__(self, key_min, key_max, key_stride, values):
        """Initialize a RegularCatalog.

        The catalog is initialized by a description of how the keys
        are distributed along the number line, and an iterable series of
        corresponding values.

        Args:
            key_min: The minimum key.
            key_max: The maximum key.
            key_stride: The difference between successive keys.
            values: An iterable series of values corresponding to the keys.

        Raises:
            ValueError: There is any inconsistency in the keys, stride,
                and/or values.
        """
        if key_min > key_max:
            raise ValueError("key_min {} is greater than key_max {}, but key_min must be equal or less-than key_max."
                             .format(key_min, key_max))

        key_range = key_max - key_min
        if key_range % key_stride != 0:
            raise ValueError("{} key range {!r} is not "
                             "a multiple of stride {!r}".format(self.__class__.__name__,
                                                                key_stride, key_range))
        self._key_min = key_min
        self._key_max = key_max
        self._key_stride = key_stride
        self._values = list(values)
        num_keys = 1 + key_range // key_stride
        if num_keys != len(self._values):
            raise ValueError("{} key range with length {} and values with length {} are inconsistent"
                             .format(self.__class__.__name__, num_keys, len(self._values)))

    def __getitem__(self, key):
        if not (self._key_min <= key <= self._key_max):
            raise KeyError("{!r} key {!r} out of range".format(self, key))
        offset = key - self._key_min
        if offset % self._key_stride != 0:
            raise KeyError("{!r} does not contain key {!r}".format(self, key))
        index = offset // self._key_stride
        return self._values[index]

    def __len__(self):
        return len(self._values)

    def __contains__(self, key):
        return (self._key_min <= key <= self._key_max) and \
               ((key - self._key_min) % self._key_stride == 0)

    def __iter__(self):
        return iter(range(self._key_min,
                          self._key_max + 1,
                          self._key_stride))

    def __repr__(self):
        return '{}(key_min={}, key_max={}, key_stride={}, values=[{} items])'.format(
            self.__class__.__name__,
            self._key_min,
            self._key_max,
            self._key_stride,
            len(self._values))

class LinearRegularCatalog(Mapping):
    """A mapping which assumes a linear relationship between keys and values.

    A LinearRegularCatalog predicts the value v from the key according to the
    following formula:

        v = (value_max - value_min) / (key_max - key_min) * (key - key_min) + value_min
    """

    def __init__(self,
                 key_min,
                 key_max,
                 key_stride,
                 value_start,
                 value_stop,
                 value_stride):
        """Initialize a LinearRegularCatalog.

        The catalog must contain at least two items.

        Args:
            key_min: The minimum key.
            key_max: The maximum key.
            key_stride: The difference between successive keys.
            value_start: The value corresponding to the minimum key.
            value_stop: The value corresponding to the maximum key.
            value_stride:

        Raises:
            ValueError: There is any inconsistency in the keys, strides,
                and/or values.
        """
        if key_min >= key_max:
            raise ValueError("key_min {} is greater-than or equal-to key_max {}, but key_min must be less-than key_max."
                             .format(key_min, key_max))

        if key_stride <= 0:
            raise ValueError("key_stride {} is not positive".format(key_stride))

        key_range = key_max - key_min
        if key_range % key_stride != 0:
            raise ValueError("{} key range {!r} is not "
                             "a multiple of key stride {!r}".format(
                                 self.__class__.__name__,
                                 key_stride,
                                 key_range))
        self._key_stride = key_stride

        if value_stride == 0:
            raise ValueError("value_stride {} cannot be zero".format(value_stride))

        value_range = value_stop - value_start
        if value_range % value_stride != 0:
            raise ValueError("{} value range {!r} is not "
                             "a multiple of value stride {!r}".format(
                                 self.__class__.__name__,
                                 value_stride,
                                 value_range))
        self._value_stride = value_stride

        self._key_min = key_min
        self._key_max = key_max
        self._value_start = value_start
        self._value_stop = value_stop

        num_keys = 1 + (self._key_max - self._key_min) // self._key_stride
        num_values = 1 + (self._value_stop - self._value_start) // self._value_stride
        if num_keys != num_values:
            raise ValueError("{} inconsistent number of "
                             "keys {} and values {}".format(
                                 self.__class__.__name__,
                                 num_keys,
                                 num_values))

        self._m = Fraction(self._value_stop - self._value_start,
                           self._key_max - self._key_min)

    def __getitem__(self, key):
        if not (self._key_min <= key <= self._key_max):
            raise KeyError("{!r} key {!r} out of range".format(self, key))
        offset = key - self._key_min
        if offset % self._key_stride != 0:
            raise KeyError("{!r} does not contain key {!r}".format(self, key))

        v = self._m * (key - self._key_min) + self._value_start
        assert v.denominator == 1
        return v.numerator

    def __len__(self):
        return 1 + (self._key_max - self._key_min) // self._key_stride

    def __contains__(self, key):
        return (self._key_min <= key <= self._key_max) and \
               ((key - self._key_min) % self._key_stride == 0)

    def __iter__(self):
        return iter(range(self._key_min, self._key_max + 1, self._key_stride))

    def __repr__(self):
        return '{}(key_min={}, key_max={}, key_stride={}, value_first={}, value_last={}, value_stride={})'.format(
            self.__class__.__name__,
            self._key_min,
            self._key_max,
            self._key_stride,
            self._value_start,
            self._value_stop,
            self._value_stride)

def _first_then_second_index(key):
    return (key[0][0], key[0][1])

def _second_then_first_index(key):
    return (key[0][1], key[0][0])


from ..helpers.df_info import DataFrameType
from typing import Union
from abc import ABC, abstractmethod

class Shortcuts(ABC):
    @abstractmethod
    def chat(self, prompt: str) -> DataFrameType:
        """
        Run method from PandasAI class.

        Args:
            prompt (str): A prompt to be sent to LLM.

        Returns:
            DataFrameType: The response from the LLM.
        """

        pass

    def clean_data(self) -> DataFrameType:
        """
        Do data cleaning and return the dataframe.

        Returns:
            DataFrameType: The cleaned DataFrame.
        """

        return self.chat(
            """
1. Copy the dataframe to a new variable named df_cleaned.
2. Do data cleaning.
3. Return df_cleaned.
"""
        )

    def impute_missing_values(self) -> DataFrameType:
        """
        Do missing value imputation and return the dataframe.

        Returns:
            DataFrameType: The DataFrame with imputed missing values.
        """

        return self.chat(
            """
1. Copy the dataframe to a new variable named df_imputed.
2. Do the imputation of missing values.
3. Return df_imputed.
"""
        )

    def generate_features(self) -> DataFrameType:
        """
        Do feature generation and return the dataframe.

        Returns:
            DataFrameType: The DataFrame with generated features.
        """

        return self.chat(
            """
1. Copy the dataframe to a new variable named df_features.
2. Do feature generation.
3. Return df_features.
"""
        )

    def plot_pie_chart(self, labels: list, values: list) -> None:
        """
        Plot a pie chart.

        Args:
            labels (list): The labels for the pie chart.
            values (list): The values for the pie chart.

        Returns:
            None
        """

        self.chat(
            f"""
Plot a pie chart with the following labels and values:
labels = {labels}
values = {values}
"""
        )

    def plot_bar_chart(self, x: list, y: list) -> None:
        """
        Plot a bar chart.

        Args:
            x (list): The x values for the bar chart.
            y (list): The y values for the bar chart.

        Returns:
            None
        """

        self.chat(
            f"""
Plot a bar chart with the following x and y:
x = {x}
y = {y}
"""
        )

    def plot_histogram(self, column: str) -> None:
        """
        Plot a histogram.

        Args:
            column (str): The column to plot the histogram for.

        Returns:
            None
        """

        self.chat(f"Plot a histogram of the column {column}.")

    def plot_line_chart(self, x: list, y: list) -> None:
        """
        Plot a line chart.

        Args:
            x (list): The x values for the line chart.
            y (list): The y values for the line chart.

        Returns:
            None
        """

        self.chat(
            f"""
Plot a line chart with the following x and y:
x = {x}
y = {y}
"""
        )

    def plot_scatter_chart(self, x: list, y: list) -> None:
        """
        Plot a scatter chart.

        Args:
            x (list): The x values for the scatter chart.
            y (list): The y values for the scatter chart.

        Returns:
            None
        """

        self.chat(
            f"""
Plot a scatter chart with the following x and y:
x = {x}
y = {y}
"""
        )

    def plot_correlation_heatmap(self) -> None:
        """
        Plot a correlation heatmap.

        Returns:
            None
        """

        self.chat("Plot a correlation heatmap.")

    def plot_confusion_matrix(self, y_true: list, y_pred: list) -> None:
        """
        Plot a confusion matrix.

        Args:
            y_true (list): The true values.
            y_pred (list): The predicted values.

        Returns:
            None
        """

        self.chat(
            f"""
Plot a confusion matrix with the following y_true and y_pred:
y_true = {y_true}
y_pred = {y_pred}
"""
        )

    def plot_roc_curve(self, y_true: list, y_pred: list) -> None:
        """
        Plot a ROC curve.

        Args:
            y_true (list): The true values.
            y_pred (list): The predicted values.

        Returns:
            None
        """

        self.chat(
            f"""
Plot a ROC curve with the following y_true and y_pred:
y_true = {y_true}
y_pred = {y_pred}
"""
        )

    def boxplot(
        self,
        col: Union[str, list[str]] = None,
        by: Union[str, list[str]] = None,
        style: str = None,
    ):
        """
        Draw a box plot to show distributions with respect to categories.

        Args:
            col (str | list[str] | None): The column(s) of interest
            for the box plot. Defaults to None.
            by (str | list[str] | None): The grouping variable(s)
            for the box plot. Defaults to None.
            style (str | None): The textual description of the desired
            style. Defaults to None.

        Returns:
            str: LLM response.

        """

        if not isinstance(col, (str, list, type(None))):
            raise TypeError(
                "The 'col' argument must be a string, a list of strings, or None."
            )
        if not isinstance(by, (str, list, type(None))):
            raise TypeError(
                "The 'by' argument must be a string, a list of strings, or None."
            )

        prompt = "Plot a box-and-whisker plot"

        if isinstance(col, str):
            prompt += f" for the variable '{col}'"
        elif isinstance(col, list):
            var_list = [f"'{v}'" for v in col]
            if len(var_list) > 1:
                variables_str = ", ".join(var_list[:-1])
                prompt += f" for the variables {variables_str} and {var_list[-1]}"
            else:
                prompt += f" for the variable {var_list[0]}"

        if by is not None:
            prompt += f" grouped by '{by}'"

        if style is not None:
            prompt += f"\nStyle: '''{style}'''"

        self.chat(prompt)

    def rolling_mean(self, column: str, window: int) -> DataFrameType:
        """
        Calculate the rolling mean.

        Args:
            column (str): The column to calculate the rolling mean for.
            window (int): The window size.

        Returns:
            DataFrameType: The DataFrame containing the rolling mean.
        """

        return self.chat(
            f"Calculate the rolling mean of the column {column} with a window"
            f" of {window}.",
        )

    def rolling_median(self, column: str, window: int) -> DataFrameType:
        """
        Calculate the rolling median.

        Args:
            column (str): The column to calculate the rolling median for.
            window (int): The window size.

        Returns:
            DataFrameType: The DataFrame containing the rolling median.
        """

        return self.chat(
            f"Calculate the rolling median of the column {column} with a window"
            f" of {window}.",
        )

    def rolling_std(self, column: str, window: int) -> DataFrameType:
        """
        Calculate the rolling standard deviation.

        Args:
            column (str): The column to calculate the rolling standard deviation for.
            window (int): The window size.

        Returns:
            DataFrameType: The DataFrame containing the rolling standard deviation.
        """

        return self.chat(
            f"Calculate the rolling standard deviation of the column {column} with a"
            f"window of {window}.",
        )

    def segment_customers(self, features: list, n_clusters: int) -> DataFrameType:
        """
        Segment customers.

        Args:
            features (list): The features to use for the segmentation.
            n_clusters (int): The number of clusters.

        Returns:
            DataFrameType: The DataFrame containing the segmentation.
        """

        return self.chat(
            f"""
Segment customers with the following features and number of clusters:
features = {features}
n_clusters = {n_clusters}
"""
        )


"""
数据库初始化需要的模块
"""

from nonebot.log import logger
from tortoise import Tortoise

async def database_init():
    """
    初始化建表
    """
    logger.debug("正在注册数据库")
    database_path = "./data/data.db"
    db_url = f"sqlite://{database_path}"
    # 这里填要加载的表
    models = [
        "src.modules.group_info",
        "src.modules.plugin_info",
        "src.modules.user_info",
        "src.modules.ticket_info",
        "src.modules.search_record",
    ]
    modules = {"models": models}
    await Tortoise.init(db_url=db_url, modules=modules)
    await Tortoise.generate_schemas()
    logger.opt(colors=True).info("<g>数据库初始化成功。</g>")


from flasgger import swag_from
from flask import Blueprint
from flask_jwt_extended import jwt_required
from ..database.crud.playlist import (
    create_playlist, create_many, get_playlist, delete_playlist, get_playlists
)
from ..database.schema.playlist import (
    Playlist as PlaylistSchema, Playlists as PlaylistsSchema, GetPlaylist, GetPlaylists
)
from ..database.database import get_db
from http import HTTPStatus
from pydantic import ValidationError
from flask import request

#from ..decorators import admin_token_required

playlists = Blueprint("playlists", __name__)

@swag_from("./docs/add.yml", endpoint="playlists.add", methods=["POST"])
@playlists.route("/playlist", methods=["POST"])
def add():
    try:
        playlist_data = PlaylistSchema(**request.json)
    except ValidationError:
        return {'Error': 'Invalid playlist data!'}, HTTPStatus.BAD_REQUEST
    playlist = get_playlist(session=get_db, playlist_data=playlist_data)
    if playlist:
        return {'Error': f'playlist with id {playlist_data.playlist_id} already exists'}, HTTPStatus.CONFLICT
    playlist = create_playlist(playlist_data=playlist_data, session=get_db)
    resp = PlaylistSchema(
        playlist_description=playlist.playlist_description,
        playlist_id=playlist.playlist_id,
        playlist_title=playlist.playlist_title,
        playlist_thumbnail=playlist.playlist_thumbnail,
        published_at=playlist.published_at,
        privacy_status=playlist.privacy_status,
        videos_count=playlist.videos_count,
        channel_id=playlist.channel_id
    )

    return resp.model_dump_json(indent=4), HTTPStatus.CREATED

@swag_from("./docs/update.yml", endpoint="playlists.update", methods=["PUT"])
@playlists.route("/playlist", methods=["PUT"])
def update():
    return {'success':'playlist'}, HTTPStatus.CREATED

@swag_from("./docs/delete.yml", endpoint="playlists.delete", methods=["DELETE"])
@playlists.route("/playlist", methods=["DELETE"])
def delete():
    try:
        playlist_data = GetPlaylist(playlist_id=request.args.get('playlist_id'))
    except ValidationError:
        return {'error': 'Invalid input: you probably did not include the playlist id.'}, HTTPStatus.BAD_REQUEST
    playlist = get_playlist(session=get_db, playlist_data=playlist_data)
    if not playlist:
        return {'Error': f'playlist with id {playlist_data.playlist_id} does not exists'}, HTTPStatus.NOT_FOUND
    playlist = delete_playlist(get_db, GetPlaylist(playlist_id=request.args.get('playlist_id')))
    resp = PlaylistSchema(
        playlist_description=playlist.playlist_description,
        playlist_id=playlist.playlist_id,
        playlist_title=playlist.playlist_title,
        playlist_thumbnail=playlist.playlist_thumbnail,
        published_at=playlist.published_at,
        privacy_status=playlist.privacy_status,
        videos_count=playlist.videos_count,
        channel_id=playlist.channel_id
    )
    return resp.model_dump_json(indent=4), HTTPStatus.OK

@swag_from("./docs/get.yml", endpoint="playlists.get", methods=["GET"])
@playlists.route("/playlist", methods=["GET"])
def get():
    try:
        playlist_data = GetPlaylist(playlist_id=request.args.get('playlist_id'))
    except ValidationError:
        return {'error': 'Invalid input: you probably did not include the playlist id.'}, HTTPStatus.BAD_REQUEST
    playlist = get_playlist(session=get_db, playlist_data=playlist_data)
    if playlist:
        resp = PlaylistSchema(
            playlist_description=playlist.playlist_description,
            playlist_id=playlist.playlist_id,
            playlist_title=playlist.playlist_title,
            playlist_thumbnail=playlist.playlist_thumbnail,
            published_at=playlist.published_at,
            privacy_status=playlist.privacy_status,
            videos_count=playlist.videos_count,
            channel_id=playlist.channel_id
        )
        return resp.model_dump_json(indent=4), HTTPStatus.OK

    return {'Error':f'No channel with id {playlist_data.playlist_id}'}, HTTPStatus.NOT_FOUND

@playlists.route("/", methods=["POST"])
# @admin_token_required
@swag_from("./docs/add_many.yml", endpoint="playlists.add_many", methods=["POST"])
def add_many():
    try:
        playlists_data = [PlaylistSchema(**data) for data in request.json['playlists']]
        playlists = PlaylistsSchema(playlists=playlists_data)
    except ValidationError:
        return {'Error': 'Invalid playlist data!'}, HTTPStatus.BAD_REQUEST
    count: int = create_many(session=get_db, playlists_data=playlists)
    return {'Success':f'{count} playlist(s) added.'}, HTTPStatus.CREATED

@swag_from("./docs/playlists.yml", endpoint="playlists.list_all_playlists", methods=["GET"])
@playlists.route("/", methods=["GET"])
def list_all_playlists():
    offset: str = request.args.get('offset')
    limit: str = request.args.get('limit')
    playlists = get_playlists(get_db, GetPlaylists(offset=offset, limit=limit))
    playlists = [
        PlaylistSchema(
            playlist_description=playlist.playlist_description,
            playlist_id=playlist.playlist_id,
            playlist_title=playlist.playlist_title,
            playlist_thumbnail=playlist.playlist_thumbnail,
            published_at=playlist.published_at,
            privacy_status=playlist.privacy_status,
            videos_count=playlist.videos_count,
            channel_id=playlist.channel_id
        ).dict()
        for playlist in playlists
    ]
    return playlists, HTTPStatus.OK

@swag_from("./docs/playlist_videos.yml", endpoint="playlists.videos", methods=["GET"])
@playlists.route("/playlist/videos", methods=["GET"])
def videos():
    return {'success':'playlist'}, HTTPStatus.CREATED


#Uses python3
def reach(adj, x, y):
    #write your code here
    visited = [0] * len(adj)
    return explore(adj, x, y, visited)

def explore(adj, x, y, visited):
    if (x == y):
        return 1
    visited[x] = 1
    for i in range(len(adj[x])):
      if (not visited[adj[x][i]]):
          if(explore(adj, adj[x][i], y, visited)):
              return 1
    return 0

data = list(map(int, input().split()))
n, m = data
adj = [[] for _ in range(n)]
for _ in range(m):
    a,b=list(map(int, input().split()))
    adj[a-1].append(b-1)
    adj[b-1].append(a-1)

u,v=list(map(int, input().split()))
print(reach(adj, u-1, v-1))


#  The range function doesn't actually create a list of numbers;
#  it returns an iterator, which is a type of Python object specially designed to work with loops.
#  However, if we combine range with list, we get a list of numbers.
#  print(list(range(10, 20, 2))) returns [10, 12, 14, 16, 18]

#  You don’t need to stick to using the range and list functions when making for loops.
#  You could also use a list you’ve already created, such as the shopping list from Chapter 3, as follows:
wizard_list = ['spider legs',  'toe of frog',  'snail tongue', 'bat wing',  'slug butter',  'bear burp']
for i in wizard_list:
    print(i)
# prints;
# 'spider legs
# toe of frog
# snail tongue
# bat wing
# slug butter
# bear burp

huge_hairy_pants = ['huge', 'hairy', 'pants']
for i in huge_hairy_pants:
    print('> ' + i)
    for j in huge_hairy_pants:    # These lines are the FIRST block.
        print('>> ' + j)              #


#! /usr/bin/env python3

import cv2
import rospy
from cv_bridge import CvBridge
from sensor_msgs.msg import Image

rospy.init_node("Camera_Pub")
pub=rospy.Publisher("/Video",Image,queue_size=1)
# rate=rospy.Rate(10)

#OPENCV SCIRPTS
cap = cv2.VideoCapture("nvarguscamerasrc ! video/x-raw(memory:NVMM), width=(int)1280, height=(int)720, format=(string)NV12, framerate=(fraction)60/1 ! nvvidconv flip-method=0 ! video/x-raw, width=(int)1280, height=(int)720, format=(string)BGRx ! videoconvert ! video/x-raw, format=(string)BGR ! appsink", cv2.CAP_GSTREAMER)

print(cap.isOpened())

while not rospy.is_shutdown() :
    ret,img=cap.read()
    img_RGB = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    msg_frame = CvBridge().cv2_to_imgmsg(img_RGB)
    pub.publish(msg_frame)
    # rate.sleep()

cap.release()
cv2.destroyAllWindows()


from dlkp.generation import KGTrainingArguments, KGModelArguments, KGDataArguments
from dlkp.models import KeyphraseGenerator

model_args = KGModelArguments(model_name_or_path="bart-large")

data_args = KGDataArguments(
    dataset_name="midas/inspec",
    dataset_config_name="generation",
    text_column_name="document",
    keyphrases_column_name="extractive_keyphrases",
    n_best_size=10,
    num_beams=50,
    max_seq_length=512,
)

training_args = KGTrainingArguments(
    predict_with_generate=True,
    output_dir="/data/models/keyphrase/dlkp/inspec/generation/bart-large/finetuned",
    learning_rate=4e-5,
    overwrite_output_dir=True,
    num_train_epochs=100,
    per_device_train_batch_size=2,
    per_device_eval_batch_size=2,
    do_train=True,
    do_eval=True,
    do_predict=False,
    evaluation_strategy="steps",
    save_steps=500,
    eval_steps=500,
    logging_steps=500,
    metric_for_best_model="eval_F1@5",
    load_best_model_at_end=True,
)
gen = KeyphraseGenerator.train_and_eval(model_args, data_args, training_args)

gen = KeyphraseGenerator.load(
    "/data/models/keyphrase/dlkp/inspec/generation/bart-large/finetuned"
)

input_text = (
    "In this work, we explore how to learn task-specific language models aimed towards learning rich "
    "representation of keyphrases from text documents. We experiment with different masking strategies for "
    "pre-training transformer language models (LMs) in discriminative as well as generative settings. In the "
    "discriminative setting, we introduce a new pre-training objective - Keyphrase Boundary Infilling with "
    "Replacement (KBIR), showing large gains in performance (upto 9.26 points in F1) over SOTA, when LM "
    "pre-trained using KBIR is fine-tuned for the task of keyphrase extraction. In the generative setting, we "
    "introduce a new pre-training setup for BART - KeyBART, that reproduces the keyphrases related to the "
    "input text in the CatSeq format, instead of the denoised original input. This also led to gains in "
    "performance (upto 4.33 points in F1@M) over SOTA for keyphrase generation. Additionally, we also "
    "fine-tune the pre-trained language models on named entity recognition (NER), question answering (QA), "
    "relation extraction (RE), abstractive summarization and achieve comparable performance with that of the "
    "SOTA, showing that learning rich representation of keyphrases is indeed beneficial for many other "
    "fundamental NLP tasks."
)

gen_out = gen.generate(input_text, num_return_sequences=10, output_seq_score=True, num_beams=50)
print(gen_out)
print(set([kp.strip() for kp in gen_out[0][0].split("[KP_SEP]")]))


import time
import inspect
from pm.switcher import *
import pm.switcher as Switchers
import subprocess
import threading
from logging import *

class Command(object):
    def __init__(self, cmd):
        self.cmd = cmd
        self.process = None

    def run(self, timeout=10):
        def target():
            self.process = subprocess.Popen(self.cmd, shell=True, stdout=subprocess.PIPE)
            self.process.communicate()

        thread = threading.Thread(target=target)
        thread.start()

        thread.join(timeout)
        if thread.is_alive():
            self.process.terminate()
            thread.join()

        self.process.returncode

class PowerManager(object):
    _delay = 1
    _config = None
    _upower = None
    _logger = None

    def __init__(self, config=None, upower=None):
        self._config = config
        self._upower = upower
        self._logger = getLogger('power-manager')

        self._switchers = []
        for (name, module) in inspect.getmembers(Switchers, inspect.ismodule):
            if hasattr(module, name):
                identifier = getattr(module, name)
                with identifier() as switcher:
                    self._logger.debug('loaded: %s' % switcher.name)
                    self._switchers.append(switcher)

        self._config.modules = [module.name for module in self.switchers]

    @property
    def switchers(self):
        for switcher in self._switchers:
            if switcher.exists:
                yield switcher

    @property
    def managed(self):
        for switcher in self.switchers:
            if not self._config.ignored(switcher.name):
                yield switcher

    @property
    def config(self):
        return self._config

    def on_optimize(self, event, dispatcher):
        time.sleep(self._delay)
        if self._upower.is_battery:
            self._logger.info('optimize for battery')
            return self.on_powersafe(event, dispatcher)
        self._logger.info('optimize for ac')
        return self.on_perfomance(event, dispatcher)

    def on_powersafe(self, event, dispatcher):
        for switcher in self.managed:
            self._logger.debug('powersafe: %s' % switcher.name)
            for command in switcher.powersave():
                self._logger.debug('powersafe command: %s' % command)
                self._execute(command)

    def on_perfomance(self, event, dispatcher):
        for switcher in self.managed:
            self._logger.debug('perfomance: %s' % switcher.name)
            for command in switcher.perfomance():
                self._logger.debug('perfomance command: %s' % command)
                self._execute(command)

    @staticmethod
    def _execute(command):
        if command is not None:
            process = Command(command)
            process.run()

class PowerManagerService(object):
    _manager = None

    def __init__(self, config=None, dispatcher=None, upower=None):
        self._manager = PowerManager(config, upower)

        dispatcher.add_listener('power_manager.optimize', self._manager.on_optimize)
        dispatcher.add_listener('power_manager.powersafe', self._manager.on_powersafe)
        dispatcher.add_listener('power_manager.perfomance', self._manager.on_perfomance)
        dispatcher.add_listener('network_manager.optimize', self._manager.on_optimize)

    @property
    def switchers(self):
        return self._manager.switchers

    @property
    def managed(self):
        return self._manager.managed


n = int(input())
line = input()
nums = list(map(int, line.strip().split()))
#print(n)
#print(nums)
A, B = 0, 0
for item in nums:
    if item >= 0:
        A += item
    else:
        B += item
print(A - B)


from tkinter import *
from PIL import Image
import sys
from scapy.all import *

from globals import ALPHABET

BG_GRAY = "#ABB2B9"
BG_COLOR = "#17202A"
TEXT_COLOR = "#EAECEE"
BG_FOOTER = "#222"
BG_BUTTON = "#FFA630"

#   --secondary-color: #ffa630;
#   --side-nav-background: #eee;
#   --footer-background: #222;

FONT = "Helvetica 12"
FONT_BOLD = "Helvetica 13 bold"

class ChatApplication:

    def __init__(self, window, snmpConn):
        # self.window = Tk()

        self.window = window
        self.window.protocol("WM_DELETE_WINDOW", self.closeConnection)
        self.snmpConn = snmpConn

        self._setup_main_window()

    def run(self):
        self.window.mainloop()

    def _setup_main_window(self):
        self.window.title("Chat")
        self.window.resizable(width=False, height=False)
        self.window.configure(width=470, height=550, bg=BG_FOOTER)

        # head label
        head_label = Label(self.window, bg=BG_FOOTER, fg=TEXT_COLOR,
                            text="Covert Channel Chat - SNMP", font=FONT_BOLD, pady=10, padx=2)
        head_label.place(relwidth=1)

        # divider
        line = Label(self.window, width=450, bg=BG_GRAY)
        line.place(rely=0.07, relx=0.025, relheight=0.0001 ,relwidth=0.95)

        # text widget
        self.text_widget = Text(self.window, width=20, height=2, bg=BG_COLOR, fg=TEXT_COLOR,
                                font=FONT, padx=5, pady=5)
        self.text_widget.place(relheight=0.745, relwidth=0.95, rely=0.08, relx=0.025)
        self.text_widget.configure(cursor="arrow", state=DISABLED)

        # scroll bar
        scrollbar = Scrollbar(self.text_widget)
        scrollbar.place(relheight=1, relx=0.974)
        scrollbar.configure(command=self.text_widget.yview)

        # # bottom label
        bottom_label = Label(self.window, bg=BG_FOOTER, height=80)
        bottom_label.place(relwidth=0.95, relx=0.025, rely=0.825)

        # message entry box
        self.msg_entry = Entry(bottom_label, bg="#2C3E50", fg=TEXT_COLOR, font=FONT)
        self.msg_entry.place(relwidth=0.74, relheight=0.04, rely=0.008)
        self.msg_entry.focus()
        self.msg_entry.bind("<Return>", self._on_enter_pressed)

        # send button
        send_button = Button(bottom_label, text="Send", font=FONT_BOLD, width=20, bg=BG_BUTTON,
                            command=lambda: self._on_enter_pressed(None))
        send_button.place(relx=0.76, rely=0.008, relheight=0.04, relwidth=0.24)

    def validCharacters(self, text):
        if text.strip() == "q":
            return False
        for character in text:
            if character not in ALPHABET:
                return False
        return True

    def _on_enter_pressed(self, event):
        msg = self.msg_entry.get()
        if msg.strip() and self.validCharacters(msg):
            self.snmpConn.sendMsg(msg)
        self._insert_message(msg, "You")

    def _insert_message(self, msg, sender):
        if not msg:
            return

        self.msg_entry.delete(0, END)
        msg1 = f"{sender}: {msg}\n\n"
        self.text_widget.configure(state=NORMAL)
        self.text_widget.insert(END, msg1)
        self.text_widget.configure(state=DISABLED)

        self.text_widget.see(END)

    # This func is called when the CLOSE button is clicked.
    def closeConnection(self):
        print("[-] Covert Channel Chat has ended.")
        self.snmpConn.sendMsg("q")
        self.window.quit()
        sys.exit(0)

    # This func is called when the SEND button is clicked.
    # def sendClicked(self):
    #     textToSend = self.messageContainer.get("1.0",END)
    #     if textToSend.strip() and self.validCharacters(textToSend):
    #         self.messageContainer.delete('1.0', END)
    #         self.chatContainer.configure(state='normal')
    #         self.chatContainer.insert(END, " > Tu: ","bold")
    #         self.chatContainer.insert(END, textToSend)
    #         self.chatContainer.configure(state=DISABLED)
    #         self.chatContainer.see(END)
    #         self.snmpConn.sendMsg(textToSend)

if __name__ == "__main__":
    app = ChatApplication()
    app.run()


# config.pretrained.py
import torch
import torchvision
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor

# 7 class + background
def get_finetuned_model(model=None, num_classes=2, pre_path=None, weight_path=None, mask=False):
    if model:
        if weight_path:
            model.load_state_dict(torch.load(weight_path))

    else:
        if pre_path:
            model = torch.load('/home/agc2021/assets/fasterrcnn_resnet50_fpn_coco.pth')
        elif pre_path=='init':
            model = torchvision.models.detection.fasterrcnn_resnet50_fpn()
        else:
            model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)

        if weight_path:
            model.load_state_dict(model.load(weight_path))
    in_features = model.roi_heads.box_predictor.cls_score.in_features   # 1024
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)

    if mask:
        in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
        hidden_layer = 256
        model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask, hidden_layer, num_classes)
    return model