# Miguel Delapaz - CS594 - IRC Server Project
import socket
import select
import sys
from enum import Enum
try:
    import queue
except ImportError:
    import Queue as queue

class Command(Enum):
    LOGIN = 1
    LOGOUT = 2
    ADD_CHANNEL = 3
    JOIN_CHANNEL = 4
    LEAVE_CHANNEL = 5
    LIST_ROOMS = 6
    LIST_USERS = 7
    MESSAGE = 8

class ResponseCodes(Enum):
    OK = 0
    ERROR = 4

class Server:
    def __init__(self):
        self.channelList = {}
        self.clientList = {}
        self.port = 6000
        self.readList = []
        self.writeList = []
        self.initialize_listen_socket()

    def initialize_listen_socket(self):
        self.listenSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.listenSocket.setblocking(False)
        self.listenSocket.bind(('', self.port))
        self.listenSocket.listen(5)
        self.readList.append(self.listenSocket)
        print("Server listening socket created on port " + str(self.port))

    def process_add_channel(self, client, channelName):
        if len(channelName) > 10 or not channelName.isalnum():
            self.response(client, Command.ADD_CHANNEL, ResponseCodes.ERROR, "Invalid Channel - " + channelName + " - Must be alphanumeric and less than 10 characters")
        else:
            if channelName in self.channelList:
                self.send_response(client, Command.ADD_CHANNEL, ResponseCodes.ERROR, "Channel " + channelName + " already exists")
            else:
                newChan = Channel(channelName)
                self.channelList[channelName] = newChan
                self.send_response(client, Command.ADD_CHANNEL, ResponseCodes.OK, channelName)
                self.process_join_channel(client, channelName)

    def process_join_channel(self, client, channelName):
        if not channelName in self.channelList:
            self.send_response(client, Command.JOIN_CHANNEL, ResponseCodes.ERROR, "Channel " + channelName + " does not exist")
        else:
            channel = self.channelList[channelName]
            if client in channel.users:
                self.send_response(client, Command.JOIN_CHANNEL, ResponseCodes.ERROR, "Already in the channel " + channelName)
            else:
                channel.users.append(client)
                self.send_response(client, Command.JOIN_CHANNEL, ResponseCodes.OK, channelName)

    def process_leave_channel(self, client, channelName):
        if not channelName in self.channelList:
            self.send_response(client, Command.LEAVE_CHANNEL, ResponseCodes.ERROR, "Channel " + channelName + " does not exist")
        else:
            channel = self.channelList[channelName]
            if not client in channel.users:
                self.send_response(client, Command.LEAVE_CHANNEL, ResponseCodes.ERROR, "Not in the channel " + channelName)
            else:
                channel.users.remove(client)
                self.send_response(client, Command.LEAVE_CHANNEL, ResponseCodes.OK, channelName)

                # Remove channel if it is empty
                if not channel.users:
                    del(self.channelList[channelName])

    def process_list_rooms(self, client):
        response = ''
        for c in self.channelList:
            channel = self.channelList[c]
            response += channel.name + '\n'
        self.send_response(client, Command.LIST_ROOMS, ResponseCodes.OK, response)

    def process_list_participants(self, client, channelName):
        response = channelName + '\n'
        if not channelName in self.channelList:
            self.send_response(client, Command.LIST_USERS, ResponseCodes.ERROR, "Channel " + channelName + " does not exist")
        else:
            channel = self.channelList[channelName]
            for u in channel.users:
                response += u.name + '\n'
            self.send_response(client, Command.LIST_USERS, ResponseCodes.OK, response)

    def process_message_channel(self, client, data):
        channelName, message = data.split('\n', 1)
        if not channelName in self.channelList:
            self.send_response(client, Command.MESSAGE, ResponseCodes.ERROR, "Channel " + channelName + " does not exist")
        elif message == '':
            self.send_response(client, Command.MESSAGE, ResponseCodes.ERROR, "Cannot send empty message")
        else:
            channel = self.channelList[channelName]
            response = channelName + '\n' + client.name + '\n' + message
            for c in channel.users:
                self.send_response(c, Command.MESSAGE, ResponseCodes.OK, response)

    def process_leave_server(self, client):
        channelsToRemove = []
        for c in self.channelList:
            channel = self.channelList[c]
            if client in channel.users:
                channel.users.remove(client)
                # Remove channel if it is empty
                if not channel.users:
                    channelsToRemove.append(c)
        for c in channelsToRemove:
            del(self.channelList[c])
        if client.sock in self.writeList:
            self.writeList.remove(client.sock)
        self.readList.remove(client.sock)
        client.sock.close()
        del self.clientList[client.sock]

    def process_join_server(self, client, username):
        if len(username) > 10 or not username.isalnum():
            self.send_response(client, ResponseCodes.ERROR, "Invalid Username - Must be alphanumeric and less than 10 characters")
            return ResponseCodes.ERROR
        client.name = username
        client.LoggedIn = True
        self.send_response(client, Command.LOGIN, ResponseCodes.OK, None)

    def process_incoming_data(self, socket):
        client = self.clientList[socket]
        try:
            command = socket.recv(1)
        except:
            print("Client at " + str(client.addr) + " encounterd an error")
            self.process_leave_server(client)
            return

        if not command:
            # Client disconnected
            print("Client at " + str(client.addr) + " disconnected")
            self.process_leave_server(client)
            return
        else:
            try:
                command = int(command)
                length = int(socket.recv(5))
                if length > 0:
                    data = socket.recv(length)
            except:
                print("Client at " + str(client.addr) + " encounterd an error")
                self.process_leave_server(client)
                return

        if command == Command.LOGIN:
            self.process_join_server(client, data)
        elif not client.LoggedIn:
            send_response(client, command, ResponseCodes.ERROR, "Command not valid before login")
        elif command == Command.LOGOUT:
            self.process_leave_server(client)
        elif command == Command.ADD_CHANNEL:
            self.process_add_channel(client, data)
        elif command == Command.JOIN_CHANNEL:
            self.process_join_channel(client, data)
        elif command == Command.LEAVE_CHANNEL:
            self.process_leave_channel(client, data)
        elif command == Command.LIST_ROOMS:
            self.process_list_rooms(client)
        elif command == Command.LIST_USERS:
            self.process_list_participants(client, data)
        elif command == Command.MESSAGE:
            self.process_message_channel(client, data)
        elif command == Command.RESPONSE:
            # Client shouldn't be sending response messages
            send_response(client, command, ResponseCodes.ERROR, "Response messages are reserved for the server")
        else:
            # We don't know what this code is
            send_response(client, command, ResponseCodes.ERROR, "Unrecognized command code")

    def process_incoming_connection(self):
        c, addr = self.listenSocket.accept()
        c.setblocking(False)
        print("Received connection from client at " + str(addr))
        self.readList.append(c)
        server.clientList[c] = Client(c, addr)

    def process_client_exception(self, sock):
        client = self.clientList[socket]
        print("Client at " + client.addr + " encountered socket exception - closing")
        self.process_leave_server(client)

    def send_response(self, client, commandCode, responseCode, message):
        # Packet is one byte of command code, 5 bytes of length (string form)
        # followed by the data
        if message and len(message) >= 99999:
            print("Message too long")
        else:
            if message:
                length = len(str(responseCode)) + len(message)
            else:
                length = len(str(responseCode))
            data = str(commandCode) + str(length).rjust(5, '0') + str(responseCode)
            if message:
                data = data + message
            client.outbound.put(data)
            self.writeList.append(client.sock)

    def run(self):
        while self.readList:
            read, write, exception = select.select(self.readList, self.writeList, self.readList)

            for s in read:
                if s == self.listenSocket:
                    # A client is trying to connect
                    self.process_incoming_connection()
                else:
                    # Data coming in on a client socket
                    self.process_incoming_data(s)

            for s in write:
                if s in self.clientList:
                    client = self.clientList[s]
                    client.send_outgoing_data()
                    self.writeList.remove(s)

            for s in exception:
                if s == self.listenSocket:
                    print("Server listen socket encountered exception - reinitializing")
                    self.readList.remove(self.listenSocket)
                    self.listenSocket.close()
                    self.initialize_listen_socket()
                else:
                    self.process_client_exception(s)

class Channel:
    def __init__(self, name):
        self.name = name
        self.users = []

class Client:
    def __init__(self, socket, addr):
        self.name = None
        self.sock = socket
        self.addr = addr
        self.LoggedIn = False
        self.outbound = queue.Queue()

    def send_outgoing_data(self):
        while not self.outbound.empty():
            self.sock.send(self.outbound.get())

# Start of main server program
server = Server()

server.run()


from twisted.web import server
from .base import BaseResource

class DataResource(BaseResource):

    def __init__(self, dataserver):
        self.dataserver = dataserver
        BaseResource.__init__(self)

    def render(self, request):
        request.setHeader('Content-type', 'text/javascript; charset=UTF-8')
        if "uuid" in request.args:
            get_deferreds = []
            for uuid in request.args["uuid"]:
                get_deferreds.append(self.dataserver.getData(uuid))
            d = DeferredList(get_deferreds, consumeErrors = True)
            d.addCallback(self._getCallback, request.args["uuid"])
            d.addCallback(self._successResponse)
            d.addErrback(self._errorResponse)
            d.addCallback(self._immediateResponse, request)
            return server.NOT_DONE_YET
        return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))

    def _getCallback(self, data, uuids):
        response = {}
        for i in range(0, len(uuids)):
            if data[i][0] == True:
                response[uuids[i]] = data[i][1]
            else:
                response[uuids[i]] = {"error":str(data[i][1].value)}
        return response


import os
import lxml
import shutil
import requests as re
import astropy.units as u
from astroquery.vizier import Vizier
from astropy.io import fits
from astropy.coordinates import SkyCoord
from bs4 import BeautifulSoup

# Create the download file directory.
data_dir = os.path.expanduser('~/astro_data/')
if not os.path.exists(data_dir):
    os.mkdir(data_dir)

# Create the data dictionary
dic_data = {
    'dss_image': {
        'format': 'fits',
        'type': 'image',
        'filename': 'dss_image.fits',
        'description': 'It will get the DSS image, RA: 01 02 03.3, DEC: +00 01 02.2, height & width: 30 arcmin'
    },
    'm4': {
        'format': 'fits',
        'type': 'catelog',
        'filename': 'm4_tmc.fits',
        'description': 'It will get m4 '
    }
}

def get_dss_image():
    '''
    example of DSS URL
    https://archive.stsci.edu/cgi-bin/dss_search?
        v=poss2ukstu_red&   => type of data (poss2, red for instance)
        r=01%3A02%3A03.3&   => ra in hex decimal
        d=%2B00%3A01%3A02.2& => dec in hex decimal
        e=J2000&             => epoch (J2000 for instance)
        h=30&                => height of the image (30 arcmin)
        w=30&                => width of the image (30 arcmin)
        f=fits&              => types of the image (fits or gif)
        c=none&              => compress
        s=on&
        fov=NONE&
        v3=
    '''
    ## Download digital sky survey fits.
    urlroot = 'https://archive.stsci.edu/cgi-bin/dss_search?'
    v = 'poss2ukstu_red'
    e = 'J2000'
    ra = str(1) + '%3A' + str(2) + '%3A' + str(03.3)
    de = '%2B' + str(0) + '%3A' + str(1) + '%3A' + str(02.2)
    h = str(10)
    w = str(10)
    f = 'fits'
    urlend = '&c=none&s=on&fov=NONE&v3='
    result = re.get(urlroot+'v='+v+'&r='+ra+'&d='+de+'&e='+e+'&h='+h+'&w='+w+'&f='+f+urlend, stream=True)

    with open(data_dir + 'dss_image.fits', 'wb') as out_file:
        shutil.copyfileobj(result.raw, out_file)
    del result

def get_m4():
    ## Query on vizier and save to FITS bin table
    # Query setup
    Vizier.ROW_LIMIT = -1
    target = 'M4' # Another name: M4
    r = 0.01

    # Making query
    # 2MASS catalog
    try:
        tmc = Vizier.query_region(target, radius = r*u.deg, catalog = 'II/246/out')
    except:
        payloads = {'-r': (None, 'SNV'), '-name': (None, target)}
        results = re.post('http://cds.u-strasbg.fr/cgi-bin/Sesame', data = payloads)
        soup = BeautifulSoup(results.text, 'lxml')
        pos = SkyCoord(soup.find_all('pre')[0].string.split('±')[0], frame='icrs', unit=(u.hourangle, u.deg))
        tmc = Vizier.query_region(pos, radius=r*u.deg, catalog='II/246/out')

    # PPMXL proper motion
    #ppmxl = Vizier.query_region(target, radius=r*u.deg, catalog='I/317/sample')

    # Get catalog only
    jhk = tmc[0]
    #pm = ppmxl[0]

    # Retrive data only from 2MASS columns [3,4,9,11,13,15,17,19,21]
    col1 = fits.Column(name=jhk['Jmag'].name, format=jhk['Jmag'].dtype, array=jhk['Jmag'])
    col2 = fits.Column(name=jhk['e_Jmag'].name, format=jhk['e_Jmag'].dtype, array=jhk['e_Jmag'])
    col3 = fits.Column(name=jhk['Hmag'].name, format=jhk['Hmag'].dtype, array=jhk['Hmag'])
    col4 = fits.Column(name=jhk['e_Hmag'].name, format=jhk['e_Hmag'].dtype, array=jhk['e_Hmag'])
    col5 = fits.Column(name=jhk['Kmag'].name, format=jhk['Kmag'].dtype, array=jhk['Kmag'])
    col6 = fits.Column(name=jhk['e_Kmag'].name, format=jhk['e_Kmag'].dtype, array=jhk['e_Kmag'])
    col7 = fits.Column(name=jhk['Qflg'].name, format=jhk['Qflg'].dtype, array=jhk['Qflg'])
    cols = fits.ColDefs([col1, col2, col3, col4, col5, col6, col7])
    tmc_hdu = fits.BinTableHDU.from_columns(cols)
    tmc_hdu.writeto(data_dir + target + '_tmc.fits', overwrite=True)


from AutoSequencerV2.composer import Composer
from AutoSequencerV2.runnable import Runnable

class SequentialCommandGroup(Runnable, Composer):

    def __init__(self, cmdList=None):
        self.cmdList = cmdList if cmdList else []
        self._curCmdIdx = 0

    def execute(self):
        if(self._curCmdIdx < len(self.cmdList)):
            # If we've got a valid command, execute it
            curCmd = self.cmdList[self._curCmdIdx]
            curCmd.execute()

            if(curCmd.isDone()):
                # Time to move on to the next command
                # First, Naturally end the current command
                print(f"[Auto] Ending {curCmd.getName()}")
                curCmd.end(False)
                # Move onto the next command
                self._curCmdIdx += 1
                # Init it if it exists
                if(self._curCmdIdx < len(self.cmdList)):
                    curCmd = self.cmdList[self._curCmdIdx]
                    print(f"[Auto] Starting {curCmd.getName()}")
                    curCmd.initialize()
                    # That's it, next loop we'll execute it.

    # Default group init - just init each command
    def initialize(self):
        self._curCmdIdx = 0
        if(self._curCmdIdx < len(self.cmdList)):
            # Init the first command
            curCmd = self.cmdList[self._curCmdIdx]
            print(f"[Auto] Starting {curCmd.getName()}")
            curCmd.initialize()

    # Default group end - end everything with same interrupted status
    def end(self, interrupted):
        # Only need to end the current command
        if(self._curCmdIdx < len(self.cmdList)):
            curCmd = self.cmdList[self._curCmdIdx]
            print(f"[Auto] Ending {curCmd.getName()}")
            curCmd.end(interrupted)

    def isDone(self):
        # We're done when we hit the end of the list
        return self._curCmdIdx >= len(self.cmdList)

    ##################################################
    ## composition handlers

    def andThen(self, other):
        self.cmdList.append(other)
        return self


#-----------------------------------------------------------------------------
# Name:        actors with images example
# Purpose:     And Example file demoing actors
#
# Author:      Mr. Brooks
# Created:     01-Oct-2020
# Updated:     01-Oct-2020
#-----------------------------------------------------------------------------
WIDTH = 200
HEIGHT = 200

gameState = ''

def on_key_up(key):
    '''Check to see if a key has been released'''
    global gameState

    if key == keys.A:
        gameState = 'initializeStartScreen'

    elif key == keys.B:
        gameState = 'initializeGameScreen'

def update():
    global gameState
    #music.play('handel_mp3')   #This is a bad idea!
    if gameState == 'initializeStartScreen':
        music.play('handel_mp3')    #Play the song over and over
        gameState = 'startScreen'
    if gameState == 'StartScreen':
        pass
    elif gameState == 'initializeGameScreen':
        music.play_once('birthday') #https://freemusicarchive.org/music/Monk_Turner__Fascinoma/The_New_Birthday_Song_Contest/Its_Your_Birthday_1839
        gameState = 'gameScreen'

def on_music_end():
    music.play_once('one') #You could make this pick a random song from a list for example
    #https://freemusicarchive.org/music/Doctor_Turtle/5f91e09024ca8/one-person-listening-now

def draw():
    '''Draw loop for all the graphical elements to display'''
    #Empty the screen for each animation frame
    if gameState == 'startScreen':
        screen.fill((255, 255, 255))
    #Draw the knight


"""
# Definition for Employee.
class Employee:
    def __init__(self, id: int, importance: int, subordinates: List[int]):
        self.id = id
        self.importance = importance
        self.subordinates = subordinates
"""
from collections import defaultdict
class Solution:
    def getImportance(self, employees: List['Employee'], id: int) -> int:

        adj = {}
        for emp in employees:
            adj[emp.id] = emp

        total = 0
        def dfs(node):
            nonlocal total

            if not node:
                return

            total += node.importance
            for id in node.subordinates:
                dfs(adj[id])

        dfs(adj[id])
        return total


import turtle
from turtle import *
turtle.speed(100)

#def draw_circle(x,y,size):
 #   turtle.penup()
  #  turtle.goto(x,y)
#def fun(x,y):
    #turtle.pendown()
    #t>>> turtle.fillcolor("violet")
#turtle.fillcolor()
#'violet'
#col = turtle.pencolor()
#col(50.0, 193.0, 143.0)
 #turtle.fillcolor(col)
 #turtle.fillcolor()
#(50.0, 193.0, 143.0)
 #turtle.fillcolor('#ffffff')
 #turtle.fillcolor()
#(255.0, 255.0, 255.0)urtle.goto(x,y)

#turtle.onclick(fun(10,10),btn=2,add = True)f
#turtle.onscreenclick(fun,btn=  1,add = True)
#turtle.ondrag(fun,btn =1 ,add = True)
#turtle.onclick(fun(10,10),btn=1,add = True)
color = 'red'
shape = 'circle'
def square(x,y):
    global color
    turtle.penup()
    turtle.goto(0+x,0+y)
    turtle.pendown()
    begin_fill()
    fillcolor(color)
    turtle.forward(50)
    turtle.right(90)
    turtle.forward(50)
    turtle.right(90)
    turtle.forward(50)
    turtle.right(90)

    turtle.forward(50)
    end_fill()
    turtle.right(90)
def circle (x,y):
    global color
    turtle.penup()
    turtle.goto(x,y)
    turtle.pendown()
    begin_fill()
    fillcolor(color)
    turtle.circle(10)
    end_fill()
#turtle.onscreenclick(square,btn = 1 ,add = True)
#turtle.ondrag(square,btn=1,add= True)
#flag = 1
#def flagg():
   #  flag = 2

#turtle.getscreen().onkeypress(flagg,"c")
#turtle.getscreen().listen()
#if flag == 2 :
 #   turtle.onscreenclick(square,btn =1 ,add = True)
#def draw ():
 #    turtle.forward(50)

#turtle.getscreen().onkeypress(draw,"c")

#turtle.getscreen.listen()
#color = ["red","blue","black"]
#r=0
#def count(n):
 #   global r++
#def color_list(r):
 #   color[r]

#color_list(0)

#count(r)
#color_list(r)
def draw(x,y):
    if shape == 'circle':
        circle(x,y)
    elif shape == 'square':
        square(x,y)

def choose_circle():
     global shape
     shape = 'circle'
def choose_square():
    global shape
    shape = 'square'
turtle.getscreen().onkeypress(choose_circle,"c")
turtle.getscreen().onkeypress(choose_square,"s")
turtle.onscreenclick(draw)

def change_blue():
    global color
    color = 'blue'
def change_red():
    global color
    color = 'red'
turtle.getscreen().onkeypress(change_red,"r")
turtle.getscreen().onkeypress(change_blue,"b")

listen()
turtle.mainloop()


import os
import json

def get_fields_names(fdf_file):
    with open(fdf_file, "r") as f:
        content = f.read()

    field_names = []

    start = 0
    offset = len("/T (")
    while not content.find("/T", start) == -1:
        field_start = content.find("/T", start)
        field_end = content.find(")", field_start)
        start = field_end
        field_name = content[field_start + offset:field_end]
        field_names.append(field_name)

    return field_names

def write_sorted_json_dict_file(file_name, keys):
    """
    Previously this was used but it is not in the order of the form
      with open("fvmf.json", "w") as f:
        json.dump(field_names_dict, f, indent=4, sort_keys=True)
    """
    file_data = "{\n"
    for key in keys[:-1]:
        file_data += "    \"" + str(key) + "\": \"\",\n"

    # Handle the last line here since json does not allow a "," on the last element
    file_data += "    \"" + str(keys[len(keys)-1]) + "\": \"\"\n"
    file_data += "}\n"
    with open(file_name, "w") as f:
        f.write(file_data)

class Field_Value_Mapping:

    def __init__(self, pdf_name, fvmf=None):
        """
        Creates a field value mapping file that can be manually filled in. This file is stored as a json dictionary.
        """
        self.pdf_name = pdf_name

        if fvmf is None:
            os.system("pdftk " + pdf_name + " generate_fdf " + "output " + "fvmf.txt")
            field_names = get_fields_names("fvmf.txt")

            write_sorted_json_dict_file("fvmf.json", field_names)
            self.fvmf = "fvmf.json"
            os.remove("fvmf.txt")
        else:
            self.fvmf = fvmf

    def create_mapped_fdf_file(self, data_file):
        """
        Given a datafile and the class's field value mapping file creates a mapped fdf file

        """
        # get the name of the fdf file
        file_extension = data_file.find(".")
        fdf_file_name = data_file[:file_extension] + "_mapped.fdf"

        # create a fdf file to overwrite with the data
        os.system("pdftk " + self.pdf_name + " generate_fdf " + "output " + fdf_file_name)

        # create mapped json values file
        mapped_values_file = self.map_values(data_file)

        # create a mapped fdf file
        self.populate_fdf_file(mapped_values_file, fdf_file_name)

    def map_values(self, data_file):
        """
        Creates a mapped json value tuples file. Uses the name of the datafile to create mapped
        values with unique names.
        """
        with open("fvmf.json", "r") as fmvf:
            contents = json.load(fmvf)
        with open(data_file, "r") as df:
            data = json.load(df)

        for field in contents.keys():
            for attribute in data.keys():
                if contents[field] == attribute:
                    contents[field] = data[attribute]

        contents = self.shrink_dict_(contents)

        file_extension = data_file.find(".")
        mapped_values_file = data_file[:file_extension] + "_mapped.json"
        with open(mapped_values_file, "w") as fmvf:
            json.dump(contents, fmvf, indent=4)

        return mapped_values_file

    def shrink_dict_(self, sparse_dict):
        """
        Creates a new dict that has all the keys whose corresponding values
        are not the empty string.
        """
        new_dict = {}
        for key in sparse_dict.keys():
            if sparse_dict[key] != "":
                print(sparse_dict[key])
                new_dict[key] = sparse_dict[key]
        return new_dict

    def remove_empty_fdf_fields(self, fdf_file, fvmf_dict):
        with open(fdf_file, "r") as fdf:
            fdf_str = fdf.read()
        field_names = get_fields_names(fdf_file)
        for field in field_names:
            if not field in fvmf_dict.keys():
                #print(fdf_str)
                #print("========================================")
                fdf_str = self.remove_fdf_field(fdf_str, field)
                #print(fdf_str)

        print(fdf_str)
        with open(fdf_file, "w") as fdf_f:
            fdf_f.write(fdf_str)

    def remove_fdf_field(self, fdf_str, field_name):
        """
        Takes the string contents of an fdf file and returns the
        string contents with the specified field removed.
        """
        index = fdf_str.find(field_name)
        print(field_name, index)
        if index == -1:
            print("error field_name not found and not removed")

        # Find and the next >>
        close_index = fdf_str.find(">>", index)
        close_index += len(">>")
        """
        Consider adjusting this
        """
        if fdf_str[close_index] != "]":
            close_index += 2

        # Find the previous <<
        open_index = fdf_str.rfind("<<", 0, index)

        # Remove the field

        if self.is_form_field(fdf_str, open_index, close_index):
            fdf_start = fdf_str[:open_index]

            fdf_end = fdf_str[close_index:]
            print("=============")
            print(fdf_str[open_index:close_index])
            fdf_str = fdf_start + fdf_end
        else:
            print(field_name, "is not a field and was not removed")

        return fdf_str

    def is_form_field(self, fdf_str, start, end):
        """
        This method checks if the item has more than one /T
        This is so that subforms are not attempted to be removed like fields.
        """
        count = 0
        ind = fdf_str.find("/T", start, end)
        ind = fdf_str.find("/T", ind+1, end)

        return -1 == ind

    def populate_fdf_file(self, mapped_values_file, fdf_file):
        """
        This function uses the mapped values file to populate the fdf file so that
        it can be used to populate the pdf
        """
        with open(mapped_values_file, "r") as mapped:
            mapped_values = json.load(mapped)

        self.remove_empty_fdf_fields(fdf_file, mapped_values)

        with open(fdf_file, "r") as fdf_f:
            fdf = fdf_f.read()

        start = 0
        offset = len("/V (")
        while not fdf.find("/V", start) == -1:
            value_start = fdf.find("/V", start)
            value_end = fdf.find("/T", value_start)
            start = value_end

            # get the name of the field we are currently replacing
            field_end = fdf.find(")", value_end)
            field_name = fdf[value_end + len("/V ("): field_end]

            # get the replacement value
            value = mapped_values[field_name] + ")\n"

            # insert the value into the string
            first_content = fdf[:value_start + len("/V (")]
            last_content = fdf[value_end:]
            fdf = first_content + str(value) + last_content

        with open("test.fdf", "wb") as f:
            f.write(fdf)


# Django imports.
from django.db import models

__author__ = 'Jason Parent'

class Task(models.Model):

    PENDING = 'PENDING'
    SUCCESS = 'SUCCESS'
    FAILURE = 'FAILURE'

    STATUS_CHOICES = (
        (PENDING, PENDING),
        (SUCCESS, SUCCESS),
        (FAILURE, FAILURE),
    )

    objects = models.Manager()

    status = models.CharField(max_length=7, choices=STATUS_CHOICES, default=PENDING)
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)

    class Meta:
        ordering = ('id',)


import requests
import cv2
import cmath
import numpy as np
import time
from globals import *
from cam import tracker
import tlm

def set_resolution(url: str, index: int=1, verbose: bool=False):
    try:
        if verbose:
            resolutions = "10: UXGA(1600x1200)\n9: SXGA(1280x1024)\n8: XGA(1024x768)\n7: SVGA(800x600)\n6: VGA(640x480)\n5: CIF(400x296)\n4: QVGA(320x240)\n3: HQVGA(240x176)\n0: QQVGA(160x120)"
            print("available resolutions\n{}".format(resolutions))

        if index in [10, 9, 8, 7, 6, 5, 4, 3, 0]:
            requests.get(url + "/control?var=framesize&val={}".format(index))
        else:
            print("Wrong index")
    except:
        print("SET_RESOLUTION: something went wrong")

def set_quality(url: str, value: int=1, verbose: bool=False):
    try:
        if value >= 10 and value <=63:
            requests.get(url + "/control?var=quality&val={}".format(value))
    except:
        print("SET_QUALITY: something went wrong")

def set_awb(url: str, awb: int=1):
    try:
        awb = not awb
        requests.get(url + "/control?var=awb&val={}".format(1 if awb else 0))
    except:
        print("SET_QUALITY: something went wrong")
    return awb

def setTarget(tracker, frame):
    bbox = cv2.selectROI("Tracking", frame, False, True)
    ok = tracker.init(frame, bbox)
    return True

def rotate(frame, roll):
    (h, w) = frame.shape[:2]
    (cX, cY) = (w//2, h//2)
    M = cv2.getRotationMatrix2D((cX,cY), -roll * 180.0/cmath.pi, 1.0)
    return cv2.warpAffine(frame,M,(w,h))

def translate(frame, psi, theta):
    (h, w) = frame.shape[:2]
    x = psi*57.0/65.0*w
    y = theta*57.0/(65.0*3.0/4.0)*h
    M = np.array([
        [1,0,x],
        [0,1,y]
    ], dtype=np.float32)
    return cv2.warpAffine(frame,M,(w,h))
class stabilization:
    def __init__(self):
        self.time_trans = time.time()
        self.psi_trans = 0.0
        self.theta_trans = 0.0

    def stabilize(self, frame, ahrs, coeff):
        now = time.time()
        ts = now - self.time_trans
        self.time_trans = now
        self.psi_trans += ahrs.R*ts
        self.psi_trans -= self.psi_trans*coeff
        self.theta_trans += ahrs.Q*ts
        self.theta_trans -= self.theta_trans*coeff

        frame = translate(frame,self.psi_trans,-self.theta_trans)
        res = rotate(frame,-ahrs.phi)
        return res

class targetInd:
    def __init__(self):
        self.x = 0.0
        self.y = 0.0
        self.w = .07
        self.h = self.w /3.0*4.0
        self.track = tracker.tracker()
        self.isTracking = False

    def move(self, dx, dy):
        self.x += dx
        self.y += dy
        self.x = min(self.x,1.0-self.w)
        self.x = max(self.x,-1.0+self.w)
        self.y = min(self.y,1.0-self.h)
        self.y = max(self.y,-1.0+self.h)

    def show(self, track, frame):
        (h, w) = frame.shape[:2]
        w = float(w)/2.0
        h = float(h)/2.0
        mX = w*self.x + w
        mY = h*self.y + h

        startP = (int(mX + self.w*w), int(mY + self.h*h))
        endP = (int(mX - self.w*w), int(mY - self.h*h))

        if True:#not track:
            frame = cv2.rectangle(frame, startP, endP, (0,255,0), 2)
        # else:
        #     frame = cv2.rectangle(frame, startP, endP, (0,255,0), 2)

        return frame

    def refresh(self, frame):
            coeff = .015
            track = False
            while not conVidQue.empty():
                data = conVidQue.get()

                if data.on:
                    self.isTracking = True
                    bbox = self.getbbox(frame)
                    self.track.setTarget(frame,bbox)
                # else:
                self.move(data.up*coeff, data.right*coeff)
                    # track = data.on

            if self.isTracking:
                track, bbox = self.track.update(frame)
                # if track:
                #     self.movebbox(frame,bbox)
                self.isTracking = track

            return self.show(self.isTracking, frame)

    def getbbox(self,frame):
        (h, w) = frame.shape[:2]
        w = float(w)/2.0
        h = float(h)/2.0
        mX = w*self.x + w
        mY = h*self.y + h
        L = int(mX-self.w*w)
        D = int(mY-self.h*h)
        WW = int(2*self.w*w)
        HH = int(2*self.h*h)
        bbox = (L,D,WW, HH)
        return bbox

    def movebbox(self,frame,bbox):
        (h, w) = frame.shape[:2]

        # (L,D,W,H) = bbox
        L= bbox[0]
        D = bbox[1]
        W = bbox[2]
        H = bbox[3]
        self.x = L/w
        self.y = D/h
        self.w = W/w
        self.h = H/h

    def targetSend(self):
        cmd=tlm.trackData(False, self.x, self.y)
        cmd=tlm.prepareControl(cmd)
        if not udpSendQue.full():
            udpSendQue.put(cmd)


import os
import pytest
from pandas.api.types import is_string_dtype  # type: ignore

from dae.variants.attributes import Role

from dae.pedigrees.families_data import FamiliesData
from dae.pedigrees.loader import FamiliesLoader

@pytest.mark.parametrize(
    "pedigree",
    [
        ("pedigree_A.ped"),
        ("pedigree_B.ped"),
        ("pedigree_B2.ped"),
        ("pedigree_C.ped"),
    ],
)
def test_famlies_loader_simple(pedigree, fixture_dirname):
    filename = fixture_dirname(f"pedigrees/{pedigree}")
    assert os.path.exists(filename)
    loader = FamiliesLoader(filename)
    families = loader.load()

    assert families is not None

def test_families_loader_phenotype(fixture_dirname):
    filename = fixture_dirname("pedigrees/pedigree_D.ped")
    assert os.path.exists(filename)

    loader = FamiliesLoader(filename)
    families = loader.load()

    assert families is not None
    assert isinstance(families, FamiliesData)

    for fam_id, family in families.items():
        print(fam_id, family, family.persons)
        for person_id, person in family.persons.items():
            print(person)
            print(person.has_attr("phenotype"))
            assert person.has_attr("phenotype")

def test_families_loader_phenos(fixture_dirname):
    filename = fixture_dirname("pedigrees/pedigree_phenos.ped")
    assert os.path.exists(filename)

    loader = FamiliesLoader(filename)
    families = loader.load()

    assert families is not None
    assert isinstance(families, FamiliesData)

    for fam_id, family in families.items():
        for person_id, person in family.persons.items():
            assert person.has_attr("phenotype")
            assert person.has_attr("pheno2")
            assert person.has_attr("pheno3")

    ped_df = families.ped_df
    assert is_string_dtype(ped_df["pheno3"])
    assert is_string_dtype(ped_df["pheno2"])
    assert is_string_dtype(ped_df["phenotype"])

@pytest.mark.parametrize(
    "pedigree",
    [
        ("sample_nuc_family.ped"),
        ("pedigree_no_role_A.ped"),
        ("pedigree_no_role_B.ped"),
        ("pedigree_no_role_C.ped"),
    ],
)
def test_families_loader_no_role(pedigree, fixture_dirname):
    filename = fixture_dirname(f"pedigrees/{pedigree}")
    assert os.path.exists(filename)

    params = {
        "ped_no_role": True,
    }
    loader = FamiliesLoader(filename, **params)
    families = loader.load()

    assert families is not None
    assert isinstance(families, FamiliesData)

    fam = families["f1"]
    assert fam is not None

    persons = fam.get_members_with_roles(["prb"])
    assert len(persons) == 1

    person = persons[0]
    assert person.person_id == "f1.prb"

    persons = fam.get_members_with_roles(["sib"])
    assert len(persons) == 1

    person = persons[0]
    assert person.person_id == "f1.sib"

def test_families_loader_roles_testing(fixture_dirname):
    filename = fixture_dirname("pedigrees/pedigree_no_role_C.ped")
    assert os.path.exists(filename)

    params = {
        "ped_no_role": True,
    }
    loader = FamiliesLoader(filename, **params)
    families = loader.load()

    assert families.persons[("f1", "f1.mg_dad")].role == \
        Role.maternal_grandfather
    assert families.persons[("f1", "f1.mg_mom")].role == \
        Role.maternal_grandmother
    assert families.persons[("f1", "f1.pg_dad")].role == \
        Role.paternal_grandfather
    assert families.persons[("f1", "f1.pg_mom")].role == \
        Role.paternal_grandmother

@pytest.mark.parametrize(
    "pedigree",
    [
        ("pedigree_A.ped"),
        ("pedigree_B.ped"),
        ("pedigree_B2.ped"),
        ("pedigree_C.ped"),
    ],
)
def test_families_ped_df(pedigree, temp_filename, fixture_dirname):
    filename = fixture_dirname(f"pedigrees/{pedigree}")
    assert os.path.exists(filename)

    loader = FamiliesLoader(filename)
    families = loader.load()

    assert families._ped_df is None

    new_df = families.ped_df
    assert new_df is not None

    # assert_frame_equal(ped_df, new_df, check_like=True)


"""
File: testnode.py
Project 4.10

Add a remove function.

Tests the Node class.
"""

from node import Node

def length(head):
    """Returns the number of items in the linked structure
    referred to by head."""
    probe = head
    count = 0
    while probe != None:
        count += 1
        probe = probe.next
    return count

def insert(index, newItem, head):
    """Inserts newItem at position is the linked structure
    referred to by head.  Returns a reference to the new
    structure."""
    if index <= 0:
        # newItem goes at the head
        head = Node(newItem, head)
    else:
        # Search for node at position index - 1 or the last position
        probe = head
        while index > 1 and probe.next != None:
            probe = probe.next;
            index -= 1
        # Insert new node after node at position index - 1
        # or last position
        probe.next = Node(newItem, probe.next)
    return head

def pop(index, head):
    """Removes the item at index from the linked structure
    referred to by head and returns the tuple (head, item)
    Precondition: 0 <= index < length(head)"""
    if index < 0 or index >= length(head):
        raise IndexError("Index out of bounds")
    # Assumes that the linked structure has at least one item
    if index == 0:
        removedItem = head.data
        head = head.next
    else:
        # Search for node at position index - 1 or
        # the next to last position
        probe = head
        while index > 1 and probe.next.next != None:
            probe = probe.next
            index -= 1
        removedItem = probe.next.data
        probe.next = probe.next.next
    return (head, removedItem)

def printStructure(head):
    """Prints the items in the structure referred to by head."""
    probe = head
    while probe != None:
        print(probe.data, end = " ")
        probe = probe.next
    print()

def main():
    """Tests modifications."""
    head = None

    head = insert(0, "1", head)
    print("1:", end = " ")
    printStructure(head)

    (head, item) = pop(0, head)
    print("1:", item, end = " ")
    printStructure(head)

    # Add five nodes to the beginning of the linked structure
    for count in range(1, 6):
        head = Node(count, head)

    (head, item) = pop(0, head)
    print("5 4 3 2 1:", item, end = " ")
    printStructure(head)

    (head, item) = pop(length(head) - 1, head)
    print("1 4 3 2:", item, end = " ")
    printStructure(head)

    (head, item) = pop(1, head)
    print("3 4 2:", item, end = " ")
    printStructure(head)

    pop(4, head)

if __name__ == "__main__": main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu May 16 13:29:10 2019

@author: phoebeharmon
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu May  9 14:10:19 2019

@author: phoebeharmon
"""

import speech_recognition as sr
print(sr.__version__)

#Recognizer instance recongizes speech from an audio source
#create Recognizer instance
r = sr.Recognizer()

#7 methods to recognize speech from audio source
#only recognize_sphinx() works without internet connection
#default key provided for recognize_google() aka google web speech api
#r.recognize_google()

#AudioFile class initialized with path to an audio file
harvard = sr.AudioFile('harvard.wav')
with harvard as source: #context manager opens file, reads contents, stores data in AudioFile instance called source
    audio = r.record(source) #records data from file into AudioData instance
type(audio)

message = r.recognize_google(audio)
print(message)

#capture portion of speech in file
with harvard as source:
    audio1 = r.record(source, duration=4) #first 4 seconds of file
    audio2 = r.record(source, duration=4) #next 4 seconds of file
with harvard as source:
    audio3 = r.record(source, offset=4, duration=3) #records 3 seconds started at 4 seconds
message_4sec = r.recognize_google(audio)
print(message_4sec)

#deal with noise
with harvard as source:
    r.adjust_for_ambient_noise(source, duration=.5)
    audio = r.record(source)

print(r.recognize_google(audio, show_all = True)) #returns all possible transcripts


# Python Program to Calculate the Number of Upper Case Letters and Lower Case Letters in a String

name=input("Please input your string: ")
lower_n=0
lower="abcdefghijklmnopqrstuvwxyz"
upper_n=0
upper="ABCDEFGHIJKLMNOPQRSTUVWXYZ"
for i in name:
    if i in lower:
        lower_n+=1
    if i in upper:
        upper_n+=1
print("Number of LOWER case letters:",lower_n)
print("Number of UPPER case letters:",upper_n)


import tkinter as tk
from tkinter import ttk
from tkinter import filedialog
from ttkthemes import ThemedStyle
from tkinter import font
from win32 import win32api
from win32 import win32print
import os
import sys
import matplotlib.font_manager as fm

root = tk.Tk()
root.withdraw()
style = ThemedStyle(root)
exe_dir = os.path.dirname(sys.executable)
script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
fonts_dir = os.path.join(script_dir, "fonts")

custom_fonts = []
for font_file in os.listdir(fonts_dir):
    if font_file.endswith(".ttf"):
        font_path = os.path.join(fonts_dir, font_file)
        font_name = os.path.splitext(font_file)[0]
        installed_fonts = [f.name for f in fm.fontManager.ttflist]
        if font_name not in installed_fonts:
            try:
                fm.fontManager.addfont(font_path)
            except fm.FontManagerError:
                print(f"Failed to install font: {font_name}")
            else:
                print(f"Font installed: {font_name}")

        custom_font = font.Font(family=font_name, name=font_name)
        custom_fonts.append(custom_font)

style.theme_use("equilux")
style.configure("TCombobox", fieldbackground="white", background="white")
style.configure("TButton", foreground="white")
style.configure(".", font=("Gantari SemiBold", 13))

def change_theme():
    if style.theme_use() == "equilux":
        style.theme_use("default")
    else:
        style.theme_use("equilux")

def print_files(file_path):
    printers = win32print.EnumPrinters(2)
    printer_names = [printer[2] for printer in printers]

    pt = tk.Toplevel()
    pt.geometry("450x150")
    pt.title("Tisknu")

    font_style = ("Gantari SemiBold", 12)
    LABEL = tk.Label(pt, text="Vyber tiskárnu", fg="black", font=font_style)
    LABEL.pack()

    PRCOMBO = ttk.Combobox(pt, width=35, values=printer_names, style="TCombobox", font=font_style)
    PRCOMBO.pack()

    style.configure("Custom.Horizontal.TProgressbar", troughcolor="white", background="white")
    PROGRESS_BAR = ttk.Progressbar(pt, orient="horizontal", length=200, mode="indeterminate", style="Custom.Horizontal.TProgressbar")
    PROGRESS_BAR.pack(pady=10)

    BUTTON = ttk.Button(pt, text="Tisknout", command=lambda: print_selected_file(file_path, PRCOMBO.get(), PROGRESS_BAR), style="TButton")
    BUTTON.pack()

def print_selected_file(file_path, printer_name, progress_bar):
    root = tk.Tk()
    root.withdraw()
    file_path = filedialog.askopenfilename()
    with open(file_path, 'r') as f:
        lines = f.readlines()
        total_lines = len(lines)
        progress_bar.configure(maximum=total_lines)

        for index, line in enumerate(lines):
            file_name, directory = line.strip().split(':')
            win32api.ShellExecute(0, "print", directory, '/{}'.format(printer_name), ".", 0)
            progress_bar.step()
            progress_bar.update()

    progress_bar.stop()

print_files('file.txt')

button = ttk.Button(root, text="Change theme!", command=change_theme)
button.pack()

root.mainloop()


###### Dictionary comprehension #########
"""name_age = {
    "ijhar": 30,
    "hasan": 40,
    "majhar": 30
}"""
##################################
"""num_dict ={}
for num in range(1, 30):
    num_dict[num] = num **3
print(num_dict)"""
############## short ###############
"""num_dict_comp = { number: number**3 for number in range(1,30) if number % 2 == 0 }
print(num_dict_comp)
############"""

"""num_dict_comp = { number: number**3 if number % 2 == 0 else "Odd" for number in range(1,30)}
print(num_dict_comp)
"""

######### if age is odd or not##############
name_age = {
    "ijhar": 45,
    "hasan": 40,
    "majhar": 30,
    "elias": 25
}
age_dict = { name:"old" if age >= 40 else "yang" for name, age in name_age.items()}
print(age_dict)


from sklearn.metrics import f1_score, matthews_corrcoef
import numpy as np
from rouge import Rouge
from src.utils import qa_utils
from datasets import load_metric
import re

class App:
    def __init__(self):
        self.functions = {}

    def add(self, key):
        def adder(func):
            self.functions[key] = func
            return func

        return adder

    def __getitem__(self, __name: str):
        return self.functions[__name]

metric_dict = App()

@metric_dict.add("rouge")
def rouge(preds, labels, return_list=False):
    # https://github.com/pltrdy/rouge
    r1s, r2s, rls = [], [], []
    r = Rouge()
    for i in range(len(labels)):
        if "\n" not in preds[i]:
            preds[i] += "\n"  # to ensure rouge metrics
        if "\n" not in labels[i]:
            labels[i] += "\n"
        scores = r.get_scores(preds[i], labels[i])[0]
        r1s.append(scores["rouge-1"]["f"])
        r2s.append(scores["rouge-2"]["f"])
        rls.append(scores["rouge-l"]["f"])
    if return_list:  # used for scoring data
        return r1s
    r1 = sum(r1s) / len(r1s)
    r2 = sum(r2s) / len(r2s)
    rl = sum(rls) / len(rls)
    return r1, r2, rl

@metric_dict.add("squad")
def squad(labels, preds, return_list=False):
    """Computes SQuAD metrics, maximizing over answers per question.
    Args:
    labels: list of lists of strings
    preds: list of strings
    Returns:
    dict with score_key: squad score across all labels and predictions
    """
    labels = [[qa_utils.normalize_squad(t) for t in u] for u in labels]
    preds = [qa_utils.normalize_squad(p) for p in preds]
    if return_list:  # used for scoring data
        em, f1 = qa_utils.qa_metrics(labels, preds, return_list=True)
        return f1
    em, f1 = qa_utils.qa_metrics(labels, preds)  # em,f1
    return em, f1

@metric_dict.add("trivia_qa")
def trivia_qa(labels, preds, return_list=False):
    """Computes TriviaQA metrics, maximizing over answers per question.
    Args:
    labels: list of lists of strings
    preds: list of strings
    Returns:
    dict with score_key: squad score across all labels and preds
    """
    labels = [[qa_utils.normalize_trivia_qa(t) for t in u] for u in labels]
    preds = [qa_utils.normalize_trivia_qa(p) for p in preds]
    if return_list:  # used for scoring data
        em, f1 = qa_utils.qa_metrics(labels, preds, return_list=True)
        return f1
    em, f1 = qa_utils.qa_metrics(labels, preds)  # em,f1
    return em, f1

@metric_dict.add("simple_accuracy")
def simple_accuracy(preds, labels, return_list=False):
    if isinstance(preds[0], str):
        labels = [label.strip() for label in labels]
        preds = [pred.strip() for pred in preds]
    res = [int(preds[i] == labels[i]) for i in range(len(preds))]
    if return_list:
        return res
    acc = sum(res) / len(res)
    return acc

@metric_dict.add("pubmed_qa_acc")
def pubmed_qa_acc(preds, labels, return_list=False):
    pattern=r'([.\s]*)(the answer is)(.*)'
    regex=re.compile(pattern,re.IGNORECASE)

    res_list = []
    for i, pred in enumerate(preds):
        label = labels[i]
        if len(regex.findall(pred))>0:
            answer = regex.findall(pred)[-1][-1].lower()
            if "yes" in answer:
                acc = 1 if label=='yes' else 0
            elif "no" in answer:
                acc = 1 if label=='no' else 0
            elif "maybe" in answer:
                acc = 1 if label=='maybe' else 0
            else:
                acc = 0
        else:
            answer = None
            acc = 0
        res_list.append(acc)
    if return_list:
        return res_list
    return sum(res_list)/len(res_list)

def acc_and_f1(preds, labels):
    acc = simple_accuracy(preds, labels)
    f1 = f1_score(y_true=labels, y_pred=preds)
    return acc, f1, (acc + f1) / 2

def acc_and_matthews_corrcoef(preds, labels):
    acc = simple_accuracy(preds, labels)
    mcc = matthews_corrcoef(y_true=labels, y_pred=preds)
    return acc, mcc

def compute_bleu(preds, labels):
    BLEU = load_metric("bleu")
    predictions = [[ch for ch in text] for text in preds]
    references = [[[ch for ch in label]] for label in labels]
    return BLEU.compute(predictions=predictions, references=references)

def compute_metrics(metric, labels, preds):
    assert len(preds) == len(labels)
    if metric == "simple_accuracy":
        return {"acc": simple_accuracy(preds, labels) * 100}
    elif metric == "rouge":
        r1, r2, rl = rouge(preds, labels)
        return {"r1": r1 * 100, "r2": r2 * 100, "rl": rl * 100}
    elif metric == "acc_and_f1":
        acc, f1, acc_f1 = acc_and_f1(preds, labels)
        return {"acc": acc * 100, "f1": f1 * 100, "acc_and_f1": acc_f1 * 100}
    elif metric == "acc_and_matthews_corrcoef":
        acc, mcc = acc_and_matthews_corrcoef(preds, labels)
        return {"acc": acc * 100, "mcc": mcc * 100}
    elif metric == "f1":
        f1 = f1_score(y_true=labels, y_pred=preds)
        return {"f1": f1 * 100}
    elif metric == "squad":
        em, f1 = squad(labels=labels, preds=preds)
        return {"em": em, "f1": f1}
    elif metric == "trivia_qa":
        em, f1 = trivia_qa(labels=labels, preds=preds)
        return {"em": em, "f1": f1}
    elif metric == "bleu":
        bleu = compute_bleu(preds=preds, labels=labels)
        return {"bleu": bleu["bleu"] * 100}
    elif metric == "pubmed_qa_acc":
        acc = pubmed_qa_acc(preds=preds, labels=labels)
        return {"pubmed_qa_acc": acc * 100}

def compute_scores(metric, data):
    preds = [entry["pred"] for entry in data]
    labels = [entry["label"] for entry in data]
    if not isinstance(preds[0], str):
        preds = np.array(preds)
        labels = np.array(labels)
    scores = compute_metrics(metric, labels=labels, preds=preds)
    return scores


import os
import datetime
import ConfigParser
from tempfile import NamedTemporaryFile
from InvoiceGenerator.api import Invoice, Item, Client, Provider, Creator, Address
from InvoiceGenerator.pdf import SimpleInvoice

# choose english as language
os.environ["INVOICE_LANG"] = "en"

config = ConfigParser.ConfigParser()
config.read("config.ini")

# import info from config.ini
configFile = "myvars"
accNumber = config.get(configFile, "account_number")
sortCode = config.get(configFile, "sort_code")
address = config.get(configFile, "address")

client = Client('The Inside Story')
provider = Provider(address,
                    bank_account=accNumber, bank_code=sortCode)
creator = Creator('')

invoice = Invoice(client, provider, creator)
invoice.currency_locale = 'en_UK.UTF-8'
invoice.currency = u'\u00a3'
invoice.number = '1'

def GenerateInvoice():
    i, x = 0, 1
    itemQuantity = input("How many items would you like to add? ")

    while i < itemQuantity:
        print('Please enter the details for Item ' + str(x))
        Quantity = raw_input("Please enter a quantity of good provided: ")
        Price = raw_input("Please enter the total price: ")
        Description = raw_input("Please enter a description: ")
        invoice.add_item(Item(Quantity, Price, Description))
        x += 1
        i += 1
    pdf = SimpleInvoice(invoice)
    invoiceName = "Invoice001.pdf"
    pdf.gen(invoiceName, generate_qr_code=True)

GenerateInvoice()
print('Invoice Generated!')


import pandas as pd
import numpy as np
import nltk
nltk.download('punkt')
from nltk.tokenize import word_tokenize

df=pd.read_csv("cauhoi.csv",delimiter=";")
tt=df.Question
Answer={}

t,c=np.unique(df.Answer,return_counts=True)

for i in range(0,len(t)):
    Answer[i]=t[i]

d=['(',')',',','.','!',' ','-','?','!','@','#','$','%','&','*','/',',',':',';','[',']','{','}','>','<','_','=','bé','cho',
    'có','con','giáo','của','gì','học','không','làm','ngày','nhà','nào','như','sẽ','sao','thế','thể','trường','tôi','viên',
    'được','ở','trẻ','những','các','thì','bị']
#Tách từ#
l=[]
words=[]
for i in range(1,len(tt)):
    w=word_tokenize(tt.iloc[i].lower())
    l.extend(w)

#Mã hóa dữ liệu cột Answer#
from sklearn.preprocessing import LabelEncoder
t=df.apply(LabelEncoder().fit_transform)
t=t.Answer

#Tìm stop_word#
words=np.unique(l)
vt={'k':'không','hog':'không','ko':'không','khg':'không',
    'bt':'biết','bk':'biết','lm':'làm','hok':'học','dc':'được',
    'vt':'viết','vc':'việc','hc':'học','j':'gì','z':'vậy','h':'giờ','đc':'được'}

for i in range(0,len(words)):
  v=words[i]
  if(v in vt):
    words[i]=vt[v]
words=np.unique(words)
index=[]
for i in range(0,len(words)):
  if words[i] in d:
    index.append(i)
words=np.delete(words,index)

#Tạo dữ liệu huấn luyện#

dt=pd.DataFrame(columns=words)
s=[]
for i in words:
    for j in range(0,len(tt)):
        a=tt.iloc[j].lower()
        if i in a:
            s.append(1)
        else:
            s.append(0)
    dt[i]=s
    s=[]
#Lấy X,y#

X=dt
y=t

from sklearn.decomposition import PCA

# Khởi tạo đối tượng PCA với số comp = 50
my_pca = PCA (n_components = 50)

# Fit vào data
my_pca.fit(X)

# Thực hiện transform
X=my_pca.transform(X)
X=pd.DataFrame(X)

from imblearn.over_sampling import SMOTE
smote = SMOTE(k_neighbors=1, random_state=0)
X, y = smote.fit_resample(X,y)

#Huấn luyện mô hình
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
#cây
from sklearn import svm
from sklearn.ensemble import BaggingClassifier
s1=svm.SVC(C=100,kernel='sigmoid')
rdf=BaggingClassifier(base_estimator=s1,
                        n_estimators=100, random_state=0).fit(X, y)

import datetime
now = datetime.datetime.now()
today = now.date()
today=str(today)
import csv

def res(f):
    v=word_tokenize(f.lower())
    for i in v:
      if(i in vt):
        f=f.replace(i,vt[i])
    #Tạo DataFrame cho câu hỏi đầu vào
    ds=pd.DataFrame(columns=words)
    for i in words:
        f=f.lower()
        if i in f:
            ds[i]=[1]
        else:
            ds[i]=[0]

    #Giảm chiều dòng dữ liệuliệu
    ds=my_pca.transform(ds)
    d=rdf.predict(ds)
    #Dự đoán Answer cho câu hỏi
    tg=Answer[d[0]]
    with open('log.csv','a', encoding="utf-8") as file:
        writer = csv.writer(file,delimiter=';')
        writer.writerow([f,tg,today])
    return tg
res('học năng khiếu')


import streamlit as st
import pandas as pd
from PIL import Image
import shap
import matplotlib.pyplot as plt
from sklearn import datasets
import pickle

def main():
    st.write("""
    # Boston House Price Prediction App
    This app predicts the **Boston House Price**!
    """)

    image = Image.open('boston.jpg')
    st.image(image, width=None)

    st.write("""
    Photo by [Cloris Ying](https://unsplash.com/@clorisyy?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText) on [Unsplash](https://unsplash.com/s/photos/boston-house?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)

    ---
    """)

    # Loads the Boston House Price Dataset
    boston = datasets.load_boston()
    X = pd.DataFrame(boston.data, columns=boston.feature_names)
    Y = pd.DataFrame(boston.target, columns=["MEDV"])

    # Sidebar
    # Header of Specify Input Parameters
    st.sidebar.header('Specify Input Parameters')
    st.sidebar.write('Click the Submit button below after specifying all the features. Check the help button for more info.')
    with st.sidebar.form(key='form'):
        CRIM = st.slider('CRIM', float(X.CRIM.min()), float(X.CRIM.max()), float(X.CRIM.mean()), help="per capita crime rate by town")
        ZN = st.slider('ZN', float(X.ZN.min()), float(X.ZN.max()), float(X.ZN.mean()), help="proportion of residential land zoned for lots over 25,000 sq.ft.")
        INDUS = st.slider('INDUS', float(X.INDUS.min()), float(X.INDUS.max()), float(X.INDUS.mean()), help="proportion of non-retail business acres per town")
        CHAS = st.slider('CHAS', float(X.CHAS.min()), float(X.CHAS.max()), float(X.CHAS.mean()), help="Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)")
        NOX = st.slider('NOX', float(X.NOX.min()), float(X.NOX.max()), float(X.NOX.mean()), help="nitrogen oxides concentration (parts per 10 million)")
        RM = st.slider('RM', float(X.RM.min()), float(X.RM.max()), float(X.RM.mean()), help="average number of rooms per dwelling")
        AGE = st.slider('AGE', float(X.AGE.min()), float(X.AGE.max()), float(X.AGE.mean()), help="proportion of owner-occupied units built prior to 1940")
        DIS = st.slider('DIS', float(X.DIS.min()), float(X.DIS.max()), float(X.DIS.mean()), help="weighted mean of distances to five Boston employment centres")
        RAD = st.slider('RAD', float(X.RAD.min()), float(X.RAD.max()), float(X.RAD.mean()), help="index of accessibility to radial highways")
        TAX = st.slider('TAX', float(X.TAX.min()), float(X.TAX.max()), float(X.TAX.mean()), help="full-value property-tax rate per $10,000")
        PTRATIO = st.slider('PTRATIO', float(X.PTRATIO.min()), float(X.PTRATIO.max()), float(X.PTRATIO.mean()), help="pupil-teacher ratio by town")
        B = st.slider('B', float(X.B.min()), float(X.B.max()), float(X.B.mean()), help="1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town")
        LSTAT = st.slider('LSTAT', float(X.LSTAT.min()), float(X.LSTAT.max()), float(X.LSTAT.mean()), help="lower status of the population (percent)")
        submit = st.form_submit_button(label='Submit')

    if submit:
        data = {'CRIM': CRIM,
                'ZN': ZN,
                'INDUS': INDUS,
                'CHAS': CHAS,
                'NOX': NOX,
                'RM': RM,
                'AGE': AGE,
                'DIS': DIS,
                'RAD': RAD,
                'TAX': TAX,
                'PTRATIO': PTRATIO,
                'B': B,
                'LSTAT': LSTAT}
        df = pd.DataFrame(data, index=[0])

        # Print specified input parameters
        st.header('Input Parameters')
        st.write(df)
        st.write('---')

        # Use Regression Model
        model = pickle.load(open('boston_reg.pkl', 'rb'))

        # Apply Model to Make Prediction
        prediction = model.predict(df)

        st.header('Prediction of MEDV')
        st.write("""The prediction of median value (MEDV) of owner-occupied homes in $1000s.""")
        st.write(prediction)
        st.write(f"""The predicted value is ${'{:.2f}'.format(float(prediction*1000))}""")
        st.write('---')

        # Explaining the model's predictions using SHAP values
        # https://github.com/slundberg/shap
        explainer = shap.TreeExplainer(model)
        shap_values = explainer.shap_values(X)

        st.header('Feature Importance')
        plt.title('Feature importance based on SHAP values')
        shap.summary_plot(shap_values, X)
        st.set_option('deprecation.showPyplotGlobalUse', False) # disable warning
        st.pyplot(bbox_inches='tight')
        st.write('---')

        plt.title('Feature importance based on SHAP values (Bar)')
        shap.summary_plot(shap_values, X, plot_type="bar")
        st.pyplot(bbox_inches='tight')
        st.set_option('deprecation.showPyplotGlobalUse', False) # disable warning

    else:
        st.write('Submit input parameters from the sidebar.')

if __name__ == "__main__":
    main()


#! /usr/bin/env python
"""
Finite State Methods and Statistical NLP
"""

import nltk
import pickle
import random
from os import path
from nltk.corpus import masc_tagged
from nltk.tag import hmm
from ass3utils import train_unsupervised

nltk.download('masc_tagged')

short_sent = [
    "Once we have finished , we will go out .",
    "There is always room for more understanding between warring peoples .",
    "Strong demand oil rally ."
    ]

sentences = [
    "Once we have finished , we will go out .",
    "There is always room for more understanding between warring peoples .",
    "Evidently , this was one of Jud 's choicest tapestries , for the noble emitted a howl of grief and rage and leaped from his divan ."
    ]
sentences2 = [
    "Misjoggle in a gripty hifnipork .",
    "One fretigy kriptog is always better than several intersplicks ."
    ]
news = [
    "Strong demand triggers oil rally .",
    "Crude oil prices surged back above the $47 a barrel mark on Thursday after an energy market watchdog raised its forecasts for global demand ."
    ]
legal = ["If there is any conflict between the terms in the General Terms and the Additional Terms, then the Additional Terms govern .",
         "You may have additional rights under the law .",
         "We do not seek to limit those rights where it is prohibited to do so by law ."
          ]
def tag_count(tag='VB'):
    """
    Count the number of a given tag in corpus.
    """
    return sum([True for sent in masc_tagged.tagged_sents() for pos_tag in sent if pos_tag[1] == tag]) # check tag for each sentence

def get_unique_tags():
    """
    Get the tags in a corpus.

    Using a set ensures we get distinctly unique tags across the entire corpus
    """
    return {pos_tag[1] for sent in masc_tagged.tagged_sents() for pos_tag in sent}

def next_tags(tag='VB'):
    """
    Calculate the transition distribution.

    The transition from one tag to other tags. Returns a list of tags
    that appear next to the given tag.
    """
    tags = get_unique_tags()
    dist = dict()
    next_tags = []
    # Check for the tag that is next 'VB'
    for sent in  masc_tagged.tagged_sents():
        for i in range (len(sent)):
            if sent[i][1] == tag and i+1 < len(sent):
                next_tags.append(sent[i+1][1])
    return next_tags

def count_next_tags(tag='VB'):
    """
    Get the number of unique tags next to a given tag

    Stats of the next tag.
    """
    ntags = next_tags(tag=tag)
    corpus_tags = get_unique_tags()
    return {tag:ntags.count(tag) for tag in get_unique_tags()}

def tag_transition_dist(tag='VB'):
    """
    Calculate the transition distribution for a given tag.

    This function calculates the probabiluty that the next tag is a DT
    (Determinant) given the current tag is a 'VB (Verb):
    p(t[i+1]=DT|t[i]=VB)
    """
    vb_next = count_next_tags()
    dt = vb_next['DT']
    total = 0
    for item in vb_next:
        total += vb_next[item]
    return dt/total

def tag_and_word(tag='VB', word='feel'):
    """
    Calculate the p(w[i]='feel'|t[i]=VB)

    Calculates the probability of a given word given a tag
    """
    tags = [word_tag for sent in masc_tagged.tagged_sents() for word_tag in sent if word_tag[1] == tag]
    return sum([True for tag in tags if tag[0] == word])/len(tags)

def nltk_hmm(sentences):
    """
    Using NLTK's library to perform tagging by training a Hidden
    Markov Model, the model uses Maximum Likelihood Estimates (MLE) to
    estimate the parameters.

    Note: The model appears to assign all unseen words as NN, one
    solution as indicated in the slides(Day 3) is "smoothing".
    """
    trainer = hmm.HiddenMarkovModelTrainer()
    model = trainer.train(masc_tagged.tagged_sents())
    for sentence in sentences:
        print("Tagging sentence: {}".format(sentence))
        print("{}\n".format(model.tag(sentence.split())))

def semi_supervised_hmm(sentences):
    """
    Train and test a tagging model with both labeled and unlabeled
    data.

    Trains a semi-supervised model where the model is trained with
    partly labeled data and partly unlabled data.

    sentences: Sentences to be tagged
    """
    with open('radio_planet_tokens.txt', 'r') as f:
        unlabeled_data = [[word for word in sent.split()] for sent in f.readlines()]  # Labeled data
    labeled_data = masc_tagged.tagged_sents()  # Unlabeled data

   # Check if the model exists, saved the first time it is
   # successfully created. Delete file to update model
    if path.exists("hmm_model.pkl"):
        # Load the model from disk
        print("Loading model from the disk")
        with open("hmm_model.pkl", 'rb') as model:
            hmm_model = pickle.load(model)
    else:
        # Train the model since not found
        print("Training the model...")
        hmm_model = train_unsupervised(labeled_data, unlabeled_data)
        with open("hmm_model.pkl", 'wb') as model:
            pickle.dump(hmm_model, model)

    # Tag sentences with trained model
    for sentence in sentences:
        print("Tagging sentence: {}".format(sentence))
        print("{}\n".format(hmm_model.tag(sentence.split())))

def log_prob_hmm(sentences):
    """
    Create a language model using a trained model

    FIXME: For some reason the log_probability does not work, there is
    a Runtime warning generated when applying the model.
    """
    # Using a previously trained model
    if path.exists("hmm_model.pkl"):
        # Load the model from disk
        print("Loading model from the disk")
        with open("hmm_model.pkl", 'rb') as model:
            hmm_model = pickle.load(model)
        log_probs = hmm_model.log_probability([(word, None) for sent in sentences for word in sent.split()])
        return log_probs
    else:
        print("Model does not exist, first train the model")

def gen_hmm():
    """
    Generating a language from the HMM model

    The idea is to generate random sentences from the model.
    """
    print("Loading model from the disk")
    if path.exists("hmm_model.pkl"):
        with open("hmm_model.pkl", 'rb') as model:
            hmm_model = pickle.load(model)

    for i in range(7):
        rand_len = random.randint(6, 10)
        sent = hmm_model.random_sample(random, rand_len)

        print([" ".join([word[0] for word in sent])])

if __name__ == '__main__':
#     print("Tag count for {} tag: {}".format('VB', tag_count()))
#     print("Tags in the corpus {}".format(unique_tags()))

#     print("{}".format(next_tags()))
#     print("\n")
#     print(count_next_tags())

#     # part 1a
#    print("Transition distribution for 'DT' tag: {}".format(tag_transition_dist()))
#     # Part 1b
#    print("Computing the probability of a word given a tag {}".format(tag_and_word()))

#     # part 2
#     print("Part 2: NLTK hmm Model \n")
#    nltk_hmm(sentences) # sentences seen during training
#    nltk_hmm(sentences2)  # sentences not seen during training
    # part 3
#    semi_supervised_hmm(sentences)

    # part 4
#     nltk_hmm(news)
#     semi_supervised_hmm(news)
#     nltk_hmm(legal)
#     semi_supervised_hmm(legal)
    # part 5
    print(log_prob_hmm(short_sent))

    # part 6
#    gen_hmm()


import unittest
from itertools import chain

import torch

from pytorch_metric_learning.distances import CosineSimilarity
from pytorch_metric_learning.losses import CentroidTripletLoss
from pytorch_metric_learning.reducers import MeanReducer

from .. import TEST_DEVICE, TEST_DTYPES
from ..zzz_testing_utils.testing_utils import angle_to_coord

def normalize(embeddings):
    embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=0)
    return embeddings

class TestCentroidTripletLoss(unittest.TestCase):
    def test_indices_tuple_failure(self):
        loss_fn = CentroidTripletLoss()
        for labels_arr in [[0, 0, 0, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 1, 2, 2, 3]]:
            embeddings = torch.randn(len(labels_arr), 32, device=TEST_DEVICE)
            labels = torch.tensor(labels_arr, device=TEST_DEVICE)
            with self.assertRaises(ValueError) as context:
                loss_fn(embeddings, labels)

    def test_centroid_triplet_loss(self):
        for dtype in TEST_DTYPES:
            embedding_angles = [0, 10, 20, 30, 40, 50]
            centroid_makers = [
                [[10, 20], [30, 40, 50]],
                [[0, 20], [30, 40, 50]],
                [[0, 10], [30, 40, 50]],
                [[40, 50], [0, 10, 20]],
                [[30, 50], [0, 10, 20]],
                [[30, 40], [0, 10, 20]],
            ]
            triplets = [
                (0, (0, 0), (0, 1)),
                (1, (1, 0), (1, 1)),
                (2, (2, 0), (2, 1)),
                (3, (3, 0), (3, 1)),
                (4, (4, 0), (4, 1)),
                (5, (5, 0), (5, 1)),
            ]

            labels = torch.LongTensor([0, 0, 0, 1, 1, 1])

            self.helper(embedding_angles, centroid_makers, labels, triplets, dtype)

    def test_sorting_invariance(self):
        for dtype in TEST_DTYPES:
            centroid_makers = [
                [[10, 20], [30, 40, 50]],
                [[40, 50], [0, 10, 20]],
                [[30, 50], [0, 10, 20]],
                [[0, 20], [30, 40, 50]],
                [[0, 10], [30, 40, 50]],
                [[30, 40], [0, 10, 20]],
            ]

            embedding_angles = [0, 30, 40, 10, 20, 50]
            labels = torch.LongTensor([0, 1, 1, 0, 0, 1])

            triplets = [
                (0, (0, 0), (0, 1)),
                (1, (1, 0), (1, 1)),
                (2, (2, 0), (2, 1)),
                (3, (3, 0), (3, 1)),
                (4, (4, 0), (4, 1)),
                (5, (5, 0), (5, 1)),
            ]

            self.helper(embedding_angles, centroid_makers, labels, triplets, dtype)

    def test_imbalanced(self):
        for dtype in TEST_DTYPES:
            per_class_angles = [
                [0, 10, 20],  # class A
                [30, 40, 50, 55],  # class B
                [60, 70, 80, 90],  # class C
            ]
            embedding_angles = chain(*per_class_angles)
            labels = torch.LongTensor([0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2])

            centroid_makers = [
                [[10, 20], [30, 40, 50, 55]],
                [[10, 20], [60, 70, 80, 90]],
                [[0, 20], [30, 40, 50, 55]],
                [[0, 20], [60, 70, 80, 90]],
                [[0, 10], [30, 40, 50, 55]],
                [[0, 10], [60, 70, 80, 90]],
                [[40, 50, 55], [0, 10, 20]],
                [[40, 50, 55], [60, 70, 80, 90]],
                [[30, 50, 55], [0, 10, 20]],
                [[30, 50, 55], [60, 70, 80, 90]],
                [[30, 40, 55], [0, 10, 20]],
                [[30, 40, 55], [60, 70, 80, 90]],
                [[30, 40, 50], [0, 10, 20]],
                [[30, 40, 50], [60, 70, 80, 90]],
                [[70, 80, 90], [0, 10, 20]],
                [[70, 80, 90], [30, 40, 50, 55]],
                [[60, 80, 90], [0, 10, 20]],
                [[60, 80, 90], [30, 40, 50, 55]],
                [[60, 70, 90], [0, 10, 20]],
                [[60, 70, 90], [30, 40, 50, 55]],
                [[60, 70, 80], [0, 10, 20]],
                [[60, 70, 80], [30, 40, 50, 55]],
            ]

            triplets = [
                (0, (0, 0), (0, 1)),
                (0, (1, 0), (1, 1)),
                (1, (2, 0), (2, 1)),
                (1, (3, 0), (3, 1)),
                (2, (4, 0), (4, 1)),
                (2, (5, 0), (5, 1)),
                (3, (6, 0), (6, 1)),
                (3, (7, 0), (7, 1)),
                (4, (8, 0), (8, 1)),
                (4, (9, 0), (9, 1)),
                (5, (10, 0), (10, 1)),
                (5, (11, 0), (11, 1)),
                (6, (12, 0), (12, 1)),
                (6, (13, 0), (13, 1)),
                (7, (14, 0), (14, 1)),
                (7, (15, 0), (15, 1)),
                (8, (16, 0), (16, 1)),
                (8, (17, 0), (17, 1)),
                (9, (18, 0), (18, 1)),
                (9, (19, 0), (19, 1)),
                (10, (20, 0), (20, 1)),
                (10, (21, 0), (21, 1)),
            ]

            self.helper(embedding_angles, centroid_makers, labels, triplets, dtype)

    def helper(
        self,
        embedding_angles,
        centroid_makers,
        labels,
        triplets,
        dtype,
        ref_emb=None,
        ref_labels=None,
    ):
        embeddings = torch.tensor(
            [angle_to_coord(a) for a in embedding_angles],
            requires_grad=True,
            dtype=dtype,
        ).to(
            TEST_DEVICE
        )  # 2D embeddings

        centroids = [
            [
                torch.stack(
                    [
                        torch.tensor(
                            angle_to_coord(a), requires_grad=True, dtype=dtype
                        ).to(TEST_DEVICE)
                        for a in coords
                    ]
                ).mean(-2)
                for coords in one_maker
            ]
            for one_maker in centroid_makers
        ]

        margin = 0.2
        loss_funcA = CentroidTripletLoss(margin=margin)
        loss_funcB = CentroidTripletLoss(margin=margin, reducer=MeanReducer())
        loss_funcC = CentroidTripletLoss(margin=margin, distance=CosineSimilarity())
        loss_funcD = CentroidTripletLoss(
            margin=margin, reducer=MeanReducer(), distance=CosineSimilarity()
        )
        loss_funcE = CentroidTripletLoss(margin=margin, smooth_loss=True)

        [lossA, lossB, lossC, lossD, lossE] = [
            x(embeddings, labels, ref_emb=ref_emb, ref_labels=ref_labels)
            for x in [loss_funcA, loss_funcB, loss_funcC, loss_funcD, loss_funcE]
        ]

        correct_loss = 0
        correct_loss_cosine = 0
        correct_smooth_loss = 0
        num_non_zero_triplets = 0
        num_non_zero_triplets_cosine = 0

        for a, pc, nc in triplets:
            anchor = embeddings[a]

            positive = centroids[pc[0]][pc[1]]
            negative = centroids[nc[0]][nc[1]]

            anchor = normalize(anchor)
            positive = normalize(positive)
            negative = normalize(negative)

            ap_dist = torch.sqrt(torch.sum((anchor - positive) ** 2))
            an_dist = torch.sqrt(torch.sum((anchor - negative) ** 2))
            curr_loss = torch.relu(ap_dist - an_dist + margin)

            curr_loss_cosine = torch.relu(
                torch.sum(anchor * negative) - torch.sum(anchor * positive) + margin
            )
            correct_smooth_loss += torch.nn.functional.softplus(
                ap_dist - an_dist + margin
            )
            if curr_loss > 0:
                num_non_zero_triplets += 1
            if curr_loss_cosine > 0:
                num_non_zero_triplets_cosine += 1
            correct_loss += curr_loss
            correct_loss_cosine += curr_loss_cosine

        rtol = 1e-2 if dtype == torch.float16 else 1e-5
        self.assertTrue(
            torch.isclose(lossA, correct_loss / num_non_zero_triplets, rtol=rtol)
        )
        self.assertTrue(torch.isclose(lossB, correct_loss / len(triplets), rtol=rtol))
        self.assertTrue(
            torch.isclose(
                lossC, correct_loss_cosine / num_non_zero_triplets_cosine, rtol=rtol
            )
        )
        self.assertTrue(
            torch.isclose(lossD, correct_loss_cosine / len(triplets), rtol=rtol)
        )
        self.assertTrue(
            torch.isclose(lossE, correct_smooth_loss / len(triplets), rtol=rtol)
        )

    def test_backward(self):
        margin = 0.2
        loss_funcA = CentroidTripletLoss(margin=margin)
        loss_funcB = CentroidTripletLoss(margin=margin, reducer=MeanReducer())
        loss_funcC = CentroidTripletLoss(smooth_loss=True)
        for dtype in TEST_DTYPES:
            for loss_func in [loss_funcA, loss_funcB, loss_funcC]:
                embedding_angles = [0, 20, 40, 60, 80, 85]
                embeddings = torch.tensor(
                    [angle_to_coord(a) for a in embedding_angles],
                    requires_grad=True,
                    dtype=dtype,
                ).to(
                    TEST_DEVICE
                )  # 2D embeddings
                labels = torch.LongTensor([0, 0, 1, 1, 2, 2])

                loss = loss_func(embeddings, labels)
                loss.backward()


import torch
from transformers import BertTokenizer, BertModel
import pandas as pd
import logging

dataset = pd.read_csv('tokenized_data_without_punctuation_stopwords_lemma.csv')
dataset = dataset[:1000]
dataset = dataset['text'].astype(str).tolist()

# Set log level to ERROR to suppress warning messages
logging.getLogger("transformers").setLevel(logging.ERROR)

model_name = 'bert-base-uncased'  # Pre-trained BERT model
model = BertModel.from_pretrained(model_name)
tokenizer = BertTokenizer.from_pretrained(model_name)

def get_bert_embeddings(dataset):
    embeddings = []
    for sentence in dataset:
        encoded_input = tokenizer.encode_plus(sentence, padding='max_length', truncation=True, return_tensors='pt')
        input_ids = encoded_input['input_ids']
        with torch.no_grad():
            model_output = model(input_ids)
        sentence_embeddings = model_output.last_hidden_state.squeeze(dim=0)  # Extract embeddings
        embeddings.append(sentence_embeddings)
    return embeddings

embeddings = get_bert_embeddings(dataset)
print(embeddings)


import os
import shlex
import shutil
import sys
from pathlib import Path
from subprocess import check_call
from typing import Callable, Dict, List, Union

from pytest import fixture, skip

FILES_PATH = Path(__file__).parent / "files"

def call(cmd: Union[str, List[str]], cwd: Union[str, Path, None] = None) -> int:
    """Call a command
    if str, split into command list
    """
    if isinstance(cmd, str):
        cmd = shlex.split(cmd)
    return check_call(cmd, stdout=sys.stdout, stderr=sys.stderr, cwd=cwd)

@fixture(scope="session")
def needs_symlink(tmp_path_factory):
    if not hasattr(os, "symlink"):
        skip("requires symlink creation")
    tdir = tmp_path_factory.mktemp("check-symlinks")
    source = tdir / "source"
    source.mkdir()
    try:
        os.symlink(source, tdir / "link")
    except OSError:
        skip("requires symlink creation")

@fixture
def git_repo_factory() -> Callable[[Path], Path]:
    def factory(root_path: Path) -> Path:
        repo = root_path / "repo"
        repo.mkdir()

        call("git init", cwd=repo)

        # setup base branch
        src = FILES_PATH

        def copy(files):
            for s, d in files:
                shutil.copy(src / s, repo / d)

        copy(
            [
                ["multilevel-test-base.ipynb", "merge-no-conflict.ipynb"],
                ["inline-conflict--1.ipynb", "merge-conflict.ipynb"],
                ["src-and-output--1.ipynb", "diff.ipynb"],
            ]
        )

        call("git add *.ipynb", cwd=repo)
        call("git config user.name 'JupyterLab Git'", cwd=repo)
        call("git config user.email 'jlab.git@py.test'", cwd=repo)
        call('git commit -m "init base branch"', cwd=repo)
        # create base alias for master
        call("git checkout -b base master", cwd=repo)

        # setup local branch
        call("git checkout -b local master", cwd=repo)
        copy(
            [
                ["multilevel-test-local.ipynb", "merge-no-conflict.ipynb"],
                ["inline-conflict--2.ipynb", "merge-conflict.ipynb"],
                ["src-and-output--2.ipynb", "diff.ipynb"],
            ]
        )
        call('git commit -am "create local branch"', cwd=repo)

        # setup remote branch with conflict
        call("git checkout -b remote-conflict master", cwd=repo)
        copy([["inline-conflict--3.ipynb", "merge-conflict.ipynb"]])
        call('git commit -am "create remote with conflict"', cwd=repo)

        # setup remote branch with no conflict
        call("git checkout -b remote-no-conflict master", cwd=repo)
        copy([["multilevel-test-remote.ipynb", "merge-no-conflict.ipynb"]])
        call('git commit -am "create remote with no conflict"', cwd=repo)

        # start on local
        call("git checkout local", cwd=repo)
        assert not Path(repo / ".gitattributes").exists()
        return repo

    return factory


from copy import copy

from ldap3 import SUBTREE

from plugins.AD import PluginADScanBase
from utils.consts import AllPluginTypes

class PluginADDuplicateAccount(PluginADScanBase):
    """存在重复的账户"""
    # 出现原因是不同用户登录不同域控同时添加同一账户，域控之间同步导致的，添加后有的账户只有"CNF"字段，有的账户有"CNF"的同时
    # 在SAMAccountName里也有"$DUPLICATE-"字段

    display = "存在重复的账户"
    alias = "dup_act"
    p_type = AllPluginTypes.Scan

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def run_script(self, args) -> dict:
        result = copy(self.result)
        instance_list = []

        query = "(&(objectclass=user)(|(cn=*cnf:*)(SAMAccountName=*$duplicate-*)))"
        attributes = ["cn", "distinguishedName", "SAMAccountName"]

        entry_generator = self.ldap_cli.con.extend.standard.paged_search(search_base=self.ldap_cli.domain_dn,
                                                                         search_filter=query,
                                                                         search_scope=SUBTREE,
                                                                         get_operational_attributes=True,
                                                                         attributes=attributes,
                                                                         paged_size=1000,
                                                                         generator=True)

        for entry in entry_generator:
            if entry["type"] != "searchResEntry":
                continue
            instance = {}
            result['status'] = 1
            instance["用户名"] = entry["attributes"]["cn"]
            instance["DN"] = entry["attributes"]["distinguishedName"]
            instance["SAMAccountName"] = entry["attributes"]["SAMAccountName"]
            instance_list.append(instance)

        result['data'] = {"instance_list": instance_list}
        return result


import pandas as pd

def merge_groups_dummies(data_set_path):

    dummies_path = data_set_path.replace('.csv', '_dummies_groupscore.csv')
    dummies_df =pd.read_csv(dummies_path,index_col=[0,1])
    dummies_df.drop(['is_churn'],axis=1,inplace=True)

    groups_path = data_set_path.replace('.csv', '_nocat_groupscore.csv')
    groups_df = pd.read_csv(groups_path,index_col=[0,1])

    merged_df= groups_df.merge(dummies_df,left_index=True,right_index=True)
    save_path = data_set_path.replace('.csv', '_groupscore.csv')
    merged_df.to_csv(save_path)
    print('Saved merged group score + dummy dataset ' + save_path)

    standard_group_metrics = pd.read_csv(data_set_path.replace('.csv', '_nocat_groupmets.csv'),index_col=0)
    dummies_group_metrics = pd.read_csv(data_set_path.replace('.csv', '_dummies_groupmets.csv'),index_col=0)
    merged_col_df = standard_group_metrics.append(dummies_group_metrics)
    merged_col_df.to_csv(data_set_path.replace('.csv', '_groupmets.csv'))


from typing import Any, Dict, List
from unittest.mock import call

import pandas
import pytest
from callee.strings import Glob, String

from tests.service_test_fixtures import ServiceTestFixture
from tests.utils import DataFrameColumnMatcher, shuffled_cases
from the_census._variables.models import Group, GroupCode, GroupVariable
from the_census._variables.repository.service import VariableRepository

# pyright: reportPrivateUsage = false

api_retval = "banana"

groups_in_cache = pandas.DataFrame(
    [
        Group(GroupCode("1"), "desc1").__dict__,
        Group(GroupCode("2"), "desc2").__dict__,
    ]
)

group1_vars: List[Dict[str, Any]] = [
    dict(
        code="var1",
        group_code="group1",
        group_concept="concept1",
        name="name1",
        limit=0,
        predicate_type="int",
        predicate_only=False,
        cleaned_name="Name1",
    ),
    dict(
        code="var2",
        group_code="group1",
        group_concept="concept1",
        name="name2",
        limit=0,
        predicate_type="int",
        predicate_only=False,
        cleaned_name="Name2",
    ),
]
group2_vars: List[Dict[str, Any]] = [
    dict(
        code="var3",
        group_code="group2",
        group_concept="concept2",
        name="name3",
        limit=0,
        predicate_type="int",
        predicate_only=False,
        cleaned_name="Name3",
    ),
    dict(
        code="var4",
        group_code="group2",
        group_concept="concept2",
        name="name4",
        limit=0,
        predicate_type="int",
        predicate_only=False,
        cleaned_name="Name4",
    ),
]
group3_vars: List[Dict[str, Any]] = [
    dict(
        code="var5",
        group_code="group3",
        group_concept="concept3",
        name="name5",
        limit=0,
        predicate_type="int",
        predicate_only=False,
        cleaned_name="Name5",
    ),
]

class TestVariableRepository(ServiceTestFixture[VariableRepository]):
    @pytest.mark.parametrize(*shuffled_cases(is_cache_hit=[True, False]))
    def test_get_groups_given_cache_retval(self, is_cache_hit: bool):
        self.mocker.patch.object(
            self._service._cache,
            "get",
            return_value=groups_in_cache if is_cache_hit else None,
        )
        api_fetch = self.mocker.patch.object(
            self._service._api, "group_data", return_value=api_retval
        )
        transform = self.mocker.patch.object(
            self._service._transformer, "groups", return_value=groups_in_cache
        )

        self._service.get_groups()

        if is_cache_hit:
            api_fetch.assert_not_called()
            transform.assert_not_called()
            self.mocker.patch.object(self._service._cache, "put").assert_not_called()
        else:
            api_fetch.assert_called_once()
            transform.assert_called_once_with(api_retval)
            self.cast_mock(self._service._cache.put).assert_called_once_with(
                "groups.csv", groups_in_cache
            )

    @pytest.mark.parametrize(*shuffled_cases(cache_miss_index=[0, 1, 2]))
    def test_get_variables_by_group(self, cache_miss_index: int):
        cache_groups = [GroupCode("g1"), GroupCode("g2"), GroupCode("g3")]
        all_variables = [group1_vars, group2_vars, group3_vars]

        cache_miss_group = cache_groups[cache_miss_index]
        transformer_retval = pandas.DataFrame(all_variables[cache_miss_index])

        cache_side_effect = [
            pandas.DataFrame(variables) if i != cache_miss_index else None
            for i, variables in enumerate(all_variables)
        ]

        self.mocker.patch.object(
            self._service._cache, "get", side_effect=cache_side_effect
        )
        api_mock = self.mocker.patch.object(
            self._service._api, "variables_for_group", return_value="something"
        )
        self.mocker.patch.object(
            self._service._transformer, "variables", return_value=transformer_retval
        )
        expected_codes = ["var1", "var2", "var3", "var4", "var5"]

        res = self._service.get_variables_by_group(*cache_groups)

        assert res["code"].tolist() == expected_codes
        api_mock.assert_called_once_with(cache_miss_group)
        self.cast_mock(self._service._cache.put).assert_called_once_with(
            String() & Glob(f"**{cache_miss_group}**"), transformer_retval
        )

    def test_get_all_variables(self):
        def cachePutSideEffect(resource: str, _: Any) -> bool:
            return "group2" in resource or "group1" in resource

        self.mocker.patch.object(
            self._service._api, "all_variables", return_value=["something"]
        )
        cache_put = self.mocker.patch.object(
            self._service._cache, "put", side_effect=cachePutSideEffect
        )

        transformer_res = pandas.DataFrame(group1_vars + group2_vars + group3_vars)
        self.mocker.patch.object(
            self._service._transformer, "variables", return_value=transformer_res
        )
        expectedVariables: Dict[str, GroupVariable] = dict(
            Name1_group1=GroupVariable(**group1_vars[0]),
            Name2_group1=GroupVariable(**group1_vars[1]),
            Name3_group2=GroupVariable(**group2_vars[0]),
            Name4_group2=GroupVariable(**group2_vars[1]),
        )

        res = self._service.get_all_variables()

        assert res.to_dict() == transformer_res.drop(columns=["cleaned_name"]).to_dict()  # type: ignore
        assert dict(self._service.variables.items()) == expectedVariables

        cache_put.call_args_list == [
            call(
                "variables/group1.csv", DataFrameColumnMatcher(["var1", "var2"], "code")
            ),
            call(
                "variables/group2.csv", DataFrameColumnMatcher(["var3", "var4"], "code")
            ),
            call("variables/group3.csv", DataFrameColumnMatcher(["var5"], "code")),
        ]


from django.conf.urls import url
from schedulizer import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^addclasses/$', views.addClasses, name='addClasses'),
    url(r'^finalschedule/$', views.finalSchedule, name='finalSchedule'),
    url(r'^getdars/$', views.getDars, name='getDars'),
    url(r'^recommend/$', views.recommend, name='recommend'),
]