import acm
import ael
import DatabaseDependenciesNetworkX as DatabaseDependencies

import FLogger
import json
import os
from PackageFileRepository import PACKAGE_DEF_FILE
import PackageInstallerParametersManager

import tempfile

import traceback
import Transporters
import TransporterExport

parameters = PackageInstallerParametersManager.ParametersManager()
logger = FLogger.FLogger(name=__name__, level = int(parameters.read('message_verbosity')))
try:
    import PackageInstallerReferences
except ImportError:
    try:
        import PackageInstallerReferencesTemplate as PackageInstallerReferences
    except:
        logger.error(traceback.format_exc())

amba_message_datetime_format = parameters.read('AmbaMessageDatetimeFormat')

ignoreClassesDefault = ['FPreferences', 'FTransactionHistory', 'FTransactionHistorySubscription', 'FUserLog', 'FServerClientProcess', 'FServerClientLog']
keepChildRecord       = ['FParameterMapping', 'FInstrumentSpreadCurveBidAsk']

def acm_to_ael(object):
    if object:
        return getattr(ael, str(object.Table().Name()))[object.Oid()]

def get_parent_object(object):
    parent = acm_to_ael(object).parent()
    if parent:
        return acm.Ael().AelToFObject(parent)

def get_child_objects(object):
    return [acm.Ael().AelToFObject(child) for child in acm_to_ael(object).children()]

def BuildPackageFromGraph(graph, package_manifest_objects, exclusions = None, exclusion_types = None, update_status = None):
    try:
        objects = DatabaseDependencies.GetDependencyList(graph)
        # If there are package objects not in the graph, append them to the list of dependencies.
        for package_object in package_manifest_objects:
            if package_object not in objects:
                objects.append(package_object)
                logger.debug('Added missing object')
        if not objects:
            logger.warn('No objects, no package will be built')
            return

        path = tempfile.mkdtemp()
        transportersPerClass = dict((handler.ClassName(), (name, handler)) for name, handler in Transporters.Transporter.all_handlers.items() if isinstance(handler, Transporters.XMLTransporter))
        transporterParams = dict((var[0], '') for var in TransporterExport.ael_variables)
        transporterParams['LogDefaults'] = 'True'
        ambamessages = acm.FAMBAMessage()
        ambamessages.Type("MESSAGES")
        generator = acm.FAMBAMessageGenerator()
        packageObjects = {}
        packageDefActions = []
        for object in objects:

            table = str(object.Table().Name())
            className = str(object.ClassName())

            ambadef = PackageInstallerReferences.amba_definition_for_class(className)
            generator.Parameters(ambadef)

            if className in transportersPerClass:
                packageObjects.setdefault(className, []).append(object)
            elif table == 'TextObject':
                logger.info('%-40s%-20s%-20s UNHANDLED object type, ignoring ' % (object.StringKey(), className, table))
                continue
            else:
                packageObjects['AMBA'] = 'AMBA.xml'
                if object.IsKindOf('FTrade') and not object.OptionalKey():
                    object.OptionalKey = object.Oid()
                    message = generator.Generate(object)
                    object.Undo()
                else:
                    message = generator.Generate(object)
                if message == None:
                    logger.error('Null message.')
                    raise "Null message."
                reference = DatabaseDependencies.get_attribute(graph, object, 'reference')
                reference_description = 'reference: %s' % reference
                message.AtPut('REFERENCE_TYPE', reference)
                # Also check if this is a skipped link record.
                exclude_from_import = False
                if (exclusions and object in exclusions) or (exclusion_types and object.ClassName().AsString() in exclusion_types):
                    exclude_from_import = True
                if (PackageInstallerReferences.should_import_link_record(object) == True)  and (not exclude_from_import):
                    message.AtPut('EXCLUDE_FROM_IMPORT', 0)
                else:
                    message.AtPut('EXCLUDE_FROM_IMPORT', 1)
                logger.info('AMBA message for: %-60s%-20s %s' % (DatabaseDependencies.node_id(object), table, reference_description))
                #logger.debug(str(message))
                if className not in keepChildRecord:
                    DatabaseDependencies.remove_sub_messages(message)
                # To prevent the dreaded "inconsistent object state" with consequent mayhem,
                # we save the entity key in the message so that the message object can be fetched
                # back into the source ADS without recreating the object from the message. This will
                # have the state of the ADS, not the state of the message.
                message.AtPut('ORIGINAL_KEY', DatabaseDependencies.key_for_entity(object))
                if update_status != None:
                    update_status('Created AMBA message for: %s...' % DatabaseDependencies.key_for_entity(object))
                ambamessages.AddMessage(message)

        for objectType in packageObjects:
            if objectType == 'AMBA':
                packageDefActions.append({"type" : "ambamsg", "file" : packageObjects['AMBA']})
                ambaFile = os.path.join(path, packageObjects['AMBA'])
                output = acm.FCharacterOutputFileStream(ambaFile)
                acm.FTaggedMessageXMLFormatter().FormatStream(output, ambamessages)
                output.Close()
                logger.info('  Created AMBA file: %s.' % ambaFile)
                if update_status != None:
                    update_status('Created AMBA file: %s...' % ambaFile)
            else:
                name, handler = transportersPerClass[objectType]
                action = {"type" : "transporter", 'transporter': name}
                if name == "Workbook":
                    action["setDefaultContext"] = "True"
                objects = packageObjects[objectType]
                if len(objects) > 1:
                    action['folder'] = name.title().replace(' ', '') + 's'
                    folder = os.path.join(path, action['folder'])
                    os.mkdir(folder)
                else:
                    folder = path
                transporterParams['basepath'] = acm.FSymbol(folder)
                for obj in objects:
                    objectName = handler.Names([obj])[0]
                    output = handler.ExportSingle(transporterParams, objectName)
                    if output:
                        #Modified StringToFile to return the fileName
                        file = handler.StringToFile(folder, output, objectName)
                        fileName = os.path.relpath(file, folder)
                        logger.info('  Created %s file %s' % (name, file))
                        if update_status != None:
                            update_status('Created %s file: %s...' % (name, file))
                        if not 'folder' in action:
                            action['file'] = fileName
                packageDefActions.append(action)
        fileName = os.path.join(path, PACKAGE_DEF_FILE)
        source_ads = acm.ADSAddress().strip()
        package_manifest_object_ids = []
        for object in package_manifest_objects:
            package_object_id = str(DatabaseDependencies.node_id(object))
            logger.debug('Package manifest object ID: %s' % package_object_id)
            package_manifest_object_ids.append(package_object_id)
        logger.debug('Temporary package filename: %s' % fileName)

        with open(fileName, 'w') as f:
            data = {'source_ads': source_ads, 'actions': packageDefActions, 'objects': package_manifest_object_ids}
            text = json.dumps(data, indent=4, separators=(',', ': '), encoding='latin-1')
            f.write(text)

        logger.info('  Created package definition file.')

        if update_status != None:
            update_status('Finished creating package at: %s' % path)
        return path
    except:
        logger.error(traceback.format_exc())


def main():
    num_1 = int(input("Digite um número: "))
    num_2 = int(input("Digite um número: "))
    num_3 = int(input("Digite um número: "))

    verificar_maior(num_1,num_2,num_3)

def verificar_maior(a,b,c):
    if a > b and a > c and b > c and (a != b and a != c and b != c):
        print(a,b,c)
    elif a > b and a > c and c > b and (a != b and a != c and b != c):
        print(a,c,b)
    elif b > a and b > c and a > c and (b != a and b != c and a != c):
        print(b,a,c)
    elif b > a and b > c and c > a and (b != a and b != c and a != c):
        print(b,c,a)
    elif c > a and c > b and a > b and (c != a and c != a and b != a):
        print(c,a,b)
    elif c > a and c > b and b > a and (c != a and c != a and b != a):
        print(c,b,a)
    else:
        print("Dois ou mais números são iguais!")

main()


# https://www.hackerrank.com/challenges/s10-geometric-distribution-1/problem?h_r=next-challenge&h_v=zen&h_r=next-challenge&h_v=zen

import sys

def geometric(input1, input2):
    ratio = input1[0] / input1[1]
    result = ((1 - ratio) ** (input2 - 1)) * ratio
    return result

input1 = list(map(int, sys.stdin.readline().rstrip('\n').split()))
input2 = int(sys.stdin.readline().rstrip('\n'))

result = geometric(input1, input2)

sys.stdout.write(str(round(result, 3)))


#!/usr/bin/python3

import serial
import time
import matplotlib
matplotlib.use('TKAgg')
import matplotlib.pyplot as plt
import numpy as np

n_step = 12
n_seq  = 11
addr   = '/dev/ttyACM0'

fig = plt.figure()
win = fig.canvas.manager.window

ser = serial.Serial(addr,9600,timeout=1)

ser.write(b'led %i\r' % n_seq)
time.sleep(0.1)

def sensor_barplot():

    int_array = np.full(n_step,4095)
    rects = plt.bar(range(n_step), int_array,  align = 'center')

    while True:
        time.sleep(0.1)
        ser.write(b'adc\r')
        serline = ser.readline()
        str_array = str(serline,'ascii').rstrip("\n\r").split(",")
        int_array = np.asarray(list(map(int, str_array)))

        for rect, h in zip(rects, int_array):
            rect.set_height(h)
            fig.canvas.draw()

win.after(10, sensor_barplot)
plt.show()


#!env/bin/python
import subprocess
import requests
import time
from base64 import b64decode

URL = 'http://127.0.0.1:4646/v1'
JobID = 'fibonacci'

json_run_job = subprocess.check_output(['nomad', 'job', 'run', '-output', 'fibonacci.hcl'])
response = requests.post(URL + '/jobs', json_run_job)

response = requests.get(URL + f'/job/{JobID}/scale')

if response.json()['JobStopped'] == False:
    print(f'{JobID} stoped')
    response = requests.delete(URL + f'/job/{JobID}?purge=true')
    response = requests.post(URL + '/jobs', json_run_job)

response = requests.get(URL + '/job/fibonacci')
status_job = response.json()['Status']
time.sleep(1)
alloc_id = requests.get(URL + f'/job/{JobID}/allocations').json()[0]['ID']

print(f'Now status {JobID} job: {status_job}')

while status_job != 'dead':
    time.sleep(5)
    response = requests.get(URL + '/job/fibonacci')
    status_job = response.json()['Status']
    print(f'Now status {JobID} job: {status_job}')

params_for_get_result = {'task': JobID, 'type': 'stdout'}
response = requests.get(URL + f'/client/fs/logs/{alloc_id}', params_for_get_result)
result = b64decode(response.json()['Data']).decode().strip()
print(f'{JobID} job is complete!')
print(f'Result: {result}')


# -*- coding: utf-8 -*-
from django.template import Library
from django.core.exceptions import ObjectDoesNotExist

register = Library()

@register.filter
def translate(obj, lng):
    if not obj:
        return unicode(None)
    try:
        translation = obj.translations.only('text').get(lng=lng)
    except ObjectDoesNotExist as e:
        return unicode(None)
    return translation.text or unicode(None)


from audio import startup
from intro import *
from main_menu import *
from user import *
import time

def main():
    # Is the first time using the program? Yes? Gives intro
    user = User()
    if user.first_time == "True":
        intro()
        user.make_it_old("neltume")
        user.set_date()
    current_date = time.localtime()
    date = [current_date[0], current_date[1], current_date[2]]
    user_date = user.date
    # Change the day of the last session
    if date[0] != user_date[0] or date[1] != user_date[1] or date[2] != user_date[2]:
        user.set_date()
    startup()
    main_menu()

if __name__ == '__main__':
    main()


import time
def check_time_to_now_even(seconds):
    """
    Checks whether time difference between now and given time in seconds is even
    """
    if seconds < 0:
        raise ValueError('received negative input')

    now=time.time()
    return (seconds-now)%2 == 0


import time
import struct
import copy

import numpy as np
from periphery import I2C

import interpolate

# from defs.h of Table Number 114
PCSCALEVAL = 1e8

# registers
REG_CONFIG = 0x01
REG_STATUS = 0x02
REG_MBIT = 0x03
REG_BIAS_TOP = 0x04
REG_BIAS_BOT = 0x05
REG_CLK = 0x06
REG_BPA_TOP = 0x07
REG_BPA_BOT = 0x08
REG_PU = 0x09

# calibration settings read previously from EEPROM
CAL_MBIT = 0x2C
CAL_BIAS_TOP = 0x05
CAL_BIAS_BOT = 0x05
CAL_CLK = 0x15
CAL_BPA_TOP = 0x03
CAL_BPA_BOT = 0x03
CAL_PU = 0x88

def generate_command(register, value):
    return [I2C.Message([register, value])]

def send_command(dev_i2c, address, cmd, wait=True):
    dev_i2c.transfer(address, cmd)
    if wait:
        time.sleep(0.005)

class HTPA:
    def __init__(self, address, eeprom_address=0x50, verbose=True):
        self.i2c = I2C("/dev/i2c-1")
        self.address = address
        self.eeprom_address = eeprom_address
        self.verbose = verbose

        self.mbit = CAL_MBIT
        self.bias_top = CAL_BIAS_TOP
        self.bias_bot = CAL_BIAS_BOT
        self.clk = CAL_CLK
        self.bpa_top = CAL_BPA_TOP
        self.bpa_bot = CAL_BPA_BOT
        self.pull_up = CAL_PU

        self.wake_up_sensor()
        self.set_calibration_settings()

        eeprom = self.get_eeprom()
        self.extract_eeprom_parameters(eeprom)

        self.update_compensation_parameters()

    def wake_up_sensor(self):
        self.write_register(REG_CONFIG, 0x01)

    def close(self):
        self.write_register(REG_CONFIG, 0x00)

    def write_register(self, register, value):
        cmd = generate_command(register, value)
        send_command(self.i2c, self.address, cmd)

    def set_calibration_settings(self):
        self.write_register(REG_MBIT, self.mbit)
        self.write_register(REG_BIAS_TOP, self.bias_top)
        self.write_register(REG_BIAS_BOT, self.bias_bot)
        self.write_register(REG_CLK, self.clk)
        self.write_register(REG_BPA_TOP, self.bpa_top)
        self.write_register(REG_BPA_BOT, self.bpa_bot)
        self.write_register(REG_PU, self.pull_up)

    def ambient_temperature(self, ptats):
        return self.ptat_grad * np.mean(ptats) + self.ptat_offset

    def temperature_compensation(self, im, ptats):
        compensation = self.th_grad * np.mean(ptats) / (2 ** self.grad_scale)
        compensation += self.th_offset

        return im - compensation

    def electrical_offset_compensation(self, im):
        return im - self.electrical_offset

    def sensitivity_compensation(self, im):
        return PCSCALEVAL * im / self.pix_c

    def voltage_compensation(self, im, ptats):
        PTATAvg = np.mean(ptats)

        aux1 = self.VddCompGrad * PTATAvg / (2 ** self.VddScGrad)
        aux1 = (aux1 + self.VddCompOff) / (2 ** self.VddScOff)

        aux2_1 = self.VddAvg - self.VddTh1
        aux2_2 = (self.VddTh2 - self.VddTh1)/(self.PTATTh2 - self.PTATTh1)
        aux2_3 = PTATAvg - self.PTATTh1
        aux2 = aux2_1 - aux2_2*aux2_3
        compensation = aux1 * aux2

        return im - compensation

    def get_eeprom(self):
        q1 = [I2C.Message([0x00, 0x00]), I2C.Message([0x00]*4000, read=True)]
        q2 = [I2C.Message([0x0f, 0xa0]), I2C.Message([0x00]*4000, read=True)]
        self.i2c.transfer(self.eeprom_address, q1)
        self.i2c.transfer(self.eeprom_address, q2)

        return np.array(q1[1].data + q2[1].data)

    def extract_calibration_settings(self, eeprom):
        mbit = eeprom[0x001A]
        bias = eeprom[0x001B]
        clk = eeprom[0x001C]
        bpa = eeprom[0x001D]
        pu = eeprom[0x001E]

        if self.verbose:
            print("Calibration data from eeprom: ")
            print("MBIT (calib): ", hex(mbit))
            print("BIAS (calib): ", hex(bias))
            print("CLK (calib): ", hex(clk))
            print("BPA (calib): ", hex(bpa))
            print("PU (calib): ", hex(pu))

    def extract_temperature_compensation_data(self, eeprom):
        th_grad = eeprom[0x0740:0x0F40:2] + (eeprom[0x0741:0x0F40:2] << 8)
        th_grad = unsigned_to_signed_array(th_grad)
        th_grad = np.reshape(th_grad, (32, 32))
        th_grad[16:,:] = np.flipud(th_grad[16:,:]) # bottom values are flipped
        self.th_grad = th_grad

        th_offset = eeprom[0x0F40:0x1740:2] + (eeprom[0x0F41:0x1740:2] << 8)
        th_offset = unsigned_to_signed_array(th_offset)
        th_offset = np.reshape(th_offset, (32, 32))
        th_offset[16:,:] = np.flipud(th_offset[16:,:]) # bottom values are flipped
        self.th_offset = th_offset

        self.grad_scale = eeprom[0x0008]
        self.ptat_grad = self.eeprom_value_to_float(eeprom[0x0034:0x0038])
        self.ptat_offset = self.eeprom_value_to_float(eeprom[0x0038:0x003c])

    def extract_sensitivity_compensation_data(self, eeprom):
        pmin = self.eeprom_value_to_float(eeprom[0x0000:0x0004])
        pmax = self.eeprom_value_to_float(eeprom[0x0004:0x0008])
        epsilon = float(eeprom[0x000D])
        global_gain = eeprom[0x0055] + (eeprom[0x0056] << 8)

        P = eeprom[0x1740::2] + (eeprom[0x1741::2] << 8)
        P = np.reshape(P, (32, 32))
        P[16:, :] = np.flipud(P[16:,:])

        self.pix_c = (P * (pmax - pmin) / 65535. + pmin) * (epsilon / 100)
        self.pix_c *= float(global_gain) / 10000

    def extract_voltage_compensation_data(self, eeprom):
        self.PTATTh1 = eeprom[0x003C] + (eeprom[0x003D] << 8)
        self.PTATTh2 = eeprom[0x003E] + (eeprom[0x003F] << 8)
        self.VddTh1 = eeprom[0x0026] + (eeprom[0x0027] << 8)
        self.VddTh2 = eeprom[0x0028] + (eeprom[0x0029] << 8)
        self.VddScGrad = eeprom[0x004E]
        self.VddScOff = eeprom[0x004F]

        if self.verbose:
            print("Voltage compensation data: ")
            print("PTATTh1: {}".format(self.PTATTh1))
            print("PTATTh2: {}".format(self.PTATTh2))
            print("VddTh1: {}".format(self.VddTh1))
            print("VddTh2: {}".format(self.VddTh2))
            print("VddScGrad: {}".format(self.VddScGrad))
            print("VddScOff: {}".format(self.VddScOff))

        vdd_comp_grad_ep = eeprom[0x0340:0x0540:2] + (eeprom[0x0341:0x0540:2] << 8)
        vdd_comp_grad_ep = [v - 65536 if v >= 32768 else v for v in vdd_comp_grad_ep] # check this line. seems ok
        vdd_comp_grad_ep = np.reshape(vdd_comp_grad_ep, (8, 32))
        vdd_comp_grad_top_block = vdd_comp_grad_ep[0:4, :]
        vdd_comp_grad_bot_block = np.flipud(vdd_comp_grad_ep[4:, :])
        self.VddCompGrad = np.zeros((32, 32))
        for b in range(0, 4):
            self.VddCompGrad[4*b : 4*(b+1), :] = vdd_comp_grad_top_block
            self.VddCompGrad[16 + 4*b : 16 + 4*(b+1), :] = vdd_comp_grad_bot_block

        vdd_comp_off_ep = eeprom[0x0540:0x0740:2] + (eeprom[0x0541:0x0740:2] << 8)
        vdd_comp_off_ep = [v - 65536 if v >= 32768 else v for v in vdd_comp_off_ep] # check this line. seems ok
        vdd_comp_off_ep = np.reshape(vdd_comp_off_ep, (8, 32))
        vdd_comp_off_top_block = vdd_comp_off_ep[0:4, :]
        vdd_comp_off_bot_block = np.flipud(vdd_comp_off_ep[4:, :])
        self.VddCompOff = np.zeros((32, 32))
        for b in range(0, 4):
            self.VddCompOff[4*b : 4*(b+1), :] = vdd_comp_off_top_block
            self.VddCompOff[16 + 4*b : 16 + 4*(b+1), :] = vdd_comp_off_bot_block

    def extract_eeprom_parameters(self, eeprom):
        self.extract_calibration_settings(eeprom)
        self.extract_temperature_compensation_data(eeprom)
        self.extract_sensitivity_compensation_data(eeprom)
        self.extract_voltage_compensation_data(eeprom)

    def update_compensation_parameters(self):
        if self.verbose:
            print("Updating compensation parameters.")

        (offset, vdds) = self.capture_image(blind=True, vdd=True)
        self.electrical_offset = offset
        self.VddAvg = np.mean(vdds)

    def object_temperature(self, ta, ad):
        return interpolate.get_temperature(ta, ad)

    def im_to_temperatures(self, im, ta):
        temperatures = np.zeros((32,32))
        for row in range(32):
            for col in range(32):
                t = self.object_temperature(ta, im[row, col])/10 - 273
                temperatures[row, col] = t

        return temperatures

    def capture_temperatures(self):
        im, ptats = self.capture_image(blind=False, vdd=False)
        im = self.temperature_compensation(im, ptats)
        im = self.electrical_offset_compensation(im)
        im = self.voltage_compensation(im, ptats)
        im = self.sensitivity_compensation(im)

        ta = self.ambient_temperature(ptats)
        temps = self.im_to_temperatures(im, ta)

        return temps, ta

    def capture_image(self, blind=False, vdd=False):
        pixel_values = np.zeros(1024)
        ptats = np.zeros(8)

        for block in range(4):
            self.expose_block(block, blind=blind, vdd=vdd)

            while not self.block_capture_finished(block, blind, vdd):
                time.sleep(0.005)

            read_block = [I2C.Message([0x0A]), I2C.Message([0x00]*258, read=True)]
            self.i2c.transfer(self.address, read_block)
            top_data = np.array(copy.copy(read_block[1].data))

            read_block = [I2C.Message([0x0B]), I2C.Message([0x00]*258, read=True)]
            self.i2c.transfer(self.address, read_block)
            bottom_data = np.array(copy.copy(read_block[1].data))

            top_data = top_data[1::2] + (top_data[0::2] << 8)
            bottom_data = bottom_data[1::2] + (bottom_data[0::2] << 8)

            pixel_values[(0+block*128):(128+block*128)] = top_data[1:]
            # bottom data is in a weird shape
            pixel_values[(992-block*128):(1024-block*128)] = bottom_data[1:33]
            pixel_values[(960-block*128):(992-block*128)] = bottom_data[33:65]
            pixel_values[(928-block*128):(960-block*128)] = bottom_data[65:97]
            pixel_values[(896-block*128):(928-block*128)] = bottom_data[97:]

            ptats[block] = top_data[0]
            ptats[7-block] = bottom_data[0]

        pixel_values = np.reshape(pixel_values, (32, 32))
        return (pixel_values, ptats)

    def expose_block(self, block, blind=False, vdd=False):
        cmd = 0x09 + (block << 4)
        cmd = cmd + 0x02 if blind else cmd
        cmd = cmd + 0x04 if vdd else cmd

        self.write_register(REG_CONFIG, cmd)

    def query_capture(self):
        query = [I2C.Message([REG_STATUS]), I2C.Message([0x00], read=True)]
        self.i2c.transfer(self.address, query)

        return query[1].data[0]

    def block_capture_finished(self, block, blind, vdd):
        expected = 1 + (block << 4)
        expected = expected + 0x02 if blind else expected
        expected = expected + 0x04 if vdd else expected

        ans = self.query_capture()

        return expected == ans

    def eeprom_value_to_float(self, value):
        return struct.unpack('f', reduce(lambda a,b: a+b, [chr(v) for v in value]))[0]

def unsigned_to_signed_array(arr, bits=16):
    arr = [a - 2**bits if a >= 2**(bits-1) else a for a in arr]
    return arr

if __name__ == "__main__":
    dev = HTPA(0x1A)

    # should be run periodically. For instance from 10 to 10 seconds.
    dev.update_compensation_parameters()

    # 32x32 array of object temperatures in celsius
    # ambient temperature given in decikelvins
    temperatures, ta = dev.capture_temperatures()

    print("Ambient temperature: {}".format(ta))
    print("Temperatures: ")
    print(temperatures)

    dev.close()


import numpy as np
import matplotlib.pyplot as plt
import os
from pybullet_env import CutterEnvBase, check_input_variance
from stable_baselines3 import PPO
# import cv2
import time
from PIL import Image
from pybullet_env import CutterEnv
import pickle
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms

data_root = 'training_data'

def model_eval(net, dataset, loss_func):
    net.eval()
    with torch.no_grad():
        running_loss = 0.0
        total_items = 0
        for inputs, truth in dataset:
            n = inputs.shape[0]
            total_items += n

            outputs = net(inputs.cuda())[0].cpu()
            running_loss += loss_func(outputs, truth).item() * n

    return running_loss / total_items

class ImgActionDataset(Dataset):

    def __init__(self, file_ids, truth_dict, unnormalize = True, preprocess_func=None):
        self.data = []
        for file_id in file_ids:
            truth_val = truth_dict[file_id]
            self.data.append([file_id, torch.from_numpy(truth_val).float()])
        self.unnormalize = unnormalize
        self.preprocess_func = preprocess_func

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):

        file_id, truth_val = self.data[idx]
        file_path = os.path.join(data_root, '{}.png'.format(file_id))
        img = Image.open(file_path)
        if self.preprocess_func is not None:
            img = np.array(img)
            img = Image.fromarray(self.preprocess_func(img))

        img_tensor = transforms.ToTensor()(img).unsqueeze_(0)[0]
        if self.unnormalize:
            img_tensor = img_tensor * 255

        return img_tensor, truth_val

if __name__ == '__main__':
    action = 'get_data'
    # action = 'train'
    # action = 'eval'
    difficulty = 1.0
    width = 424
    height = 240
    grayscale = False
    use_net = True
    use_seg = False
    use_depth = False
    use_flow = False
    use_last_frame = False
    crop = ((64,424), (60,240))
    downscale = (160, 80)

    actions_dict_file = os.path.join(data_root, 'actions.pickle')

    if action == 'get_data':

        env = CutterEnv(width, height, grayscale=grayscale, use_net=use_net, use_seg=use_seg, use_depth=use_depth,
                        use_flow=use_flow, use_last_frame=use_last_frame,
                        use_gui=True, max_elapsed_time=1.0, max_vel=0.3, debug=True,
                        eval=True, eval_count=None, crop=crop, downscale=downscale, difficulty=difficulty)
        model = PPO("CnnPolicy", env, verbose=1)

        current_img_actions = {}
        try:
            with open(actions_dict_file, 'rb') as fh:
                current_img_actions = pickle.load(fh)
        except FileNotFoundError:
            pass

        # model = model.load('best_model_0_5.zip')
        obs = env.reset()
        counter = 0
        steps_counter = 0
        while True:
            if not steps_counter % 1:
                while True:
                    target_action = input("What input is appropriate? (Type either UD, RL, nothing, or S to skip, X to reset): ").lower()
                    if ('u' in target_action and 'd' in target_action) or ('l' in target_action and 'r' in target_action):
                        print("Can't have opposing directions in input!")
                        continue
                    break
                if not ('s' in target_action or 'x' in target_action):
                    target_array = np.array([0.0, 0.0])
                    if 'u' in target_action:
                        target_array[1] = 1.0
                    if 'd' in target_action:
                        target_array[1] = -1.0
                    if 'l' in target_action:
                        target_array[0] = 1.0
                    if 'r' in target_action:
                        target_array[0] = -1.0

                    for _ in range(12):
                        env.randomize_camera()
                        env.randomize_tree_textures()
                        env.randomize_scenery()

                        rgb_array = env.get_obs()
                        while counter in current_img_actions:
                            counter += 1

                        Image.fromarray(rgb_array).save(os.path.join(data_root, '{}.png'.format(counter)))
                        current_img_actions[counter] = target_array

                    with open(actions_dict_file, 'wb') as fh:
                        pickle.dump(current_img_actions, fh)

                    # Array stats
                    summary = np.array(list(current_img_actions.values()))
                    means = summary.mean(axis=0)
                    print('Current means: {:.4f}, {:.4f}'.format(*means))
                else:
                    print('Skipped')

            steps_counter += 1

            # action, _states = model.predict(obs, deterministic=True)
            action = env.action_space.sample()
            env.set_action(action)
            obs, reward, done, info = env.step(action, realtime=False)
            if done or 'x' in target_action:
                obs = env.reset()

    elif action == 'train':

        env = CutterEnvBase(width, height, grayscale=grayscale, use_net=use_net, use_seg=use_seg, use_depth=use_depth,
                        use_flow=use_flow, use_last_frame=use_last_frame, use_gui=False, crop=crop, downscale=downscale)
        model = PPO("CnnPolicy", env, verbose=1)

        with open(actions_dict_file, 'rb') as fh:
            current_img_actions = pickle.load(fh)

        # data = ImgActionDataset([0], current_img_actions, preprocess_func=env.process_rgb_image)
        all_idxs = np.array([int(x.replace('.png', '')) for x in os.listdir(data_root) if x.endswith('.png')])
        training_truth_idx = np.ones(len(all_idxs), dtype=bool)

        # Pick 15% of the images to be part of the validation set
        n_val = int(len(all_idxs) * 0.15)
        training_truth_idx[np.random.choice(len(training_truth_idx), n_val, replace=False)] = False
        train_idx = all_idxs[training_truth_idx]
        val_idx = all_idxs[~training_truth_idx]
        print('VALIDATION IDX: {}'.format(val_idx))

        train_data = ImgActionDataset(train_idx, current_img_actions)
        val_data = ImgActionDataset(val_idx, current_img_actions)
        train_loader = DataLoader(train_data, batch_size=4, shuffle=True, num_workers=0)
        val_loader = DataLoader(val_data, batch_size=4, shuffle=False, num_workers=0)

        net = model.policy.cuda()
        loss_func = nn.MSELoss()
        opt = optim.Adam(net.parameters(), lr=0.0005)

        init_loss = model_eval(net, val_loader, loss_func)
        print('Initial loss: {:.4f}'.format(init_loss))
        best_loss = init_loss

        for epoch in range(200):
            print('Starting epoch {}'.format(epoch+1))

            for i, data in enumerate(train_loader):
                net.train()
                inputs, truth = data
                opt.zero_grad()

                outputs = net(inputs.cuda())[0]
                loss = loss_func(outputs, truth.cuda())
                loss.backward()
                opt.step()

            eval_loss = model_eval(net, val_loader, loss_func)
            if eval_loss < best_loss:
                best_loss = eval_loss
                print('New best loss of {:.4f}'.format(eval_loss))
                torch.save(net.state_dict(), 'pretrained_agent.weights')
                # model.save('pretrained_agent.model')
            else:
                print('Loss was {:.4f}'.format(eval_loss))

    elif action == 'eval':

        env = CutterEnv(width, height, grayscale=grayscale, use_net=use_net, use_seg=use_seg, use_depth=use_depth, use_flow=use_flow, use_last_frame=use_last_frame,
                        use_gui=True, max_elapsed_time=2.0, max_vel=0.25, debug=True, eval=True, eval_count=None, difficulty=1.0, crop=crop, downscale=downscale)
        model = PPO("CnnPolicy", env, verbose=1)
        obs = env.reset()

        net = model.policy
        net.load_state_dict(torch.load('pretrained_agent.weights'))
        # net.eval()

        last_obs = None
        counter = 0
        while True:

            counter += 1
            if last_obs is not None:
                last_tree = last_obs[:,:,1].astype(int)
                curr_tree = obs[:,:,1].astype(int)

                tree_change = ((last_tree - curr_tree) / 2).astype(np.uint8)

            action = model.predict(obs, deterministic=True)[0]
            env.set_action(action)
            last_obs = obs
            obs, reward, done, info = env.step(action, realtime=False)
            # env.randomize_camera()
            if done:
                obs = env.reset()


import random
import os
import json
import math
from typing import Dict, List, Callable, Tuple, Iterable
from abc import ABC, abstractmethod
from collections import namedtuple

import tensorflow as tf
import numpy as np
import tqdm
from bert.modeling import BertModel, BertConfig
from bert.optimization import create_optimizer

from src.data.base import Example
from src.utils import train_test_split, get_filtered_by_length_chunks, log
from src.model.layers import StackedBiRNN

class ModeKeys:
    TRAIN = "train"  # need labels, dropout on
    VALID = "valid"  # need labels, dropout off
    TEST = "test"  # don't need labels, dropout off

BertInputs = namedtuple(
    "BertInputs",
    ["input_ids", "input_mask", "segment_ids", "first_pieces_coords", "num_tokens", "num_pieces"]
)

class BaseModel(ABC):
    """
    Interface for all models

    config = {
        "model": {
            "embedder": {
                ...
            },
            "ner": {
                "loss_coef": 0.0,
                ...
            },
            "re": {
                "loss_coef": 0.0,
                ...
            }
        },
        "training": {
            "num_epochs": 100,
            "batch_size": 16,
            "max_epochs_wo_improvement": 10
        },
        "inference": {
            "window": 1,
            "max_tokens_per_batch": 10000
        },
        "optimizer": {
            "init_lr": 2e-5,
            "num_train_steps": 100000,
            "num_warmup_steps": 10000
        }
    }
    """

    model_scope = "model"

    def __init__(self, sess: tf.Session = None, config: Dict = None):
        self.sess = sess
        self.config = config

        self.loss = None
        self.train_op = None
        self.training_ph = None

    # специфичные для каждой модели методы

    @abstractmethod
    def _build_graph(self):
        """построение вычислительного графа (без loss и train_op)"""

    @abstractmethod
    def _build_embedder(self):
        """вход - токены, выход - векторизованные токены"""

    @abstractmethod
    def _get_feed_dict(self, examples: List[Example], mode: str) -> Dict:
        """mode: {train, valid, test} (см. ModeKeys)"""

    @abstractmethod
    def _set_placeholders(self):
        pass

    @abstractmethod
    def _set_layers(self):
        pass

    @abstractmethod
    def _set_loss(self, *args, **kwargs):
        pass

    @abstractmethod
    def _set_train_op(self):
        pass

    @abstractmethod
    def predict(self, examples: List[Example], **kwargs) -> None:
        """
        Вся логика инференса должна быть реализована здесь.
        Предполагается, что модель училась не на целых документах, а на кусках (chunks).
        Следовательно, предикт модель должна делать тоже на уровне chunks.
        Но в конечном итоге нас интересуют предсказанные лейблы на исходных документах (examples).
        Поэтому схема такая:
        1. получить модельные предикты на уровне chunks
        2. аггрегировать результат из п.1 и записать на уровне examples

        :param examples: исходные документы. атрибут chunks должен быть заполнен!
        :param kwargs:
        :return:
        """

    @abstractmethod
    def evaluate(self, examples: List[Example], **kwargs) -> Dict:
        """
        Возвращаемый словарь должен обязательно содержать ключи "score" и "loss"
        :param examples: исходные документы. атрибут chunks должен быть заполнен!
        :return:
        """

    @property
    @abstractmethod
    def _is_bpe_level(self) -> bool:
        """
        используется ли bpe-токенизация. нужно для корректной фильтрации примеров по длине при обучении/инференсе.
        реализовано так, потому что надо прокидывать значение в функции train, которая общая для всех моделей.
        TODO: подумать, мб можно сделать лучше
        """

    # общие методы для всех моделей

    def build(self, mode: str = ModeKeys.TRAIN):
        self._set_placeholders()
        with tf.variable_scope(self.model_scope):
            self._set_layers()
            self._build_graph()
            if mode != ModeKeys.TEST:
                self._set_loss()
            if mode == ModeKeys.TRAIN:
                self._set_train_op()

    # альтернативная версия данной функции вынесена в src._old.wip
    # TODO: мб объекты группировать в батчи по числу элементарных объектов, на которых считается loss?
    @log
    def train(
            self,
            examples_train: List[Example],
            examples_valid: List[Example],
            train_op_name: str = "train_op",
            model_dir: str = None,
            scope_to_save: str = None,
            verbose: bool = True,
            verbose_fn: Callable = None,
    ):
        """

        :param examples_train:
        :param examples_valid:
        :param train_op_name:
        :param model_dir:
        :param scope_to_save:
        :param verbose:
        :param verbose_fn: вход - словарь с метриками (выход self.evaluate); выход - None. функция должна вывести
                           релевантные метрики в stdout
        :return:

        нет возможности переопределять batch_size, потому что есть следующая зависимость:
        batch_size -> num_train_steps -> lr schedule for adamw
        поэтому если хочется изменить batch_size, то нужно переопределить train_op. иными словами, проще сделать так:
        tf.reset_default_graph()
        sess = tf.Session()
        model = ...
        model.build()
        model.initialize()
        """
        if model_dir is not None:
            if os.path.isdir(model_dir):
                print(f"model dir {model_dir} exists")
            else:
                os.makedirs(model_dir, exist_ok=True)
                print(f"model dir {model_dir} created")
            checkpoint_path = os.path.join(model_dir, "model.ckpt")
            print(f"checkpoint path: {checkpoint_path}")

            if scope_to_save is not None:
                var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope_to_save)
            else:
                var_list = tf.trainable_variables()
            saver = tf.train.Saver(var_list)
        else:
            checkpoint_path = None
            saver = None
            print("model dir is None, so checkpoints will not be saved")

        # релзиовано так для возможности выбора train_op:
        # 1) обнолвять все веса
        # 2) обновлять только веса головы
        # *3) аккумуляция градиентов
        # TODO: реализовать последнюю (уже реализовывал, когда решал dependency parsing, нужно скопипастить сюда)
        train_op = getattr(self, train_op_name)

        maxlen = self.config["training"]["maxlen"]
        chunks_train = get_filtered_by_length_chunks(
            examples=examples_train, maxlen=maxlen, pieces_level=self._is_bpe_level
        )

        batch_size = self.config["training"]["batch_size"]
        num_epoch_steps = math.ceil(len(chunks_train) / batch_size)
        best_score = -1
        num_steps_wo_improvement = 0
        verbose_fn = verbose_fn if verbose_fn is not None else print
        train_loss = []

        for epoch in range(self.config["training"]["num_epochs"]):
            for _ in tqdm.trange(num_epoch_steps):
                if len(chunks_train) > batch_size:
                    chunks_batch = random.sample(chunks_train, batch_size)
                else:
                    chunks_batch = chunks_train
                feed_dict = self._get_feed_dict(chunks_batch, mode=ModeKeys.TRAIN)
                try:
                    _, loss = self.sess.run([train_op, self.loss], feed_dict=feed_dict)
                    assert not np.isnan(loss), "loss becomes nan"
                    train_loss.append(loss)
                except Exception as e:
                    print("current batch:", [x.id for x in chunks_batch])
                    raise e

            # pycharm bug:
            # Cannot find reference {mean, std} in __init__.pyi | __init__.pxd
            # so, np.mean(train_loss) highlights yellow
            print(f"epoch {epoch} finished. mean train loss: {np.array(train_loss).mean()}. evaluation starts.")
            performance_info = self.evaluate(examples=examples_valid, batch_size=batch_size)
            if verbose:
                verbose_fn(performance_info)
            score = performance_info["score"]

            print("current score:", score)

            if score > best_score:
                print("!!! new best score:", score)
                best_score = score
                num_steps_wo_improvement = 0

                if saver is not None:
                    saver.save(self.sess, checkpoint_path)
                    print(f"saved new head to {checkpoint_path}")
            else:
                num_steps_wo_improvement += 1
                print("best score:", best_score)
                print("steps wo improvement:", num_steps_wo_improvement)

                if num_steps_wo_improvement == self.config["training"]["max_epochs_wo_improvement"]:
                    print("training finished due to max number of steps wo improvement encountered.")
                    break

            print("=" * 50)

        if saver is not None:
            print(f"restoring model from {checkpoint_path}")
            saver.restore(self.sess, checkpoint_path)

    @log
    def cross_validate(
            self,
            examples: List[Example],
            folds: Iterable,
            valid_frac: float = 0.15,
            model_dir: str = None,
            verbose: bool = False,
            verbose_fn: Callable = None,
            **kwargs
    ) -> Tuple[np.ndarray, np.ndarray]:
        """

        :param examples:
        :param folds:
        :param valid_frac:
        :param model_dir:
        :param verbose:
        :param verbose_fn:
        :return:
        """
        assert self.sess is None

        # for x in examples:
        #     assert len(x.chunks) > 0, f"[{x.id}] example didn't split by chunks!"

        scores_valid = []
        scores_test = []

        verbose_fn = verbose_fn if verbose_fn is not None else print

        if model_dir is None:
            print("[WARNING] model dir is not set => evaluation on test data will be done on last weights, "
                  "which might differ from best ones")

        sess_config = tf.ConfigProto()
        sess_config.gpu_options.allow_growth = True

        for i, (train_files, test_files) in enumerate(folds):
            print(f"FOLDS {i}")

            train_files_set = set(train_files)
            test_files_set = set(test_files)

            examples_train_valid = [x for x in examples if x.filename in train_files_set]
            examples_test = [x for x in examples if x.filename in test_files_set]

            train_frac = 1.0 - valid_frac
            examples_train, examples_valid = train_test_split(
                examples_train_valid, train_frac=train_frac, seed=228
            )

            print("num train examples:", len(examples_train))
            print("num valid examples:", len(examples_valid))
            print("num test examples:", len(examples_test))

            # TODO: lr schedule depends on num train steps, which depends on num train samples and batch size.

            with tf.Session(config=sess_config) as sess:
                self.sess = sess
                self.reset_weights(**kwargs)
                self.train(
                    examples_train=examples_train,
                    examples_valid=examples_valid,
                    train_op_name="train_op",
                    model_dir=model_dir,
                    scope_to_save=None,
                    verbose=verbose,
                    verbose_fn=verbose_fn
                )
                print("best valid scores:")
                d_valid = self.evaluate(examples=examples_valid)
                verbose_fn(d_valid)
                print("\ntest scores:")
                d_test = self.evaluate(examples=examples_test)
                verbose_fn(d_test)

            scores_valid.append(d_valid["score"])
            scores_test.append(d_test["score"])

            print("=" * 80)

        # pycharm bug:
        # Cannot find reference {mean, std} in __init__.pyi | __init__.pxd
        # so, np.mean(scores) highlights yellow
        scores_valid = np.array(scores_valid)
        scores_test = np.array(scores_test)

        print(f"scores valid: {[round(x, 4) for x in scores_valid]} "
              f"(mean {round(scores_valid.mean(), 4)}, std {round(scores_valid.std(), 4)})")
        print(f"scores test: {[round(x, 4) for x in scores_test]} "
              f"(mean {round(scores_test.mean(), 4)}, std {round(scores_test.std(), 4)})")

        return scores_valid, scores_test

    def save_config(self, model_dir: str):
        assert self.config is not None
        assert os.path.isdir(model_dir)
        with open(os.path.join(model_dir, "config.json"), "w") as f:
            json.dump(self.config, f, indent=4)

    @classmethod
    def load(cls, sess: tf.Session, model_dir: str, scope_to_load: str = None, mode: str = ModeKeys.TEST):

        with open(os.path.join(model_dir, "config.json")) as f:
            config = json.load(f)

        model = cls(sess=sess, config=config)
        model.build(mode=mode)
        model.restore_weights(model_dir=model_dir, scope=scope_to_load)
        return model

    def save_weights(self, model_dir: str,  scope: str = None):
        self._save_or_restore(model_dir=model_dir, save=True, scope=scope)

    def restore_weights(self, model_dir: str,  scope: str = None):
        self._save_or_restore(model_dir=model_dir, save=False, scope=scope)

    def reset_weights(self, scope: str = None, **kwargs):
        variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope)
        init_op = tf.variables_initializer(variables)
        self.sess.run(init_op)

    def _save_or_restore(self, model_dir: str, save: bool, scope: str = None):
        scope = scope if scope is not None else self.model_scope
        var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope)
        saver = tf.train.Saver(var_list)
        checkpoint_path = os.path.join(model_dir, "model.ckpt")
        if save:
            saver.save(self.sess, checkpoint_path)
        else:
            saver.restore(self.sess, checkpoint_path)

class BaseModelBert(BaseModel):
    def __init__(self, sess: tf.Session = None, config: dict = None):
        super().__init__(sess=sess, config=config)

        # PLACEHOLDERS
        self.input_ids_ph = None
        self.input_mask_ph = None
        self.segment_ids_ph = None
        self.first_pieces_coords_ph = None
        self.num_pieces_ph = None  # для обучаемых с нуля рекуррентных слоёв
        self.num_tokens_ph = None  # для crf

        # LAYERS
        self.bert_dropout = None
        self.birnn_bert = None

    def _build_embedder(self):
        self.bert_out_train = self._build_bert(training=True)  # [N, T_pieces, D]
        self.bert_out_pred = self._build_bert(training=False)  # [N, T_pieces, D]

    def _build_bert(self, training):
        if self.config["model"]["bert"]["test_mode"]:
            input_shape = tf.shape(self.input_ids_ph)
            bert_dim = self.config["model"]["bert"]["params"]["hidden_size"]
            x = tf.random.uniform((input_shape[0], input_shape[1], bert_dim))
            return x
        else:
            bert_scope = self.config["model"]["bert"]["scope"]
            reuse = not training
            with tf.variable_scope(bert_scope, reuse=reuse):
                bert_config = BertConfig.from_dict(self.config["model"]["bert"]["params"])
                model = BertModel(
                    config=bert_config,
                    is_training=training,
                    input_ids=self.input_ids_ph,
                    input_mask=self.input_mask_ph,
                    token_type_ids=self.segment_ids_ph
                )
                x = model.get_sequence_output()
            return x

    def _set_layers(self):
        self.bert_dropout = tf.keras.layers.Dropout(self.config["model"]["bert"]["dropout"])
        if self.config["model"]["birnn"]["use"]:
            self.birnn_bert = StackedBiRNN(**self.config["model"]["birnn"]["params"])

    def _actual_name_to_checkpoint_name(self, name: str) -> str:
        bert_scope = self.config["model"]["bert"]["scope"]
        prefix = f"{self.model_scope}/{bert_scope}/"
        name = name[len(prefix):]
        name = name.replace(":0", "")
        return name

    def _set_placeholders(self):
        # bert inputs
        self.input_ids_ph = tf.placeholder(dtype=tf.int32, shape=[None, None], name="input_ids")
        self.input_mask_ph = tf.placeholder(dtype=tf.int32, shape=[None, None], name="input_mask")
        self.segment_ids_ph = tf.placeholder(dtype=tf.int32, shape=[None, None], name="segment_ids")

        # ner inputs
        # [id_example, id_piece]
        self.first_pieces_coords_ph = tf.placeholder(dtype=tf.int32, shape=[None, None, 2], name="first_pieces_coords")
        self.num_pieces_ph = tf.placeholder(dtype=tf.int32, shape=[None], name="num_pieces")
        self.num_tokens_ph = tf.placeholder(dtype=tf.int32, shape=[None], name="num_tokens")

        # common inputs
        self.training_ph = tf.placeholder(dtype=tf.bool, shape=None, name="training_ph")

    def reset_weights(self, scope: str = None, **kwargs):
        """в kwargs должен быть bert_dir!"""
        super().reset_weights(scope=scope)

        if not self.config["model"]["bert"]["test_mode"]:
            bert_scope = self.config["model"]["bert"]["scope"]
            var_list = {
                self._actual_name_to_checkpoint_name(x.name): x for x in tf.trainable_variables()
                if x.name.startswith(f"{self.model_scope}/{bert_scope}")
            }
            saver = tf.train.Saver(var_list)
            bert_dir = kwargs["bert_dir"]
            checkpoint_path = os.path.join(bert_dir, "bert_model.ckpt")
            saver.restore(self.sess, checkpoint_path)

    def _set_train_op(self):
        num_samples = self.config["training"]["num_train_samples"]
        batch_size = self.config["training"]["batch_size"]
        num_epochs = self.config["training"]["num_epochs"]
        num_train_steps = int(num_samples / batch_size) * num_epochs
        warmup_proportion = self.config["optimizer"]["warmup_proportion"]
        num_warmup_steps = int(num_train_steps * warmup_proportion)
        init_lr = self.config["optimizer"]["init_lr"]
        self.train_op = create_optimizer(
            loss=self.loss,
            init_lr=init_lr,
            num_train_steps=num_train_steps,
            num_warmup_steps=num_warmup_steps,
            use_tpu=False
        )

    @property
    def _is_bpe_level(self) -> bool:
        return True

    def _get_token_level_embeddings(self, bert_out: tf.Tensor) -> tf.Tensor:
        # dropout
        bert_out = self.bert_dropout(bert_out, training=self.training_ph)

        # pieces -> tokens
        x = tf.gather_nd(bert_out, self.first_pieces_coords_ph)  # [batch_size, num_tokens, bert_dim]

        # birnn  TODO: мб это преобразования стоит делать перед преобразованием "pieces -> tokens"?
        if self.birnn_bert is not None:
            sequence_mask = tf.sequence_mask(self.num_tokens_ph)
            x = self.birnn_bert(x, training=self.training_ph, mask=sequence_mask)  # [N, num_tokens, cell_dim * 2]
        return x

    def _get_bert_input_for_feed_dict(self, examples: List[Example]) -> BertInputs:
        input_ids = []
        input_mask = []
        segment_ids = []
        first_pieces_coords = []
        num_tokens = []
        num_pieces = []

        for i, x in enumerate(examples):
            input_ids_i = []
            input_mask_i = []
            segment_ids_i = []
            first_pieces_coords_i = []
            num_tokens_i = 0

            # [CLS]
            input_ids_i.append(self.config["model"]["bert"]["cls_token_id"])
            input_mask_i.append(1)
            segment_ids_i.append(0)

            for t in x.tokens:
                n = len(t.token_ids)
                assert n != 0, f"[{x.id}] token {t} could not be split by pieces! " \
                    f"unicode code points: {[ord(c) for c in t.text]}"
                first_pieces_coords_i.append((i, len(input_ids_i)))
                input_ids_i += t.token_ids
                input_mask_i += [1] * n
                segment_ids_i += [0] * n
                num_tokens_i += 1

            # [SEP]
            input_ids_i.append(self.config["model"]["bert"]["sep_token_id"])
            input_mask_i.append(1)
            segment_ids_i.append(0)

            input_ids.append(input_ids_i)
            input_mask.append(input_mask_i)
            segment_ids.append(segment_ids_i)
            first_pieces_coords.append(first_pieces_coords_i)
            num_tokens.append(num_tokens_i)
            num_pieces.append(len(input_ids_i))

        # padding
        pad_token_id = self.config["model"]["bert"]["pad_token_id"]
        num_tokens_max = max(num_tokens)
        num_pieces_max = max(num_pieces)
        for i in range(len(examples)):
            input_ids[i] += [pad_token_id] * (num_pieces_max - num_pieces[i])
            input_mask[i] += [0] * (num_pieces_max - num_pieces[i])
            segment_ids[i] += [0] * (num_pieces_max - num_pieces[i])
            first_pieces_coords[i] += [(i, 0)] * (num_tokens_max - num_tokens[i])

        return BertInputs(
            input_ids=input_ids,
            input_mask=input_mask,
            segment_ids=segment_ids,
            first_pieces_coords=first_pieces_coords,
            num_tokens=num_tokens,
            num_pieces=num_pieces
        )

class BaseModelNER(BaseModel):
    ner_scope = "ner"

    def __init__(self, sess, config: Dict = None, ner_enc: Dict = None):
        super().__init__(sess=sess, config=config)
        self._ner_enc = None
        self._inv_ner_enc = None

        self.ner_enc = ner_enc

        self.ner_labels_ph = None

    def _build_graph(self):
        self._build_embedder()
        with tf.variable_scope(self.ner_scope):
            self._build_ner_head()

    def save_config(self, model_dir: str):
        assert self.ner_enc is not None
        super().save_config(model_dir=model_dir)
        with open(os.path.join(model_dir, "ner_enc.json"), "w") as f:
            json.dump(self.ner_enc, f, indent=4)

    @classmethod
    def load(cls, sess: tf.Session, model_dir: str, scope_to_load: str = None, mode: str = ModeKeys.TEST):
        model = super().load(sess=sess, model_dir=model_dir, scope_to_load=scope_to_load, mode=mode)
        with open(os.path.join(model_dir, "ner_enc.json")) as f:
            model.ner_enc = json.load(f)
        return model

    @abstractmethod
    def _build_ner_head(self):
        pass

    @property
    def ner_enc(self):
        return self._ner_enc

    @property
    def inv_ner_enc(self):
        return self._inv_ner_enc

    @ner_enc.setter
    def ner_enc(self, ner_enc: Dict):
        self._ner_enc = ner_enc
        if ner_enc is not None:
            self._inv_ner_enc = {v: k for k, v in ner_enc.items()}

class BaseModelRelationExtraction(BaseModelNER):
    """
    сущности уже известны. требуется найти отношения между ними.
    наследуется от BaseModelNER, потому что ner_enc тоже нужен для возмонжости использовать в модели лейблы сущностей.
    """
    re_scope = "re"

    def __init__(self, sess, config: Dict = None, ner_enc: Dict = None, re_enc: Dict = None):
        super().__init__(sess=sess, config=config)
        self._ner_enc = None
        self._inv_ner_enc = None
        self.ner_enc = ner_enc

        self._re_enc = None
        self._inv_re_enc = None
        self.re_enc = re_enc

        self.ner_labels_ph = None

    def _build_graph(self):
        self._build_embedder()
        with tf.variable_scope(self.re_scope):
            self._build_re_head()

    # TODO: костыль
    def _build_ner_head(self):
        pass

    def save_config(self, model_dir: str):
        assert self.ner_enc is not None
        assert self.re_enc is not None
        super().save_config(model_dir=model_dir)
        with open(os.path.join(model_dir, "ner_enc.json"), "w") as f:
            json.dump(self.ner_enc, f, indent=4)
        with open(os.path.join(model_dir, "re_enc.json"), "w") as f:
            json.dump(self.re_enc, f, indent=4)

    @classmethod
    def load(cls, sess: tf.Session, model_dir: str, scope_to_load: str = None, mode: str = ModeKeys.TEST):
        model = super().load(sess=sess, model_dir=model_dir, scope_to_load=scope_to_load, mode=mode)
        with open(os.path.join(model_dir, "ner_enc.json")) as f:
            model.ner_enc = json.load(f)
        with open(os.path.join(model_dir, "re_enc.json")) as f:
            model.re_enc = json.load(f)
        return model

    @abstractmethod
    def _build_re_head(self):
        pass

    @property
    def re_enc(self):
        return self._re_enc

    @property
    def inv_re_enc(self):
        return self._inv_re_enc

    @re_enc.setter
    def re_enc(self, re_enc: Dict):
        self._re_enc = re_enc
        if re_enc is not None:
            self._inv_re_enc = {v: k for k, v in re_enc.items()}

# лучше делать отдельный класс под joint модели (ner + re, mentions + coreference):
# * для ner + re в конфиге нужна секция model.ner, а в re - нет
# * для ner + re нужно создавать слои под ner, а для re - нет
# * для инференса модели re нужны истинные ner-лейблы, а для инференса модели ner + re - нет.
class BaseModelNerAndRelationExtracion(BaseModelRelationExtraction):
    """
    требуется найти сущности и отношения между ними
    """
    def __init__(self, sess, config: Dict = None, ner_enc: Dict = None, re_enc: Dict = None):
        super().__init__(sess=sess, config=config, ner_enc=ner_enc, re_enc=re_enc)

    def _build_graph(self):
        self._build_embedder()
        with tf.variable_scope(self.ner_scope):
            self._build_ner_head()
        with tf.variable_scope(self.re_scope):
            self._build_re_head()

class BaseModeDependencyParsing(BaseModel):
    dep_scope = "dependency_parser"

    def __init__(self, sess: tf.Session = None, config: Dict = None, rel_enc: Dict = None):
        super().__init__(sess=sess, config=config)

        # PLACEHOLDERS
        self.labels_ph = None  # [id_example, id_head, id_dep, id_rel]

        # LAYERS
        self.birnn = None
        self.arc_enc = None
        self.type_enc = None

        # TENSORS
        self.logits_arc_train = None
        self.logits_type_train = None
        self.s_arc = None
        self.type_labels_pred = None
        # for debug:
        self.total_loss_arc = None
        self.total_loss_type = None

        self._rel_enc = None
        self._inv_rel_enc = None

        self.rel_enc = rel_enc

    def _build_graph(self):
        self._build_embedder()
        with tf.variable_scope(self.dep_scope):
            self._build_dependency_parser()

    @abstractmethod
    def _build_dependency_parser(self):
        pass

    @property
    def rel_enc(self):
        return self._rel_enc

    @property
    def inv_rel_enc(self):
        return self._inv_rel_enc

    @rel_enc.setter
    def rel_enc(self, rel_enc: Dict):
        self._rel_enc = rel_enc
        if rel_enc is not None:
            self._inv_rel_enc = {v: k for k, v in rel_enc.items()}

    def save_config(self, model_dir: str):
        assert self.rel_enc is not None
        super().save_config(model_dir=model_dir)
        with open(os.path.join(model_dir, "rel_enc.json"), "w") as f:
            json.dump(self.rel_enc, f, indent=4)

    @classmethod
    def load(cls, sess: tf.Session, model_dir: str, scope_to_load: str = None, mode: str = ModeKeys.TEST):
        model = super().load(sess=sess, model_dir=model_dir, scope_to_load=scope_to_load, mode=mode)
        with open(os.path.join(model_dir, "rel_enc.json")) as f:
            model.rel_enc = json.load(f)
        return model

class BaseModeCoreferenceResolution(BaseModel):
    coref_scope = "coref"
    coref_rel = "COREFERENCE"

    def __init__(self, sess: tf.Session = None, config: Dict = None):
        super().__init__(sess=sess, config=config)

        # PLACEHOLDERS
        self.mention_spans_ph = None  # [id_example, start, end]
        self.labels_ph = None  # [id_example, id_anaphora, id_antecedent]

        # LAYERS
        self.birnn = None

        # TENSORS
        self.logits_train = None
        self.logits_pred = None
        self.total_loss = None
        self.loss_denominator = None

    def _build_graph(self):
        self._build_embedder()
        with tf.variable_scope(self.coref_scope):
            self._build_coref_head()

    @abstractmethod
    def _build_coref_head(self):
        pass


import json
from shapely.geometry import shape
class GeojsonConn():
    def __init__(self,segments_geojson_path,segment_id_field):
        self._lines_feats = {}
        with open(segments_geojson_path, 'r') as lines_file:
            lines_json = json.load(lines_file)

            for feature in lines_json['features']:
                feat_id = feature['properties'][segment_id_field]
                geom = shape(feature['geometry'])
                # check data
                if feat_id in self._lines_feats.keys():
                    raise Exception("Input layer contains conflicting data: row with same id %s" % feat_id)
                self._lines_feats[feat_id] = geom
    def get_segment_length(self,line_id):
        return self._lines_feats[line_id].length


#!/usr/bin/env python
import os
import rospy
from rospkg import RosPack
import xml.etree.ElementTree as ET
import zlib

'''
Created on 10.01.2017

@author: Philipp Schillinger
'''

class BehaviorLibrary(object):
	'''
	Provides a list of all known behavior manifests.
	'''

	def __init__(self):
		self._rp = RosPack()
		self._behavior_lib = dict()
		self.parse_packages()

	def parse_packages(self):
		"""
		Parses all ROS packages to update the internal behavior library.
		"""
		self._behavior_lib = dict()
		for pkg in self._rp.list():
			for export in self._rp._load_manifest(pkg).exports:
				if export.tag == "flexbe_behaviors":
					self._add_behavior_manifests(self._rp.get_path(pkg), pkg)

	def _add_behavior_manifests(self, path, pkg=None):
		"""
		Recursively add all behavior manifests in the given folder to the internal library.
		If a package name is specified, only manifests referring to this package are added.

		@type path: string
		@param path: Path of the folder to be traversed.

		@type pkg: string
		@param pkg: Optional name of a package to only add manifests referring to this package.
		"""
		for entry in os.listdir(path):
			entry_path = os.path.join(path, entry)
			if os.path.isdir(entry_path):
				self._add_behavior_manifests(entry_path, pkg)
			elif entry.endswith(".xml") and not entry.startswith("#"):
				m = ET.parse(entry_path).getroot()
				# structure sanity check
				if m.tag != "behavior" \
				or len(m.findall(".//executable")) == 0 \
				or m.find("executable").get("package_path") is None \
				or len(m.find("executable").get("package_path").split(".")) != 2:
					continue;
				e = m.find("executable")
				if pkg is not None and e.get("package_path").split(".")[0] != pkg:
					continue # ignore if manifest not in specified package
				be_id = zlib.adler32(e.get("package_path"))
				self._behavior_lib[be_id] = {
					"name": m.get("name"),
					"package": e.get("package_path").split(".")[0],
					"file": e.get("package_path").split(".")[1],
					"class": e.get("class")
				}

	def get_behavior(self, be_id):
		"""
		Provides the library entry corresponding to the given ID.

		@type be_id: int
		@param be_id: Behavior ID to look up.

		@return Corresponding library entry or None if not found.
		"""
		try:
			return self._behavior_lib[be_id]
		except KeyError:
			rospy.logwarn("Did not find ID %d in libary, updating..." % be_id)
			self.parse_packages()
			return self._behavior_lib.get(be_id, None)

	def find_behavior(self, be_name):
		"""
		Searches for a behavior with the given name and returns it along with its ID.

		@type be_name: string
		@param be_name: Behavior ID to look up.

		@return Tuple (be_id, be_entry) corresponding to the name or (None, None) if not found.
		"""
		find = lambda: next((id, be) for (id, be) in self._behavior_lib.items() if be["name"] == be_name)
		try:
			return find()
		except StopIteration:
			rospy.logwarn("Did not find behavior '%s' in libary, updating..." % be_name)
			self.parse_packages()
			return find()

	def count_behaviors(self):
		"""
		Counts the available behaviors.

		@return Number of behaviors.
		"""
		return len(self._behavior_lib)


class CardFeatureVectorCompute:
    feature_num = 18

    def fetch_feature(self, hole, community):
        cards = hole + community
        assert(len(hole) is 2)
        feature_vector = self.hole_eval(hole)
        feature_vector += self.value_count(cards)
        feature_vector += self.suit_count(cards)
        feature_vector += self.seq_count(cards)
        return feature_vector

    def fetch_feature_essential(self, hole, community):
        f = self.fetch_feature(hole, community)
        f_val_comb = f[3] + f[4] + f[5] + f[6] + f[7]
        f_suit_comb = f[9] + f[10] + f[11] + f[12] + f[13]
        f_seq_comb = f[15] + f[16] + f[17]
        return [f[0], f[1], f[2], f_val_comb, f[8], f_suit_comb, f[14], f_seq_comb]

    def fetch_feature_2(self, community):
        cards = community
        feature_vector = [0, 0]
        feature_vector += self.value_count(cards)
        feature_vector += self.suit_count(cards)
        feature_vector += self.seq_count(cards)
        return feature_vector

    def hole_eval(self, cards):
        max_val = max(cards[0].rank, cards[0].rank)
        diff = abs(cards[0].rank - cards[1].rank)
        scaled_max_val = (max_val - 1) / float(14)
        scaled_diff = (12 - diff) / float(13)
        return [scaled_max_val, scaled_diff]

    def value_count(self, cards):
        memo = 0
        pair_memo = 0
        three_memo = 0
        four_memo = 0
        for card in cards:
            mask = 1 << card.rank
            four_memo |= three_memo & mask
            three_memo |= pair_memo & mask
            pair_memo |= memo & mask
            memo |= mask
        pair = 1 if pair_memo != 0 else 0
        pair_num = self.__count_bit(pair_memo) / float(3)
        three = 1 if three_memo != 0 else 0
        three_num = self.__count_bit(three_memo) / float(2)
        four = 1 if four_memo != 0 else 0
        full_house = 1 if pair_memo != three_memo else 0
        return [pair, pair_num, three, three_num, four, full_house]

    def suit_count(self, cards):
        memo = 0
        two_memo = 0
        three_memo = 0
        four_memo = 0
        five_memo = 0
        for card in cards:
            mask = card.suit
            five_memo |= four_memo & mask
            four_memo |= three_memo & mask
            three_memo |= two_memo & mask
            two_memo |= memo & mask
            memo |= mask

        pair = 1 if two_memo != 0 else 0
        pair_num = self.__count_bit(two_memo) / float(3)
        three = 1 if three_memo != 0 else 0
        three_num = self.__count_bit(three_memo) / float(2)
        four = 1 if four_memo != 0 else 0
        five = 1 if five_memo != 0 else 0
        return [pair, pair_num, three, three_num, four, five]

    def seq_count(self, cards):
        memo = 0
        for card in cards:
            mask = 1 << card.rank
            memo |= mask
        s_len = self.__count_longest_seq(memo)
        two = 1 if s_len > 1 else 0
        three = 1 if s_len > 2 else 0
        four = 1 if s_len > 3 else 0
        five = 1 if s_len > 4 else 0
        return [two, three, four, five]

    def __count_bit(self, n):
        count = 0
        while n:
            count += n & 1
            n = n >> 1
        return count

    def __count_longest_seq(self, n):
        count = 0
        while n:
            count = count + 1
            n = n & (n >> 1)
        return count


# -*- coding: utf-8 -*-
"""
Created on Sun Sep  8 12:03:52 2019

@author: Brian Chan
"""

import pandas as pd

df = pd.read_csv('https://archive.ics.uci.edu/ml/'
                 'machine-learning-databases'
                 '/breast-cancer-wisconsin/wdbc.data', header=None)

# if the Breast Cancer dataset is temporarily unavailable from the
# UCI machine learning repository, un-comment the following line
# of code to load the dataset from a local path:

# df = pd.read_csv('wdbc.data', header=None)

#https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)
#1) ID number
#2) Diagnosis (M = malignant, B = benign)
#3-32)
#
#Ten real-valued features are computed for each cell nucleus:
#
#a) radius (mean of distances from center to points on the perimeter)
#b) texture (standard deviation of gray-scale values)
#c) perimeter
#d) area
#e) smoothness (local variation in radius lengths)
#f) compactness (perimeter^2 / area - 1.0)
#g) concavity (severity of concave portions of the contour)
#h) concave points (number of concave portions of the contour)
#i) symmetry
#j) fractal dimension ("coastline approximation" - 1)

df.head()

df.shape

from sklearn.preprocessing import LabelEncoder

X = df.loc[:, 2:].values
y = df.loc[:, 1].values
le = LabelEncoder()
y = le.fit_transform(y)
le.classes_

le.transform(['M', 'B'])

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y,  test_size=0.20, stratify=y, random_state=1)

# =============================================================================
# Combining transformers and estimators in a pipeline
# =============================================================================

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline

pipe_lr = make_pipeline(StandardScaler(), PCA(n_components=2), LogisticRegression(random_state=1))

pipe_lr.fit(X_train, y_train)
y_pred = pipe_lr.predict(X_test)
print('Test Accuracy: %.3f' % pipe_lr.score(X_test, y_test))

#Image(filename='images/06_01.png', width=500)

#Image(filename='images/06_02.png', width=500)

#Image(filename='images/06_03.png', width=500)

# =============================================================================
# Using k-fold cross validation to assess model performance
# =============================================================================

import numpy as np
from sklearn.model_selection import StratifiedKFold

kfold = StratifiedKFold(n_splits=10, random_state=1).split(X_train, y_train)

scores = []
for k, (train, test) in enumerate(kfold):
    pipe_lr.fit(X_train[train], y_train[train])
    score = pipe_lr.score(X_train[test], y_train[test])
    scores.append(score)
    print('Fold: %2d, Class dist.: %s, Acc: %.3f' % (k+1,
          np.bincount(y_train[train]), score))

print('\nCV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

from sklearn.model_selection import cross_val_score

# fitting with different training sample size with cross validation of 10
scores = cross_val_score(estimator=pipe_lr, X=X_train, y=y_train, cv=10, n_jobs=1)

print('CV accuracy scores: %s' % scores)
print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

# =============================================================================
# Diagnosing bias and variance problems with learning curves
# =============================================================================

#Image(filename='images/06_04.png', width=600)

import matplotlib.pyplot as plt
from sklearn.model_selection import learning_curve

pipe_lr = make_pipeline(StandardScaler(),
                        LogisticRegression(penalty='l2', random_state=1))

train_sizes, train_scores, test_scores =\
                learning_curve(estimator=pipe_lr,
                               X=X_train,
                               y=y_train,
                               train_sizes=np.linspace(0.1, 1.0, 10),
                               cv=10,
                               n_jobs=1)

train_mean = np.mean(train_scores, axis=1)
train_std  = np.std(train_scores, axis=1)
test_mean  = np.mean(test_scores, axis=1)
test_std   = np.std(test_scores, axis=1)

plt.figure()
plt.plot(train_sizes, train_mean,
         color='blue', marker='o',
         markersize=5, label='training accuracy')

plt.fill_between(train_sizes,
                 train_mean + train_std,
                 train_mean - train_std,
                 alpha=0.15, color='blue')

plt.plot(train_sizes, test_mean,
         color='green', linestyle='--',
         marker='s', markersize=5,
         label='validation accuracy')

plt.fill_between(train_sizes,
                 test_mean + test_std,
                 test_mean - test_std,
                 alpha=0.15, color='green')

plt.grid()
plt.xlabel('Number of training samples')
plt.ylabel('Accuracy')
plt.legend(loc='lower right')
plt.ylim([0.8, 1.03])
plt.tight_layout()
#plt.savefig('images/06_05.png', dpi=300)
plt.show()

# =============================================================================
# Addressing over- and underfitting with validation curves
# =============================================================================

from sklearn.model_selection import validation_curve

param_range = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]
train_scores, test_scores = validation_curve( estimator=pipe_lr,
                X=X_train,
                y=y_train,
                param_name='logisticregression__C',
                param_range=param_range,
                cv=10)

train_mean = np.mean(train_scores, axis=1)
train_std  = np.std(train_scores, axis=1)
test_mean  = np.mean(test_scores, axis=1)
test_std   = np.std(test_scores, axis=1)

plt.figure()
plt.plot(param_range, train_mean,
         color='blue', marker='o',
         markersize=5, label='training accuracy')

plt.fill_between(param_range, train_mean + train_std,
                 train_mean - train_std, alpha=0.15,
                 color='blue')

plt.plot(param_range, test_mean,
         color='green', linestyle='--',
         marker='s', markersize=5,
         label='validation accuracy')

plt.fill_between(param_range,
                 test_mean + test_std,
                 test_mean - test_std,
                 alpha=0.15, color='green')

plt.grid()
plt.xscale('log')
plt.legend(loc='lower right')
plt.xlabel('Parameter C')
plt.ylabel('Accuracy')
plt.ylim([0.8, 1.0])
plt.tight_layout()
# plt.savefig('images/06_06.png', dpi=300)
plt.show()

# =============================================================================
# Tuning hyperparameters via grid search
# =============================================================================

from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC

pipe_svc = make_pipeline(StandardScaler(),
                         SVC(random_state=1))

param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]

param_grid = [{'svc__C': param_range,
               'svc__kernel': ['linear']},
              {'svc__C': param_range,
               'svc__gamma': param_range,
               'svc__kernel': ['rbf']}]

gs = GridSearchCV(estimator=pipe_svc,
                  param_grid=param_grid,
                  scoring='accuracy',
                  cv=10,
                  n_jobs=-1)
gs = gs.fit(X_train, y_train)
print(gs.best_score_)
print(gs.best_params_)

clf = gs.best_estimator_
clf.fit(X_train, y_train)
print('Test accuracy: %.3f' % clf.score(X_test, y_test))

# =============================================================================
# Reading a confusion matrix
# =============================================================================

#Image(filename='images/06_08.png', width=300)

from sklearn.metrics import confusion_matrix

pipe_svc.fit(X_train, y_train)
y_pred = pipe_svc.predict(X_test)
confmat = confusion_matrix(y_true=y_test, y_pred=y_pred)
print(confmat)

fig, ax = plt.subplots(figsize=(2.5, 2.5))
ax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)
for i in range(confmat.shape[0]):
    for j in range(confmat.shape[1]):
        ax.text(x=j, y=i, s=confmat[i, j], va='center', ha='center')

plt.xlabel('Predicted label')
plt.ylabel('True label')

plt.tight_layout()
#plt.savefig('images/06_09.png', dpi=300)
plt.show()

le.transform(['M', 'B'])

confmat = confusion_matrix(y_true=y_test, y_pred=y_pred)
print(confmat)

confmat = confusion_matrix(y_true=y_test, y_pred=y_pred, labels=[1, 0])
print(confmat)

from sklearn.metrics import precision_score, recall_score, f1_score

print('Precision: %.3f' % precision_score(y_true=y_test, y_pred=y_pred))
print('Recall: %.3f' % recall_score(y_true=y_test, y_pred=y_pred))
print('F1: %.3f' % f1_score(y_true=y_test, y_pred=y_pred))

from sklearn.metrics import make_scorer

scorer = make_scorer(f1_score, pos_label=0)

c_gamma_range = [0.01, 0.1, 1.0, 10.0]

param_grid = [{'svc__C': c_gamma_range,
               'svc__kernel': ['linear']},
              {'svc__C': c_gamma_range,
               'svc__gamma': c_gamma_range,
               'svc__kernel': ['rbf']}]

gs = GridSearchCV(estimator=pipe_svc,
                  param_grid=param_grid,
                  scoring=scorer,
                  cv=10,
                  n_jobs=-1)
gs = gs.fit(X_train, y_train)
print(gs.best_score_)
print(gs.best_params_)

# =============================================================================
# Dealing with class imbalance
# =============================================================================

pre_scorer = make_scorer(score_func=precision_score,
                         pos_label=1,
                         greater_is_better=True,
                         average='micro')

X_imb = np.vstack((X[y == 0], X[y == 1][:40]))
y_imb = np.hstack((y[y == 0], y[y == 1][:40]))

y_pred = np.zeros(y_imb.shape[0])
np.mean(y_pred == y_imb) * 100

from sklearn.utils import resample

print('Number of class 1 samples before:', X_imb[y_imb == 1].shape[0])

X_upsampled, y_upsampled = resample(X_imb[y_imb == 1],
                                    y_imb[y_imb == 1],
                                    replace=True,
                                    n_samples=X_imb[y_imb == 0].shape[0],
                                    random_state=123)

print('Number of class 1 samples after:', X_upsampled.shape[0])

X_bal = np.vstack((X[y == 0], X_upsampled))
y_bal = np.hstack((y[y == 0], y_upsampled))

y_pred = np.zeros(y_bal.shape[0])
np.mean(y_pred == y_bal) * 100


import csv
import itertools
import sys

PROBS = {

    # Unconditional probabilities for having gene
    "gene": {
        2: 0.01,
        1: 0.03,
        0: 0.96
    },

    "trait": {

        # Probability of trait given two copies of gene
        2: {
            True: 0.65,
            False: 0.35
        },

        # Probability of trait given one copy of gene
        1: {
            True: 0.56,
            False: 0.44
        },

        # Probability of trait given no gene
        0: {
            True: 0.01,
            False: 0.99
        }
    },

    # Mutation probability
    "mutation": 0.01
}

def main():

    # Check for proper usage
    if len(sys.argv) != 2:
        sys.exit("Usage: python heredity.py data.csv")
    people = load_data(sys.argv[1])

    # Keep track of gene and trait probabilities for each person
    probabilities = {
        person: {
            "gene": {
                2: 0,
                1: 0,
                0: 0
            },
            "trait": {
                True: 0,
                False: 0
            }
        }
        for person in people
    }

    # Loop over all sets of people who might have the trait
    names = set(people)
    for have_trait in powerset(names):

        # Check if current set of people violates known information
        fails_evidence = any(
            (people[person]["trait"] is not None and
             people[person]["trait"] != (person in have_trait))
            for person in names
        )
        if fails_evidence:
            continue

        # Loop over all sets of people who might have the gene
        for one_gene in powerset(names):
            for two_genes in powerset(names - one_gene):

                # Update probabilities with new joint probability
                p = joint_probability(people, one_gene, two_genes, have_trait)
                update(probabilities, one_gene, two_genes, have_trait, p)

    # Ensure probabilities sum to 1
    normalize(probabilities)

    # Print results
    for person in people:
        print(f"{person}:")
        for field in probabilities[person]:
            print(f"  {field.capitalize()}:")
            for value in probabilities[person][field]:
                p = probabilities[person][field][value]
                print(f"    {value}: {p:.4f}")

def load_data(filename):
    """
    Load gene and trait data from a file into a dictionary.
    File assumed to be a CSV containing fields name, mother, father, trait.
    mother, father must both be blank, or both be valid names in the CSV.
    trait should be 0 or 1 if trait is known, blank otherwise.
    """
    data = dict()
    with open(filename) as f:
        reader = csv.DictReader(f)
        for row in reader:
            name = row["name"]
            data[name] = {
                "name": name,
                "mother": row["mother"] or None,
                "father": row["father"] or None,
                "trait": (True if row["trait"] == "1" else
                          False if row["trait"] == "0" else None)
            }
    return data

def powerset(s):
    """
    Return a list of all possible subsets of set s.
    """
    s = list(s)
    return [
        set(s) for s in itertools.chain.from_iterable(
            itertools.combinations(s, r) for r in range(len(s) + 1)
        )
    ]

def joint_probability(people, one_gene, two_genes, have_trait):
    """
    Compute and return a joint probability.

    The probability returned should be the probability that
        * everyone in set `one_gene` has one copy of the gene, and
        * everyone in set `two_genes` has two copies of the gene, and
        * everyone not in `one_gene` or `two_gene` does not have the gene, and
        * everyone in set `have_trait` has the trait, and
        * everyone not in set` have_trait` does not have the trait.
    """
    probability_output = 1

    for person in people:

        # Calculate the number of genes.
        if person in one_gene:
            no_genes = 1
        elif person in two_genes:
            no_genes = 2
        else:
            no_genes = 0

        # Check if person has parents.
        # Note: In case there is an error in input file and only one parent is filled in, ignore it.
        if people[person]['father'] and people[person]['mother']:
            has_parents = True
        else:
            has_parents = False

        if has_parents:
            # Gene probability from parents.

            # TODO should put many of these in separate functions rather than copy/paste
            # Probability from mother
            if people[person]['mother'] in one_gene:
                no_genes_mother = 0.5
            elif people[person]['mother'] in two_genes:
                no_genes_mother = 0.99  # Not 1 because of mutation possibility!
            else:
                no_genes_mother = 0.01  # Not 0 because of mutation possibility!

            # Probability from Father
            if people[person]['father'] in one_gene:
                no_genes_father = 0.5
            elif people[person]['father'] in two_genes:
                no_genes_father = 0.99  # Not 1 because of mutation possibility!
            else:
                no_genes_father = 0.01  # Not 0 because of mutation possibility!

            if no_genes == 1:
                prob_gene = (1 - no_genes_father) * no_genes_mother + (1 - no_genes_mother) * no_genes_father
            elif no_genes == 2:
                prob_gene = no_genes_father * no_genes_mother
            else:
                prob_gene = (1 - no_genes_father) * (1 - no_genes_mother)
        else:
            # Gene probability from PROBS class.
            prob_gene = PROBS['gene'][no_genes]

        # Check if the person has trait.
        if person in have_trait:
            trait = True
        else:
            trait = False

        # Calculate probability of trait
        prob_trait = PROBS['trait'][no_genes][trait]

        probability_output = probability_output * prob_gene * prob_trait

    return probability_output

def update(probabilities, one_gene, two_genes, have_trait, p):
    """
    Add to `probabilities` a new joint probability `p`.
    Each person should have their "gene" and "trait" distributions updated.
    Which value for each distribution is updated depends on whether
    the person is in `have_gene` and `have_trait`, respectively.
    """
    for person in probabilities:

        # Calculate the number of genes.
        if person in one_gene:
            no_genes = 1
        elif person in two_genes:
            no_genes = 2
        else:
            no_genes = 0

        # Update gene probability
        probability = probabilities[person]['gene'][no_genes]
        probabilities[person]['gene'][no_genes] = probability + p

        # Check if the person has trait.
        if person in have_trait:
            trait = True
        else:
            trait = False

        # Update trait probability
        probability = probabilities[person]['trait'][trait]
        probabilities[person]['trait'][trait] = probability + p

def normalize(probabilities):
    """
    Update `probabilities` such that each probability distribution
    is normalized (i.e., sums to 1, with relative proportions the same).
    """
    for person, gene_trait_dist in probabilities.items():
        for gene_trait, dist in gene_trait_dist.items():
            total = sum(dist.values())
            for value in dist:
                probability = probabilities[person][gene_trait][value]
                probabilities[person][gene_trait][value] = probability / total

if __name__ == "__main__":
    main()


import os, sys
import cv2
localPath = sys.path[0]

dirFile = os.listdir(localPath)

TEMPLATE_EXTERN = "// extern uint8_t {}[{}];"
TEMPLATE_BUFF = "const uint8_t {0}[{1}] = {{\r\n\t{2}\r\n}};"

def bytes2hexstr(data_in):
    data_out = ''
    num = 0
    for i in data_in:
        hexstr = hex(i)
        if len(hexstr) == 3:
            hexstr = hexstr[:2] + '0' + hexstr[-1:]
        data_out += hexstr + ', '
        data_out += '\r\n\t' if num == 16 else ''
        num = num + 1 if num < 16 else 0
    return data_out

def array2hexstr(array_in):
    data_out = ''
    num = 0
    for i in array_in:
        for j in i:
            data  = ((j[2] >> 3) << 11) | ((j[1] >> 2) << 5) | (j[0] >> 3)
            hexstr = hex(data)
            hexstr = hexstr[:2] + '0'*(6 - len(hexstr)) + hexstr[2:]
            data_out += hexstr + ', '
            data_out += '\r\n\t' if num == 10 else ''
            num = num + 1 if num < 10 else 0
    return data_out

def createJpgFile():
    jpgFileList = []

    for i in dirFile:
        if '.jpg' in i:
            jpgFileList.append(i)

    for i in jpgFileList:
        jpgDir = localPath + '/' + i
        with open(jpgDir, 'rb') as fs:
            data = fs.read()
        with open(jpgDir[:-4] + '.c', 'w+') as fs:
            fs.write(TEMPLATE_EXTERN.format(i[:-4], len(data)))
            fs.write('\r\n')
            fs.write(TEMPLATE_BUFF.format(i[:-4], len(data), bytes2hexstr(data)))
            fs.write('\r\n')

def createBitMap():
    jpgFileList = []

    for i in dirFile:
        if '.jpg' in i:
            jpgFileList.append(i)

    for i in jpgFileList:
        jpgDir = localPath + '/' + i
        img = cv2.imread(jpgDir)
        imgSize = img.shape[0] * img.shape[1]
        with open(jpgDir[:-4] + '.h', 'w+') as fs:
            fs.write("// extern const uint16_t {}[{}];".format(i[:-4], imgSize))
            fs.write('\r\n')
            fs.write("const uint16_t {0}[{1}] = {{\r\n\t{2}\r\n}};".format(i[:-4], imgSize, array2hexstr(img)))
            fs.write('\r\n')

createBitMap()


from __future__ import print_function
__author__ = 'chick'

import unittest

from stencil_code.backend.local_size_computer import product, LocalSizeComputer
from test.fixtures import MockCPU, MockIrisPro

class TestLocalSizeComputer(unittest.TestCase):
    def _are_lists_equal(self, list1, list2):
        self.assertEqual(sorted(list1), sorted(list2))

    def _are_lists_unequal(self, list1, list2):
        self.assertNotEqual(sorted(list1), sorted(list2))

    def test_prod(self):
        self.assertTrue(product([1]) == 1)
        self.assertTrue(product([2, 3, 4]) == 24)

    def test_compute_local_group_size_1d(self):
        # chooses the minimum of shape / 2 and max local group size
        self.assertTrue(
            LocalSizeComputer([100], MockIrisPro).compute_local_size_thin() == (50,),
            "when smaller than work group divide by 2"
        )
        # print("ls1d {}".format(tools.compute_local_size([1000])))

        self.assertTrue(
            LocalSizeComputer([1000], MockIrisPro).compute_local_size_thin() == (500,),
            "when smaller than work group divide by 2"
        )

        self.assertTrue(
            LocalSizeComputer([10000], MockIrisPro).compute_local_size_thin() == (512,),
            "when smaller than work group divide by 2"
        )

    def test_compute_local_group_size_2d(self):
        # this device looks like a 2014 Iris Pro
        # the following numbers have not yet been tested for optimality
        # they are mostly a product of a desire for consistency and
        # minimization of unused cycles
        # TODO: fix both generator and number to reflect optimality

        test_cases = [
            [[1, 101], (1, 101)],
            [[101, 1], (101, 1)],
            [[512, 101], (19, 26)],
            [[512, 513], (1, 257)],
            [[300, 1025], (1, 342)],
            [[5120, 32], (16, 32)],
            [[5120011, 320001], (1, 512)],
            [[102, 7], (102, 4)]
        ]

        for grid_size, predicted_local_size in test_cases:
            local_size = LocalSizeComputer(grid_size, MockIrisPro).compute_local_size_thin()
            self.assertListEqual(list(local_size), list(predicted_local_size))

    def test_compute_local_group_size_3d(self):
        test_cases = [
            [[1, 1, 101], (1, 1, 1)],
            [[1, 101, 1], (1, 1, 1)],
            [[101, 1, 1], (101, 1, 1)],
            [[100, 512, 101], (100, 1, 1)]
        ]

        for grid_size, predicted_local_size in test_cases:
            local_size = LocalSizeComputer(grid_size, MockCPU).compute_local_size_thin()
            self.assertListEqual(list(local_size), list(predicted_local_size))

        test_cases = [
            [[1, 1, 101], (1, 1, 101)],
            [[1, 101, 1], (1, 101, 1)],
            [[101, 1, 1], (101, 1, 1)],
            [[100, 512, 101], (1, 19, 26)]
        ]

        for grid_size, predicted_local_size in test_cases:
            local_size = LocalSizeComputer(grid_size, MockIrisPro).compute_local_size_thin()
            self.assertListEqual(list(local_size), list(predicted_local_size))

    def test_local_size_computer_bulky(self):
        test_grid_shape = [4, 128]
        lsc = LocalSizeComputer(test_grid_shape, MockIrisPro)
        local_size = lsc.compute_local_size_bulky()
        print("{} ls {}".format(test_grid_shape, local_size))

        sizes = [
            [[4], (4,), (4,)],
            [[5], (5,), (5,)],
            [[255], (255,), (255,)],
            [[1023], (1023,), (341,)],
            [[1024], (1024,), (512,)],
            [[1025], (205,), (205,)],

            [[1, 4], (1, 1), (1, 4)],
            [[4, 1], (4, 1), (4, 1)],
            [[4, 4], (4, 1), (4, 4)],
            [[4, 128], (4, 1), (4, 128)],
            [[128, 4], (128, 1), (32, 4)],
            [[128, 7], (128, 1), (32, 7)],
            [[128, 128], (128, 1), (16, 32)],

            [[4, 4, 4], (4, 1, 1), (4, 4, 4)],
            [[4, 4, 512], (4, 1, 1), (4, 4, 32)],
            [[512, 512, 4], (512, 1, 1), (8, 8, 4)],
            [[512, 512, 512], (512, 1, 1), (8, 8, 8)],

            [[3, 3, 666], (3, 1, 1), (3, 3, 37)],
            [[99, 99, 99], (99, 1, 1), (3, 11, 11)],
        ]
        for grid_shape, expected_cpu_local_size, expected_gpu_local_size in sizes:
            print("size {!s:16s}".format(grid_shape), end="")
            cpu_local_size = LocalSizeComputer(grid_shape, MockCPU).compute_local_size_bulky()
            gpu_local_size = LocalSizeComputer(grid_shape, MockIrisPro).compute_local_size_bulky()

            print(" d0 cpu local_size {!s:15s} gpu local_size {!s:15s}".format(cpu_local_size, gpu_local_size))

            self.assertListEqual(list(cpu_local_size), list(expected_cpu_local_size))
            self.assertListEqual(list(gpu_local_size), list(expected_gpu_local_size))


import random
from django.core.exceptions import ObjectDoesNotExist
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn.externals import joblib
from sklearn.naive_bayes import GaussianNB

def get_data_for_train(inputs_file_name="X.pkl", outputs_file_name="Y.pkl"):
    """
        Load training data from files if data exist
         in other way call method that create this data
    :param inputs_file_name:
    :param outputs_file_name:
    :return:
        X: features array,
        Y:  labels array
    """
    try:
        X = joblib.load(inputs_file_name)
        y = joblib.load(outputs_file_name)
    except ObjectDoesNotExist:
        X,y = create_training_data()

    return X,y

def get_estimator(file_name="CLF.pkl"):
    try:
        return joblib.load(file_name)
    except ObjectDoesNotExist:
        return create_estimator()

def create_training_data(n_examples=2000, inputs_file_name="X.pkl", outputs_file_name="Y.pkl"):
    """
    Generate trainig data for Neural network.
    3 input. 2 output.
     if inputs are correspondingly 1.0,2.0 and 3.0
      then outputs should be 9.0 and 1.0, otherwise outputs should be 5.0
    Saved data into files
    :param n_examples: Number of examples
    :param inputs_file_name: File name, where features will be saved
    :param outputs_file_name: File name, where labels will be saved
    """
    X = []
    Y = []
    for i in xrange(n_examples):
        p = random.randint(1,3)
        if p == 1:
            X.append([1,2,3])
            Y.append([9,1])
        else:
            Y.append([5,5])
            X.append([random.randint(1,100),random.randint(1,100),random.randint(1,100)])

    joblib.dump(np.array(X), inputs_file_name)
    joblib.dump(np.array(Y), outputs_file_name)

    return X,Y

def create_estimator(file_name="CLF.pkl"):

    X,y = get_data_for_train()
    clf = GaussianNB()

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=0)

    clf.fit(X_train, y_train.ravel())

    joblib.dump(clf, file_name)

    return clf

def get_score(file_name="CLF.pkl"):
    clf = get_estimator(file_name="CLF.pkl")
    return  clf.score()

def predict_value(input):
    """
    :param input:
    :return: predicted value
    """

    clf = get_estimator()

    return clf.predict(input())


import pandas as pd
#fever,nausea,lumpain,urine,micpain,burning,inf,neph,
df = pd.read_csv("diagnosis.csv")
df.head()
#inputs = pd.read_csv('diagnosis.txt', index_col=0)

#inouts = drop(['neph', 'inf',], axis=1)
#inputs = inputs.drop('inf',axis='columns')

inputs = df

from sklearn.preprocessing import LabelEncoder
le_fever = LabelEncoder()
le_nausea = LabelEncoder()
le_lumpain = LabelEncoder()
le_urine = LabelEncoder()
le_micpain = LabelEncoder()
le_burning = LabelEncoder()
le_inf = LabelEncoder()
le_neph = LabelEncoder()

inputs['fever_n'] = le_fever.fit_transform(inputs['fever'])
inputs['nausea_n'] = le_nausea.fit_transform(inputs['nausea'])
inputs['lumpain_n'] = le_lumpain.fit_transform(inputs['lumpain'])
inputs['urine_n'] = le_urine.fit_transform(inputs['urine'])
inputs['micpain_n'] = le_micpain.fit_transform(inputs['micpain'])
inputs['burning_n'] = le_burning.fit_transform(inputs['burning'])
#inputs['inf_n'] = le_inf.fit_transform(inputs['inf'])
#inputs['neph_n'] = le_neph.fit_transform(inputs['neph'])

target = inputs['inf']
inputs = df.drop(['neph','inf'],axis='columns')
inputs = df.loc[:, ~df.columns.str.contains('^Unnamed')]
print(inputs)
inputs_n = inputs.drop(['fever','nausea','lumpain','urine','micpain','burning','neph','inf'],axis='columns')
print(inputs_n)
print (target)

from sklearn import tree
model = tree.DecisionTreeClassifier()
model.fit(inputs_n, target)
model.score(inputs_n,target)

print("Do they have it? \n",model.predict([[0,0,0,1,1,1]]))
print(model.predict([[1,1,1,1,1,1]]))
print(model.predict([[0,0,0,0,0,0]]))