# This file is part of Tryton.  The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
from trytond import backend
from trytond.model import ModelSQL, fields
from trytond.pyson import Eval
from trytond.pool import PoolMeta, Pool
from trytond.tools.multivalue import migrate_property
from trytond.modules.company.model import (
    CompanyMultiValueMixin, CompanyValueMixin)

class Party(CompanyMultiValueMixin, metaclass=PoolMeta):
    __name__ = 'party.party'
    sale_price_list = fields.MultiValue(fields.Many2One(
            'product.price_list', "Sale Price List",
            help="The default price list for new sales.",
            domain=[
                ('company', '=', Eval('context', {}).get('company', -1)),
                ],
            states={
                'invisible': ~Eval('context', {}).get('company'),
                }))
    sale_price_lists = fields.One2Many(
        'party.party.sale_price_list', 'party', "Sale Price Lists")

    @classmethod
    def default_sale_price_list(cls, **pattern):
        pool = Pool()
        Configuration = pool.get('sale.configuration')
        config = Configuration(1)
        price_list = config.get_multivalue('sale_price_list', **pattern)
        return price_list.id if price_list else None

class PartySalePriceList(ModelSQL, CompanyValueMixin):
    "Party Sale Price List"
    __name__ = 'party.party.sale_price_list'
    party = fields.Many2One(
        'party.party', "Party", ondelete='CASCADE', select=True)
    sale_price_list = fields.Many2One(
        'product.price_list', "Sale Price List",
        domain=[
            ('company', '=', Eval('company', -1)),
            ],
        depends=['company'])

    @classmethod
    def __register__(cls, module_name):
        exist = backend.TableHandler.table_exist(cls._table)

        super(PartySalePriceList, cls).__register__(module_name)

        if not exist:
            cls._migrate_property([], [], [])

    @classmethod
    def _migrate_property(cls, field_names, value_names, fields):
        field_names.append('sale_price_list')
        value_names.append('sale_price_list')
        fields.append('company')
        migrate_property(
            'party.party', field_names, cls, value_names,
            parent='party', fields=fields)


import ast
import asyncio
import httpreq
import time
from logging import info as printf

proxy_list = []
index = 0

async def Init():
   printf("init")
   await GetHttpsProxyList()

async def GetHttpsProxyList():
    proxies = await httpreq.GetHtml('https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list')
    if proxies is None:
        printf("get raw proxy list error")
        return None
    # 使用'\n' 切分字符串,得到一个list
    resp = str(proxies['resp'], encoding='utf-8')
    lines = resp.splitlines(False)
    for line in lines:
        # 只保留高密代理,https的一定是高密的
        if (line.find('https') > 0):
            try:
                proxy = ast.literal_eval(line)
                proxy['used'] = False
                proxy['url'] = proxy['type'] + '://' + \
                               proxy['host']+ ':' + str(proxy['port'])
                proxy['fail_cnt'] = 0
                proxy_list.append(proxy)
            except Exception as e:
                printf(str(e))
    printf("got %d proxies", len(proxy_list))

def GetUnusedProxy():
    global index
    for proxy in proxy_list[index:]:
        if (proxy['used'] == False):
            index = index + 1
            if index == len(proxy_list):
                index = 0
            return proxy
    return None

def GetProxy():
    for i in range(10):
        proxy = GetUnusedProxy()
        if (proxy):
            return proxy
        printf("no available proxy,sleep 1s to retry")
        time.sleep(1)
    printf("no available proxy, retry 10 times")
    return None

def SetUnused(proxy):
    proxy['used'] = False

def SetUsed(proxy):
    proxy['used'] = True

def RemoveProxy(proxy):
    proxy_list.remove(proxy)

def IncFailCnt(proxy):
    proxy['fail_cnt'] = proxy['fail_cnt'] + 1
    if proxy['fail_cnt'] >= 3:
       proxy_list.remove(proxy)


#!/usr/bin/env python
#===============================================================================
# RouterAssign_Main
#
# Description:
#    Menu function to run router assignment and visualization scripts
#
# Version:
#    MM.mm    DD/MM/YY
#    00.00    09/08/17    First version with header description
#    00.01    14/12/17    Expand to 16 options (0-15) plus A-C
#
# Example/Usage:
#
#
#===============================================================================

import re

from archive.BuildAETable import buildAETableUnit
from archive.CompareChassisLayouts import buildRouterCompareUnit
from archive.FindListofRouters import createRouterListDevs, createRouterListHdwr
from archive.GetRouterOpenPorts import getOpenPortsUnit
from archive.GetRouterOpenPortsAll import getAllOpenPortsUnit
from archive.HelpDiagram import printHelpDiagram
from archive.ReadRouterConfig import readDeviceConfigFileUnit
from archive.ReadRouterConfigAll import buildAllRouterConfigs
from archive.ReadRouterHardware import readHardwareDescriptionFileUnit
from archive.ReadRouterHardwareAll import buildAllRouterHardware
from archive.SetChassisLayout import buildChassisLayoutUnit
from archive.SetChassisLayoutAll import buildAllRouterLayout
from archive.SetHardwareLayout import buildRouterHWLayoutUnit
from archive.SetHardwareLayoutAll import buildAllHardwareLayout
from archive.SetSimpleHWLayout import buildRouterSHWLayoutUnit
from archive.SetSimpleHWLayoutAll import buildAllSimpleHWLayout

from src import ClearRouterArtifacts

def displayMenu(menu, cfgStr, writeToFileFlag, printFlag):
    options = menu.keys()
    format1 = '{0:3} {1:2} {2:30} {3:3} {4:}'
    format2 = '{0:3} {1:2} {2:30}'

    print("MENU OPTIONS")
    print('Key    Task                              Flag/Notes')
    print('---    -------------------------------   ------------------')
    for entry in options: #sorted(options):
        if isinstance(entry, int):
            print(format1.format(entry, '::', menu[entry], '>>', cfgStr[entry]))
        elif entry == 'B':
            print(format1.format(entry, '::', menu[entry], '::', printFlag))
        elif entry == 'C':
            print(format1.format(entry, '::', menu[entry], '::', writeToFileFlag))
        else:
            print(format2.format(entry, '::', menu[entry]))
    print('\n')

def executeMenu(menu, cfgStr, writeToFileFlag, printFlag):
    # runs menu with a while loop
    while True:
        displayMenu(menu, cfgStr, writeToFileFlag, printFlag)

        selection = input('Please Select A Task (by typing a number/letter):')
        if selection == '0':
            readDeviceConfigFileUnit(writeToFileFlag, printFlag)
        elif selection == '1':
            buildAllRouterConfigs(printFlag)
        elif selection == '2':
            readHardwareDescriptionFileUnit(writeToFileFlag, printFlag)
        elif selection == '3':
            buildAllRouterHardware(printFlag)
        elif selection == '4':
            buildAETableUnit(writeToFileFlag, printFlag)

        elif selection == '5':
            buildChassisLayoutUnit(writeToFileFlag, printFlag)
        elif selection == '6':
            buildAllRouterLayout(printFlag)
        elif selection == '7':
            buildRouterHWLayoutUnit(writeToFileFlag, printFlag)
        elif selection == '8':
            buildAllHardwareLayout(printFlag)
        elif selection == '9':
            buildRouterSHWLayoutUnit(writeToFileFlag, printFlag)
        elif selection == '10':
            buildAllSimpleHWLayout(printFlag)

        elif selection == '11':
            getOpenPortsUnit(printFlag)
        elif selection == '12':
            getAllOpenPortsUnit(writeToFileFlag, printFlag)
        elif selection == '13':
            createRouterListDevs(printFlag)
        elif selection == '14':
            createRouterListHdwr(printFlag)

        elif selection == '15':
            buildRouterCompareUnit(writeToFileFlag, printFlag)

        elif selection == '16':
            ClearRouterArtifacts.main()

        elif selection == 'A' or selection == 'a':
            print('program terminated\n\n')
            break

        elif selection == 'B' or selection == 'b':
            programQuery = input('Print output tables (type True/t or False/f)?:')
            if re.search('t',programQuery) or re.search('T',programQuery):
                printFlag = True
            else:
                printFlag = False
            print('\n')

        elif selection == 'C' or selection == 'c':
            programQuery = input('Save file to hard drive (type True/t or False/f)?:')
            if re.search('t',programQuery) or re.search('T',programQuery):
                writeToFileFlag = True
            else:
                writeToFileFlag = False
            print('\n')
        elif selection == 'D' or selection == 'd':
            printHelpDiagram(printFlag=True)

        else:
            print('Please Choose A Valid Task!\n')

def buildMenu(writeToFileFlag, printFlag):
    # build dictionary to hold menu options
    menu = {}
    menu[0] = 'Read Router Config'
    menu[1] = '---- ALL: Router Configs'
    menu[2] = '---- Router Hardware'
    menu[3] = '---- ALL: Router Hardware'
    menu[4] = 'Build AE Summary Table'
    menu[5] = '----- Router Ports Table'
    menu[6] = '---- ALL: Router Ports Tables'
    menu[7] = 'Get: Layout: Hardware'
    menu[8] = '--- ALL: Layouts: Hardware'
    menu[9] = '---: Layout: Simple HW'
    menu[10] = '--- ALL: Layouts: Simple HW'
    menu[11] = 'Print: Router Opens'
    menu[12] = '----- ALL: Routers Opens'
    menu[13] = '-----: Router list from CFG'
    menu[14] = '-----: Router list from HW'
    menu[15] = 'Compare Router Ports Table'
    menu[16] = 'CLEAR: Clear all py Artifacts'

    menu['A'] = 'Exit Program'
    menu['B'] = 'Set Verbose Print Flag'
    menu['C'] = 'Set File Write Flag'
    menu['D'] = 'Print Help Diagram'

    cfgStr = {}
    cfgStr[0] = 'Build Router-specific config file from raw file'
    cfgStr[1] = '....: ALL Router-specific config files from raw file'
    cfgStr[2] = '....: Router-specific hardware file from raw file'
    cfgStr[3] = '....: ALL Router-specific hardware files from raw file'
    cfgStr[4] = 'Build AE summary from config file (*CFG.txt)'
    cfgStr[5] = '....: Router slot/port layout from config file (*CFG.txt): 1 Site'
    cfgStr[6] = '....: Router slot/port layouts from config file (*CFG.txt): All Sites'
    cfgStr[7] = '....: Router hardware layout from hardware file (*HW.txt): 1 Site'
    cfgStr[8] = '....: Router hardware layouts from hardware file (*HW.txt): All Sites'
    cfgStr[9] = '....: Simple hardware layout from HW layout file (*HWL.txt): 1 Site'
    cfgStr[10] = '....: Simple hardware layouts from HW layout file (*HWL.txt): All Sites'
    cfgStr[11] = 'Print Router open ports/slots quantity from layout file (*VA.txt): 1 Site'
    cfgStr[12] = '....: Router open ports/slots quantities from layout files (*VA.txt): All Sites'
    cfgStr[13] = '....: List of Router names from config file (*CFG.txt)'
    cfgStr[14] = '....: List of Router names from hardware file (*HW.txt)'
    cfgStr[15] = 'Compare Router slot/port layouts from config file (*CFG.txt)'
    cfgStr[16] = 'Clear artifacts (py-type files) from directory'

    executeMenu(menu, cfgStr, writeToFileFlag, printFlag)

def main():
    # call build menu method
    printFlag = True
    writeToFileFlag = False
    buildMenu(writeToFileFlag, printFlag)

if __name__ == '__main__':
    main()


import random

population = []
total = []

def addPopulation():
    temp = []
    for i in range (0, 8):
        temp.append(random.randint(0, 7))
    population.append(temp)

def calculateFitness(lis):
    count = 0
    values = []
    for i in range (0,8):
        for j in range (i+1, 8):
            if(lis[i] == lis[j]):
                count = count + 1
                if lis[i] not in values:
                    values.append(lis[i])

            if((lis[j] - lis[i]) == (j - i)):
                count = count + 1
                if lis[i] not in values:
                    values.append(lis[i])

            if((lis[i] - lis[j]) == (j - i)):
                count = count + 1
                if lis[i] not in values:
                    values.append(lis[i])
    fitness = 28-count
    return fitness, values

def crossover(lis1, lis2):
    index = random.randint(0, 7)
    for i in range (0, index+1):
        lis1[i], lis2[i] = lis2[i], lis1[i]
    fitness, values = calculateFitness(lis1)
    if(fitness != 28):
        lis1 = mutation(lis1, values)
    fitness, values = calculateFitness(lis2)
    if(fitness != 28):
        lis2 = mutation(lis2, values)
    return lis1, lis2

def mutation(lis, values):
    n = random.randint(0, len(values)-1)
    idx = lis.index(values[n])
    x = random.randint(0, 7)
    while(x == values[n]):
        x = random.randint(0, 7)
    lis[idx] = x
    return lis

for i in range (0, 8):
    addPopulation()
count = 0
while(True):
    count = count + 1
    x = random.randint(0, len(population)-1)
    y = random.randint(0, len(population)-1)
    while y==x:
        y = random.randint(0, len(population)-1)

    population[x], population[y] = crossover(population[x], population[y])

    fitness, values =  calculateFitness(population[x])
    if fitness == 28:
        total = population[x]
        break;

    fitness, values =  calculateFitness(population[y])
    if fitness == 28:
        total = population[x]
        break;
print("*****Resulting 8-Queen Sequence(Row and column starts from 0)*****")
print(total)


'''
MMU Simulator in python
Assignment 3, Computer Architecture
Authors: Akilesh K, Arjun Menon V
Sept 2021
Main Module: - Create kernel memory, user memory and process objects
             - Call functions for pagewalk, page eviction (LRU policy) and writing evicted PageTable
               back to Disk (WriteBack policy)
'''
import numpy as np
import argparse as ap
from inc.opts import *
from inc.parser import inputParser
from memory import memory
from translator import translate
from process import proc

def args():
    '''
    Parse command-line arguments

    '''
    parser = ap.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, metavar='file', help="Input file with requests", required=True)
    args = parser.parse_args()
    return args

def main():
    '''
    Main simulation function

    '''
    activeProcesses = [None]*NPROC
    userMem = memory()
    '''
     - kernel memory allocation type 1: give MMU access to the entire kernel space
     - for 0 page table evictions for the given page access pattern, only 40 page frames are needed by the mmu
    '''
    kernelMem = memory(256, 1024, "kernel")
    # kernel memory allocation type 2: Reserving only 64K for Page Directories and Page Tables
    #kernelMem = memory(16, 1024, "kernel")
    inFile = args().input
    fileParser = inputParser(inFile)
    reqs = fileParser.parse()
    totalRequests = len(reqs)
    totalHits = 0
    totalMisses = 0
    totalPageReplacements = 0
    totalTableReplacements = 0
    totalReads = 0
    totalWrites = 0
    for req in reqs:
        if req.typ == 'r':
            totalReads += 1
        elif req.typ == 'w':
            totalWrites += 1
        # check if the request is from a new process
        if activeProcesses[req.pid] == None:
            newProc = proc(req.pid)
            activeProcesses[req.pid] = newProc
        offsets = translate(req.va)
        p = activeProcesses[req.pid]
        page, rv1, rv2 = p.pagewalk(kernelMem, userMem, offsets)

        if rv1[0] != 0:
            evictedPID, evictedOffset, evictedTable = rv1[1], rv1[2], rv1[3]

            for pr in activeProcesses:
                if pr != None:
                    if pr.pid == evictedPID:
                        pr.pTableCopy[evictedOffset] = evictedTable

        if rv2[0] != 0:
            evictedUserFrame = rv2[1]
            for pr in activeProcesses:
                if pr != None:
                    for table in pr.TableCopy:
                        for entry in table:
                            if entry == evictedUserFrame:
                                entry = None

    for p in activeProcesses:
        if p != None:
            totalHits += p.hits
            totalMisses += p.misses
    print('Total Requests: {}'.format(totalRequests))
    print('Hit Rate: {:.2f}%'.format(100*totalHits/totalRequests))
    print('Miss Rate: {:.2f}%'.format(100*totalMisses/totalRequests))
    print('Read Requests: {}'.format(totalReads))
    print('Write Requests: {}'.format(totalWrites))
    for p in activeProcesses:
        if p != None:
            print('PID {}: {} requests'.format(p.pid, p.requests))
            totalPageReplacements += p.pageEvictions
            totalTableReplacements += p.tableEvictions
    print('Dirty Page Evictions: {}'.format(totalPageReplacements))
    print('Dirty Table Evictions: {}'.format(totalTableReplacements))

    totalKernelPFUsed = max(kernelMem.LRUctr)+1 # count starts from 0
    totalUserPFUsed = max(userMem.LRUctr)+1
    numActiveProcesses = 0
    for p in activeProcesses:
        if p != None:
            numActiveProcesses += 1
    print('Final number of pageframes used: {}'.format(totalKernelPFUsed+totalUserPFUsed+numActiveProcesses))
    print('\tKernel pageframes: {}'.format(totalKernelPFUsed+numActiveProcesses))
    print('\tUser pageframes: {}'.format(totalUserPFUsed))

if __name__=="__main__":
    main()
    # print("Please check back later. In development")


def main():
    module = AnsibleModule(argument_spec=dict(src=dict(type='path', required=True, aliases=['path']), minfw=dict(type='str'), executable=dict(default='hponcfg', type='str'), verbose=dict(default=False, type='bool')))
    changed = True
    src = module.params['src']
    minfw = module.params['minfw']
    executable = module.params['executable']
    verbose = module.params['verbose']
    options = (' -f %s' % src)
    if verbose:
        options += ' -v'
    if minfw:
        options += (' -m %s' % minfw)
    (rc, stdout, stderr) = module.run_command(('%s %s' % (executable, options)))
    if (rc != 0):
        module.fail_json(rc=rc, msg='Failed to run hponcfg', stdout=stdout, stderr=stderr)
    module.exit_json(changed=changed, stdout=stdout, stderr=stderr)


PREAMBLE_SIZE = 25
START_POSITION = 26

def read_file(filename):
    with open(filename) as f:
        content = f.readlines()
    content = [int(x.strip()) for x in content]
    return content

def not_in_preamble(preamble, n):
    b = False
    for k in preamble:
        for j in preamble:
            if k == j:
                continue
            if (k+j) == n:
                b = True
    return not b

def main(filename):
    c = read_file(filename)
    for i in range(START_POSITION, len(c), 1):
        b = not_in_preamble(c[i-PREAMBLE_SIZE-1:i], c[i])
        if b:
            return c[i]

if __name__ == "__main__":
    print(main("input-9.txt"))


from SpotifyETL import SpotifyETL, clean_entities
import datetime
import time
import logging
import sqlalchemy
import pyodbc
import config

date_stage = datetime.datetime.today()
current_date = date_stage.strftime('%m-%d-%Y')

#defining logger conditions
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

formatter = logging.Formatter('	%(asctime)s - %(name)s - %(message)s')

file_handler = logging.FileHandler(f'./logs/{current_date} app.log')
file_handler.setFormatter(formatter)

start_time = time.time()

#initiate ShopifyETL Class
s = SpotifyETL()

#call get_recent_songs method
print('Getting Recent Songs')
s.get_recent_songs()
recent_length = len(s.get_recent_songs())
print(f'Retrieved {recent_length} songs')

#call get artist ids method
print('Getting Recent Artists')
s.get_artist_ids()
artist_length = len(s.get_artist_ids())
print(f'Retrieved {artist_length} artists')

#call get album ids method
print('Getting Recent Albums')
s.get_album_ids()
album_length = len(s.get_album_ids())
print(f'Retrieve {album_length} albums')

#create_Spotify csvs for staging
print('Creating Track Entity')
track_entity = s.create_track_entity()
print('Track Entity Created')
print('Creating Artist Entity')
artist_entity = s.create_artist_entity()
print('Artist Entity Created')
print('Creating Album Entity')
album_entity = s.create_album_entity()
print('Album Entity Created')

print('Storing Entities as CSVs')
track_entity.to_csv(f'./output/tracks {current_date}.csv')
artist_entity.to_csv(f'./output/artists {current_date}.csv')
album_entity.to_csv(f'./output/albums {current_date}.csv')

#establishing SQL Server Engine to Azure DB
engine = config.engine

# write the DataFrame to a table in the sql database
print('Writing DataFrames to Azure DB Tables')
track_entity.to_sql("tracks", schema='dbo', if_exists='append', con=engine)
artist_entity.to_sql("artists", schema='dbo', if_exists='append', con=engine)
album_entity.to_sql("albums", schema='dbo', if_exists='append', con=engine)

print('Cleaning Album and Artist Tables')
clean_tracks_df = clean_entities('tracks.sql', 'track_id')
clean_artist_df = clean_entities('artists.sql', 'artist_id')
clean_album_df = clean_entities('albums.sql', 'album_id')
track_data_types = {'track_id': sqlalchemy.types.VARCHAR(50)}
artist_data_types = {'artist_id': sqlalchemy.types.VARCHAR(50)}
album_data_types = {'album_id': sqlalchemy.types.VARCHAR(50)}
clean_tracks_df.to_sql("tracks", schema='dbo', if_exists='replace', dtype=track_data_types, con=engine)
clean_artist_df.to_sql("artists", schema='dbo', if_exists='replace', dtype=artist_data_types, con=engine)
clean_album_df.to_sql("albums", schema='dbo', if_exists='replace', dtype=album_data_types, con=engine)
print('Azure DB Tables Populated')

end_time = time.time()

total_time = end_time - start_time

logger.debug(f'Process took {int(total_time/60)} minutes and {(total_time % 60):.2f} seconds')


# -*- coding: utf-8 -*-
"""
Created on Tue Feb 25 14:08:25 2020

@author: xinyu
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Feb 25 11:24:46 2020

@author: xinyu
"""

#import ruptures as rpt
from scipy import signal, stats, fftpack, arange
from scipy.signal import butter, lfilter
import matplotlib.pyplot as plt
from pylab import plot, show, title, xlabel, ylabel, subplot, bar
from sklearn.model_selection import train_test_split, ShuffleSplit
import tensorflow as tf
import random
import numpy as np
import matplotlib.ticker as ticker
import os

class dataClass:
    def __init__(self, dataIn, idNum, measNum):
        dataIn = dataIn[(dataIn[:,0]==idNum) & (dataIn[:,6]==measNum)]
        self.t = dataIn[:,1]
        self.s = dataIn[:,2]
        self.p = dataIn[:,3]
        self.F = dataIn[:,4]
        self.A = dataIn[:,5]

class dataSet:
    def __init__(self, varIn):
        self.dat = varIn

class resClass:
    def __init__(self, resIn, idNum, measNum):
        res = resIn[(resIn[:,0]==idNum) & (resIn[:,1]==measNum)].T
        self.to = res[2]
        self.Fo = res[3]
        self.so = res[4]
        self.tc = res[5]
        self.Fc = res[6]
        self.sc = res[7]

class infoClass:
    def __init__(self, infoIn, idNum):
        self.info = infoIn[(infoIn[:,0]==idNum)].T
        self.p_set = self.info[1]
        self.F_tol = self.info[2]
        self.s_tol = self.info[3]
        self.p_sys = self.info[4]
        #self.DN = self.info[5]
        self.PN = self.info[5]
        self.d0 = self.info[6]
        self.kalt_heiss = self.info[7]

class timeSeries:
    def __init__(self, db, idNum, measNum):
        data = db[0]
        result = db[1]
        info = db[2]
        self.idNum = idNum
        self.measNum = measNum
        self.data = dataClass(data, idNum, measNum)
        self.points = resClass(result, idNum, measNum)
        self.info = infoClass(info,idNum)

###############################################################################

def loop_from_to(IDs, id_from, steps):
    if id_from == 0:
        id_from = 1
        id_to = IDs + 1
    else:
        id_to = id_from + steps + 1
    return id_from, id_to

def safe_div(x,y):
    if y == 0:
        return 0
    return x / y

def delta(var):
    if var.size > 0:
        dvar = var[1:]-var[:-1]
        dvar = np.append(dvar,0)
        dvar[-1] = dvar[-2]
        return dvar
    else:
        return var

def grad(var, idNum):
    if var.size > 0:
        if idNum < 775:
            grad = np.gradient(var, 0.1)
        else:
            grad = np.gradient(var, 0.05)
        return grad
    else:
        return var

def diff(var):
    dvar = np.diff(var)
    dvar = np.append(dvar,dvar[-1])
    return dvar

def diff2(var):
    dvar = np.diff(var,n=2)
    dvar = np.append(dvar,dvar[-1])
    return dvar

def grad_fwd(var,idNum,win_len):
    if var.size > 0:
        if idNum < 775:
            sampRate = 0.1
        else:
            sampRate = 0.05

        grad_fwd = np.zeros(len(var))

        for l in range(0,len(var)-win_len):
            grad_fwd[l] = (var[l+win_len] - var[l])/(win_len*sampRate)
        for l in range(len(var)-win_len, len(var)-1):
            w = (len(var)-1)-l
            grad_fwd[l] = (var[l+w] - var[l])/(w*sampRate)
        grad_fwd[len(var)-1] = (var[-1] - var[-2])/sampRate
        return grad_fwd
    else:
        return var

def grad_bwd(var,idNum,win_len):
    if var.size > 0:
        if idNum < 775:
            sampRate = 0.1
        else:
            sampRate = 0.05

        grad_bwd = np.zeros(len(var))
        grad_bwd[0] = (var[1] - var[0])/sampRate
        for l in range(1,win_len):
            grad_bwd[l] = (var[l] - var[0])/(win_len*sampRate)
        for l in range(win_len, len(var)):
            grad_bwd[l] = (var[l] - var[l-win_len])/(win_len*sampRate)
        return grad_bwd
    else:
        return var

def grad_win(var,idNum,win_len):
    if var.size > 0:
        if idNum < 775:
            sampRate = 0.1
        else:
            sampRate = 0.05

        grad_win = np.zeros(len(var))

        grad_win[0] = (var[1] - var[0])/sampRate
        for l in range(1,win_len):
            grad_win[l] = (var[l+l] - var[l-l])/(2*l*sampRate)
        for l in range(win_len,len(var)-win_len):
            grad_win[l] = (var[l+win_len] - var[l-win_len])/(2*win_len*sampRate)
        for l in range(len(var)-win_len, len(var)):
            grad_win[l] = (var[-1] - var[l -((len(var)-1)-l)])/(2*((len(var)-1)-l)*sampRate)
        grad_win[-1] = (var[-1] - var[-2])/sampRate
        return grad_win
    else:
        return var

def smooth(var,win_len,polyorder):
    try:
        var = signal.savgol_filter(var, window_length=win_len, polyorder=polyorder, mode='nearest')
        return var
    except ValueError:
        return var

def norm_std(var):
    var = (var-var.mean(0))/var.std(0)
    return var
def norm(var):
    if max(var)-min(var) != 0:
        var = (var-min(var))/(max(var)-min(var))
        return var
    else:
        #print("Input could not be normalized")
        return var

def norm2(var):
    if max(abs(var)) != 0:
        var = var/(max(abs(var)))
        return var
    else:
        return var

def norm3(var):
    for i in range(0,len(var)):
        if var[i] < 0:
            var[i]=0
    try:
        var = var/(max(abs(var)))
        return var
    except ValueError:
        return var

def detectOpt(signalY, signalX):

    signal = np.column_stack((signalY,signalX))
    n_bkps = 4
    #c = rpt.costs.CostLinear().fit(signal)
    # change point detection
    model = "linear" # "l1", "l2", "rbf", "normal", "linear"
    algo = rpt.Dynp(model=model, min_size=1, jump=1).fit(signal)
    bkps = algo.predict(n_bkps)
    return bkps

def butter_lowpass(lowcut, fs, order=2):
    nyq = 0.5 * fs
    low = lowcut / nyq
    b, a = butter(order, low, btype='lowpass')
    return b, a

def butter_lowpass_filter(data, lowcut, fs, order=5):
    b, a = butter_lowpass(lowcut, fs, order=order)
    y = lfilter(b, a, data)
    return y

def butter_zerophase_filter(data,lowcut,fs,order=2):
    b, a = butter_lowpass(lowcut, fs, order=order)
    y = signal.filtfilt(b, a, data)
    return y

def butter_bandpass(lowcut, highcut, fs, order=5):
    nyq = 0.5 * fs
    low = lowcut / nyq
    high = highcut / nyq
    b, a = butter(order, [low, high], btype='band')
    return b, a

def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
    b, a = butter_bandpass(lowcut, highcut, fs, order=order)
    y = lfilter(b, a, data)
    return y

def round_arr(data):
    data[:,2] = np.round(data[:,2],3)
    data[:,3] = np.round(data[:,3],7)
    return data

def create_y(data,idxo):
    y = np.zeros(len(data))
    y[idxo[0]:] = 1
    return y

def frequency_sepectrum(x, sf):
    x = x - np.average(x)  # zero-centering
    n = len(x)
    k = np.arange(n)
    tarr = n / float(sf)
    frqarr = k / float(tarr)  # two sides frequency range

    frqarr = frqarr[range(n // 2)]  # one side frequency range

    x = fftpack.fft(x) / n   # fft computing and normalization
    x = x[range(n // 2)]
    return frqarr, abs(x)

def interpo_point(x, p): #makes sure that Fo exists in resampled data
    n = 0
    for i in x:
        if i == p:
            break
        elif i > p:
            if (i-p)>(x[n-1]-p):
                x[n-1] = p
                n -= 1
            else:
                x[n] = p
            break
        n += 1
    return x, n

def generate_train(data,Po,l):
    data = signal.resample(data,l)
    train_x, n = interpo_point(data,Po)
    if n >= l:
        return np.zeros(l),0,0
    train_y = np.zeros(l)
    #train_y[n] = 2
    train_y[n:] = 1
    return train_x, train_y, n
def padding_len(xseq):
    Len = []
    for i in range(len(xseq)):
        Len.append(len(xseq[i]))
    pad_len = np.array(Len).max()
    return pad_len
def data_padding(Xseq,Yseq):
    padlen = padding_len(Xseq)
    padded_xseq = np.zeros((len(Xseq),padlen,Xseq[0].shape[-1]))
    padded_yseq = np.zeros((len(Yseq),padlen,1))
    for i in range(len(Xseq)):
        padded_xseq[i,...] = np.pad(Xseq[i],((0,padlen-len(Xseq[i])),(0,0)),'constant',constant_values = 0.)
        padded_yseq[i,...] = np.pad(Yseq[i].reshape(-1,1),((0,padlen-len(Yseq[i])),(0,0)),'constant',constant_values = 0.)
    return padded_xseq, padded_yseq

def gate(y_pred):
    y_bool = y_pred!=y_pred[-1]
    y =y_pred[y_bool==True]
    idx = np.where(y_pred == y[-1])[0][-1]
    y_ = y_pred[:idx+1]
    return y_
def evaluate_plot(*ypre,to):
    To = np.zeros(to.shape)
    for y_pre in ypre:
        for i in range(y_pre.shape[0]):
            y_ = gate(y_pre[i])
            To[i] = np.argmax(y_)
        diff_to = np.array(To)-np.array(to)
        diff_to = np.where(np.abs(diff_to)>=3,3,diff_to)
        unique, count = np.unique(diff_to, return_counts=True)
        fig = plt.figure(1)
        ax = fig.add_subplot(1,1,1)
        ax.bar(unique, 100*count/len(to))

        x_ticks_labels = ['-2','-1','0','1','2','>3']
        fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
        yticks = ticker.FormatStrFormatter(fmt)
        ax.yaxis.set_major_formatter(yticks)
        plt.xticks(np.arange(6)-2, x_ticks_labels)
        #ax.set_xticklabels(x_ticks_labels)

        xlabel('Error in timesteps')
        ylabel('Distribution')
        plt.show()
    return unique, count, diff_to

def FoErrEval(y_,F,Fos):
    Fo_pred = [F[i][np.argmax(gate(y_[i]))] for i in range(y_.shape[0])]
    FoErr = (np.array(Fos)-np.array(Fo_pred))/np.array(Fos)
    FoErr = np.array([round(x,2) for x in FoErr])
    FoErr = np.where(FoErr<-1,-1,FoErr)
    for i in np.linspace(0,0.4,5):
        low = np.where(FoErr>i)[0]
        up = np.where(FoErr<=i+0.1)[0]
        FoErr[np.intersect1d(low, up)]=i+0.1
        low = np.where(FoErr>=-i-0.1)[0]
        up = np.where(FoErr<-i)[0]
        FoErr[np.intersect1d(low,up)]=-i-0.1
    FoErr = np.where(FoErr>0.5,1,FoErr)
    FoErr = np.where(FoErr<-0.5,-1,FoErr)
    unique, count = np.unique(FoErr, return_counts=True)
    fig = plt.figure(1)
    ax = fig.add_subplot(1,1,1)
    ax.bar(100*unique, 100*count/len(FoErr),width=8)
    fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
    yticks = ticker.FormatStrFormatter(fmt)
    ax.yaxis.set_major_formatter(yticks)
    fmt = '%.2f%%'
    xticks = ticker.FormatStrFormatter(fmt)
    ax.xaxis.set_major_formatter(xticks)
    xlabel('Fo_err/Fo [%]')
    ylabel('Distribution')
    plt.show()
    return Fo_pred,unique,count

def evaluate_dense(y_pred,Yseq):
    xfrom=0
    xto=0
    idxo_pred = []
    idxo_true = []
    for i in Yseq:
        xto = xto + len(i)
        idxo_pred.append(np.argmax(y_pred[xfrom:xto]))
        idxo_true.append(np.argmax(i))
        xfrom = xto

    Err = np.array([idxo_pred[i] - idxo_true[i] for i in range(len(idxo_pred))])

    err = np.where(Err>=3,3,Err)
    err = np.where(err<=-3,-3,err)
    unique, count = np.unique(err, return_counts=True)
    fig = plt.figure(1)
    ax = fig.add_subplot(1,1,1)
    ax.bar(unique, 100*count/len(err))

    x_ticks_labels = ['<-3','-2','-1','0','1','2','>3']
    fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
    yticks = ticker.FormatStrFormatter(fmt)
    ax.yaxis.set_major_formatter(yticks)
    plt.xticks(np.arange(7)-3, x_ticks_labels)
    ax.set_xticklabels(x_ticks_labels)

    xlabel('Error in timesteps')
    ylabel('Distribution')
    plt.show()
    return Err

def FoErrEvalDense(y_,F,Fos):
    xfrom=0
    xto=0
    Fo_pred = []
    j=0
    for i in Yseq:
        xto = xto + len(i)
        Fo_pred.append(F[j][np.argmax(y_[xfrom:xto])])
        xfrom = xto
        j+=1
    FoErr = (np.array(Fos)-np.array(Fo_pred))/np.array(Fos)
    FoErr = np.array([round(x,2) for x in FoErr])
    FoErr = np.where(FoErr<-1,-1,FoErr)
    for i in np.linspace(0,0.4,5):
        low = np.where(FoErr>i)[0]
        up = np.where(FoErr<=i+0.1)[0]
        FoErr[np.intersect1d(low, up)]=i+0.1
        low = np.where(FoErr>=-i-0.1)[0]
        up = np.where(FoErr<-i)[0]
        FoErr[np.intersect1d(low,up)]=-i-0.1
    FoErr = np.where(FoErr>0.5,1,FoErr)
    FoErr = np.where(FoErr<-0.5,-1,FoErr)
    unique, count = np.unique(FoErr, return_counts=True)
    fig = plt.figure(1)
    ax = fig.add_subplot(1,1,1)
    ax.bar(100*unique, 100*count/len(FoErr),width=8)
    fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
    yticks = ticker.FormatStrFormatter(fmt)
    ax.yaxis.set_major_formatter(yticks)
    fmt = '%.2f%%'
    xticks = ticker.FormatStrFormatter(fmt)
    ax.xaxis.set_major_formatter(xticks)
    xlabel('Fo_err/Fo [%]')
    ylabel('Distribution')
    plt.show()
    return Fo_pred,unique,count

###############################################################################

'''load measurement data and result data from respective txt-files, skip header rows'''
dataArr = np.loadtxt("U:/Data/OUTPUT_diagram_ll_clean.txt", skiprows=1) #contains raw measurement data
resArr = np.loadtxt("U:/Data/OUTPUT_tocke_ll_clean_2020.txt", skiprows=1) #contains determined opening point for each measurement
infoArr = np.loadtxt("U:/Data/OUTPUT_meritve_ll.txt", skiprows=1) #additional information
IDs = int(dataArr[-1,0]) #Anzahl Messdurchläufe in der Datenbank mit jeweils bis zu 3 Messungen

DB = [dataArr,resArr,infoArr]

initialize = 1
trim = 1

loopThroughData = 1
sequence = 1

ruleBased = 0
findBySort = 0
findBySum = 1

optSegment = 0
segFlinear = 1
segFslinear = 0
segPlt = 0
savefigure = 0

NNseq = 1

showPlt = 0
evaluation = 1

gradWin = 0
gradFwd = 0
gradBwd = 1
loadmodel = 1

if initialize != 0:
    dataList = []
    featureList = []
    X = np.zeros(shape = (1,12))
    Y = np.zeros(shape=1)
    F_origin = np.zeros(shape=1)
    raw_F = []
    FSA = []
    FSAtrim = []
    tos = np.zeros(1+IDs*3)
    Fos = np.zeros(1+IDs*3)
    idxos = np.zeros(1+IDs*3)
    Fos_pred = np.zeros(1+IDs*3)
    idxos_pred = np.zeros(1+IDs*3)
    Xseq = []
    Yseq = []
    Frng = np.zeros(1+IDs*3)
    label=[]
    measLen = np.zeros(1+IDs*3)

    #errArr = np.zeros((IDs,6))
if loopThroughData !=0:
    #r = random.randrange(1,935)
    id_from, id_to = loop_from_to(IDs, id_from=0, steps=0) #id_from=0 loops through all
    loopNo = (id_from-1)*3 #adjust initial value according to id_from
    for i in range(id_from,id_to):
        for j in range(1,4):
            print(str(i) + ", " + str(j))
            loopNo = loopNo + 1

            dat = timeSeries(DB,i,j).data #data
            res = timeSeries(DB,i,j).points #data labels
            info = timeSeries(DB,i,j).info #additional labels

            dataList.append(dat)

            t = dat.t
            F = dat.F
            s = dat.s
            p = dat.p
            A = dat.A

            to = res.to #opening time
            Fo = res.Fo #opening force

            if F.size > 0:
                Frng[loopNo] = max(F)-min(F)
            if to.size > 0:
                tos[loopNo] = to

            if F.size > 0 and Fo.size > 0 and to.size > 0 and to > 0.1:
                to = round(to[0],2)
                idxo = np.searchsorted(t,to,'left')
                idxos[loopNo] = idxo
                Frng[loopNo] = max(F)-min(F)
                raw_F.append(F)
                y = np.zeros_like(F)
                y[idxo] = 1
                Fos[loopNo] = Fo
            else:
                 continue

            if trim != 0:
                trim_to = np.argmax(F) + 5
                measLen[loopNo] = trim_to
                label.append([i,j])
                F = F[:trim_to]
                s = s[:trim_to]
                A = A[:trim_to]
                t = t[:trim_to]
                y = y[:trim_to]

            if gradFwd != 0:
                gradF = grad_fwd(F,i,1)
                grad2F = grad_fwd(gradF,i,1)
                grad3F = grad_fwd(grad2F,i,1)
                grads = grad_fwd(s,i,3)
                grad2s = grad_fwd(grads,i,3)
                grad3s = grad_fwd(grad2s,i,3)
                #gradA = grad_fwd(A,i,5)
                #grad2A = grad_fwd(gradA,i,5)
                #grad3A = grad_fwd(grad2A,i,5)
            elif gradBwd !=0:
                gradF = grad_bwd(F,i,1)
                grad2F = grad_fwd(gradF,i,1)
                grad3F = grad_bwd(grad2F,i,1)
                grads = grad_bwd(s,i,1)
                grad2s = grad_fwd(grads,i,1)
                grad3s = grad_bwd(grad2s,i,1)
                gradA = grad_bwd(A,i,1)
                grad2A = grad_fwd(gradA,i,1)
                grad3A = grad_bwd(grad2A,i,1)
            else:
                gradF = grad_win(F,i,1)
                grad2F = grad_win(gradF,i,1)
                grad3F = grad_win(grad2F,i,1)
                grads = grad_win(s,i,3)
                grad2s = grad_win(grads,i,3)
                grad3s = grad_win(grad2s,i,3)
                #gradA = grad_win(A,i,1)
                #grad2A = grad_win(gradA,i,1)
                #grad3A = grad_win(grad2A,i,1)

            Fn = norm2(F)
            gradFn = norm2(gradF)
            grad2Fn = norm2(grad2F)
            grad3Fn = norm2(grad3F)

            sn = norm2(s)
            gradsn = norm2(grads)
            grad2sn = norm2(grad2s)
            grad3sn = norm2(grad3s)

            An = norm2(A)
            gradAn = norm2(gradA)
            grad2An = norm2(grad2A)
            grad3An = norm2(grad3A)
            if NNseq != 0:
                x = np.column_stack((Fn, gradFn, grad2Fn, grad3Fn,
                                     sn, gradsn, grad2sn, grad3sn,
                                     An, gradAn, grad2An, grad3An))

                if X.size != 0:
                    X = np.concatenate((X, x), axis=0)
                    Y = np.concatenate((Y, y), axis=0)
                    F_origin = np.concatenate((F_origin, F), axis=0)
                else:
                    X = x
                    Y = y
                    F_origin = F

###########-Form data in sequential
            if sequence > 0:
                Xseq.append(np.array(x))
                Yseq.append(np.array(y))

#####################################################################################
# Datasplit

n = np.arange(len(Xseq))
label = np.array(label)
test_size = 0.15
trial,uniidx = np.unique(label[:,0],return_index=True)
test_vol = test_size*len(Xseq)/len(trial)
ss = ShuffleSplit(n_splits=1,test_size=test_vol,random_state=10)
for trainidx, testidx in ss.split(trial):
    test = uniidx[testidx]

train = n[np.isin(n, test,invert=True)]

entire = n

#Data for Feedforward Dense NN
Trainx = [Xseq[i] for i in train]
Testx = [Xseq[i] for i in test]
Trainy = [Yseq[i] for i in train]
Testy = [Yseq[i] for i in test]
x_train = np.array([y for x in Trainx for y in x]  )
x_test = np.array([y for x in Testx for y in x] )
y_train = np.array([y for x in Trainy for y in x])
y_test= np.array([y for x in Testy for y in x] )
XX = np.array([y for x in Xseq for y in x]  )
YY = np.array([y for x in Yseq for y in x]  )

#Data for Recurrent layer
sequential_x, sequential_y = data_padding(Xseq,Yseq)
idxo_masked = np.array([np.argmax(y) for y in sequential_y])
recurr_xtrain = np.array([sequential_x[i] for i in train])
recurr_xtest = np.array([sequential_x[i] for i in test])
recurr_ytrain = np.array([sequential_y[i] for i in train])
recurr_ytest = np.array([sequential_y[i] for i in test])

########################################################################
if loadmodel:
    model_dense = tf.keras.models.load_model('dense_withA.h5')
    model_lstm = tf.keras.models.load_model('lstm_withA.h5')
    model_bilstm = tf.keras.models.load_model('bilstm_withA.h5')

else:
    # Feedforward Dense NN
    model_dense = tf.keras.models.Sequential()
    model_dense.add(tf.keras.layers.Dense(80, input_dim=12, activation='relu'))
    model_dense.add(tf.keras.layers.Dense(50, activation='relu'))
    model_dense.add(tf.keras.layers.Dense(30, activation='relu'))
    model_dense.add(tf.keras.layers.Dense(1, activation='sigmoid'))
    model_dense.summary()
    model_dense.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.Precision(),tf.keras.metrics.Recall()])
    history = model_dense.fit(x_train, y_train,validation_split=0.15, shuffle = True,
                  epochs=20)

    # LSTM NN
    def create_lstm(input_shape):
        inputs = tf.keras.Input(shape = input_shape)
        x = tf.keras.layers.Masking(mask_value=0.)(inputs)
        x1 = tf.keras.layers.LSTM(20, activation='relu',return_sequences = True,name='lstm')(x)
        x1 = tf.keras.layers.BatchNormalization()(x1)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(100, activation='relu'))(x1)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(20, activation='relu'))(x6)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(20, activation='relu'))(x6)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        y_ = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(1, activation='sigmoid'))(x6)
        return tf.keras.Model(inputs=inputs,outputs=y_)

    model_lstm = create_lstm(recurr_xtrain.shape[1:])

    loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
    model_lstm.compile(loss=loss,metrics=[tf.keras.metrics.Precision(),tf.keras.metrics.Recall()],optimizer='adam')
    model_lstm.summary()
    model_lstm.fit(recurr_xtrain, recurr_ytrain,
              validation_split=0.15,
              shuffle = True,
              epochs=50,
              verbose=1)

    # Bidirectional LSTM
    def create_bilstm(input_shape):
        inputs = tf.keras.Input(shape = input_shape)
        x = tf.keras.layers.Masking(mask_value=0.)(inputs)
        x1 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(20, activation='relu',return_sequences = True,name='lstm'))(x)
        x1 = tf.keras.layers.BatchNormalization()(x1)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(100, activation='relu'))(x1)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(20, activation='relu'))(x6)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(20, activation='relu'))(x6)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        y_ = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(1, activation='sigmoid'))(x6)
        return tf.keras.Model(inputs=inputs,outputs=y_)

    model_bilstm = create_bilstm(recurr_xtrain.shape[1:])

    loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
    model_bilstm.compile(loss=loss,metrics=[tf.keras.metrics.Precision(),tf.keras.metrics.Recall()],optimizer='adam')
    model_bilstm.summary()
    model_bilstm.fit(recurr_xtrain, recurr_ytrain,
              validation_split=0.15,
              shuffle = True,
              epochs=50,
              verbose=1)

if evaluation:

    dataset = 'test' #  test or entire

    if dataset == 'entire':
        dataset = n
        data = XX
        recurr_data = sequential_x
    elif dataset == 'test':
        dataset = test
        data = x_test
        recurr_data = recurr_xtest

    Force = [raw_F[i] for i in dataset]
    Fo = Fos[np.where(Fos!=0)[0]]
    Force_open = np.array([Fo[i] for i in dataset])

    y_pred_dense = model_dense.predict(data)
    Err_dense = evaluate_dense(y_pred_dense,np.array(Yseq)[dataset])
    #FoPre, FoErr, count =  FoErrEvalDense(y_pred_dense,Force,Force_open)
    print('Error (Dense) <= 2 steps: %.4f'%(100*len(np.where(abs(Err_dense)<=2)[0])/len(Err_dense)))

    y_pred_lstm = model_lstm.predict(sequential_x[dataset])
    Err_lstm, lstm_err = evaluate_plot(y_pred_lstm,to=idxo_masked[dataset])
    #FoPre, FoErr, count =  FoErrEval(y_pred_lstm,Force,Force_open)
    print('Error (LSTM) <= 2  steps: %.4f'%(100*np.sum(lstm_err[np.where(Err_lstm<=2)[0]])/np.sum(lstm_err)))

    y_pred_bilstm = model_bilstm.predict(sequential_x[dataset])
    Err_bilstm, bilstm_err = evaluate_plot(y_pred_bilstm,to=idxo_masked[dataset])
    #FoPre, FoErr, count =  FoErrEval(y_pred_bilstm,Force,Force_open)
    print('Error (BiLSTM) <= 2  steps: %.4f '%(100*np.sum(bilstm_err[np.where(Err_bilstm<=2)[0]])/np.sum(bilstm_err)))


from django.shortcuts import render, redirect
from django.http import HttpResponse
import requests

def get_api(request):
    rdata = requests.get('https://covid19.th-stat.com/api/open/timeline')
    data = rdata.json()
    total = [num for num in range(len(data['Data']))]
    num = str(len(total)-1)
    if request.POST:
        num = request.POST.get('date')
        print(num)

    context = {'datas': zip(reversed(total),
        reversed(data['Data'])), 'current': data['Data'][int(num)]}

    return render(request, 'timeline/timeline.html', context)


# !/usr/bin/env python
# -*- coding:utf-8 -*-
# Author:pylarva
# bolg:www.lichengbing.com

from __future__ import division

import os
import re
import sys
import time
import json
import math
import socket
import hashlib

# 客户端程序运行的本地目录
base_dir = os.path.dirname(os.path.abspath(__file__))

def connect_server():
    """
    客户端连接服务端：IP + PORT
    :return:
    """
    while True:
        ip_port = input('请输入服务端地址： ')
        if not re.match('^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}:\d+$', ip_port):
            print('地址有误...')
            continue
        else:
            ip_port = (ip_port.split(':')[0], int(ip_port.split(':')[1]))
            return ip_port

def login(s):
    """
    FTP登陆 如果登陆不成功即不允许与服务端会话
    :param s:
    :return:
    """
    while True:
        send_data = input('>>: ').strip()
        if len(send_data) == 0: continue
        s.send(bytes(send_data, encoding='utf-8'))
        recv_data = s.recv(1024)
        print(recv_data.decode())

        if re.match('欢迎', recv_data.decode()):
            break

def progressbar(cur, total):
    """
    上传文件或者下载文件过程中的进度条显示
    :param cur:
    :param total:
    :return:
    """
    percent = '{:.2%}'.format(cur / total)
    sys.stdout.write('\r')
    sys.stdout.write('[%-50s] %s' % ('=' * int(math.floor(cur * 50 / total)), percent))
    sys.stdout.flush()
    if cur == total:
        sys.stdout.write('\n')

def file_md5(file_path):
    """
    文件的MD5校验
    :param file_path:
    :return:
    """
    f = open(file_path, 'rb')
    md5obj = hashlib.md5()
    md5obj.update(f.read())
    md5_hash = md5obj.hexdigest()
    f.close()
    return str(md5_hash).upper()

def task_put(s, cmd_list):
    """
    执行上传文件put方法
    :param s:
    :param cmd_list:
    :return:
    """
    abs_filepath = cmd_list[1]
    if os.path.isfile(abs_filepath):
        file_size = os.stat(abs_filepath).st_size
        file_name = abs_filepath.split('\\')[-1]
        print('file:{} size:{}'.format(file_name, file_size))

        # 给服务端发送一个包含文件名 文件大小等信息的json文件
        msg_data = {"action": "put",
                    "file_name": file_name,
                    "file_size": file_size,
                    "file_path": abs_filepath}

        s.sendall(bytes(json.dumps(msg_data), encoding='utf-8'))

        # 接收服务端的确认信息
        server_confirm_msg = s.recv(1024)
        confirm_data = json.loads(server_confirm_msg.decode())

        if confirm_data['status'] == 200:

            # 如果服务端已有文件 得到文件大小 进行断点续传
            read_size = confirm_data['read_size']
            if read_size != 0:
                send_size = read_size
                print('服务器上检测到相同文件 已经启动断点续传...')
                time.sleep(0.5)
            else:
                send_size = 0
            print('Start sending file \033[31;0m{}\033[0m!'.format(msg_data['file_name']))

            # 打开文件 进行指针偏移 开始发送数据
            f = open(msg_data['file_path'], 'rb+')
            f.seek(read_size, 0)
            for line in f:
                s.send(line)
                send_size += len(line)
                progressbar(send_size, file_size)
            f.close()
            print('上传文件 \033[31;0m{}\033[0m 成功!'.format(file_name))

            # 上传成功后进行MD5校验
            print('正在校验 MD5 值...')
            md5_hash = file_md5(abs_filepath)
            ret = s.recv(1024)
            if md5_hash == ret.decode():
                print('MD5值 \033[31;0m{}\033[0m 校验成功!'.format(md5_hash))
            else:
                print('MD5值 \033[31;0m{}\033[0m 校验失败!'.format(md5_hash))

    else:
        print('\033[31;0m{}\033[0m文件不存在...'.format(abs_filepath))

def task_pull(s, cmd_list):
    """
    下载文件pull方法
    :param s:
    :param cmd_list:
    :return:
    """
    file_name = cmd_list[-1]

    file = os.path.join(base_dir, file_name)

    # 判断本地路径下是否有相同文件 得到该文件大小 方便断点续传
    if os.path.exists(file):
        recv_size = os.stat(file).st_size
        print('本地检测到相同文件 已经启动断点续传...')
    else:
        recv_size = 0

    # 发送请求给服务端
    msg_data = {"action": "pull", "file_name": file_name}
    s.send(bytes(json.dumps(msg_data), encoding='utf-8'))

    # 服务端收到下载请求 等待确认开始传输
    data = s.recv(1024)
    data = json.loads(data.decode())
    file_name = data.get('file_name')
    file_size = data.get('file_size')
    server_response = {'status': 200, "read_size": recv_size}
    s.send(bytes(json.dumps(server_response), encoding='utf-8'))
    f = open(os.path.join(base_dir, file_name), 'ab+')

    # 收到总的文件大小 开始循环接收
    while recv_size < file_size:
        data = s.recv(4096)
        f.write(data)
        recv_size += len(data)
        progressbar(recv_size, file_size)
    print('下载文件 \033[31;0m{}\033[0m 成功!\n本地路径：\033[31;0m{}\033[0m'.format(file_name, file))
    f.close()

    # 接受完成MD5校验
    print('正在校验 MD5 值...')
    md5_hash = file_md5(file)
    ret = s.recv(1024)
    if md5_hash == ret.decode():
        print('MD5值 \033[31;0m{}\033[0m 校验成功!'.format(md5_hash))
    else:
        print('MD5值 \033[31;0m{}\033[0m 校验失败!'.format(md5_hash))

def task_types(s, cmd_list):
    """
    对客户端命令进行有效性判断
    :param s:
    :param cmd_list:
    :return:
    """
    if len(cmd_list) == 1:
        if cmd_list[0] == 'mkdir':
            print('ERROR: mkdir + 文件夹名...')
        if cmd_list[0] == 'rm':
            print('ERROR: rm + 文件夹名...')
        else:
            msg_data = {"action":  cmd_list[0],
                        "file_name": None}
            msg_send(s, msg_data)

    if len(cmd_list) == 2:
        task_type = cmd_list[0]
        if task_type == 'put':
            task_put(s, cmd_list)
        if task_type == 'pull':
            task_pull(s, cmd_list)
        if task_type in ['cd', 'ls', 'mkdir', 'rm']:
            msg_data = {"action": cmd_list[0], "file_name": cmd_list[-1]}
            msg_send(s, msg_data)

def msg_send(s, msg_data):
    """
    向服务端发送一条消息并接收一条消息并打印
    :param s:
    :param msg_data:
    :return:
    """
    s.send(bytes(json.dumps(msg_data), encoding='utf-8'))
    recv_data = s.recv(1024)
    print(recv_data.decode())

def main():
    """
    客户端主程序函数：线路连接 + 循环会话
    :return:
    """
    print('  \033[32;0mFTP客户端程序\033[0m  '.center(50, '-'))

    while True:
        # 连接服务端
        ip_port = connect_server()
        # ip_port = ('10.0.0.150', 8000)
        s = socket.socket()
        s.settimeout(2)
        ret = s.connect_ex(ip_port)
        if ret != 0:
            print('服务器：\033[31;0m{}\033[0m 端口：\033[31;0m{}\033[0m 连接失败...'.format(ip_port[0], ip_port[1]))
            print('请检查IP和端口并重试!\n')
            continue

        welcome_msg = s.recv(1024)
        print(welcome_msg.decode())
        login(s)

        # 开始循环会话
        while True:
            task_list = ['ls', 'put', 'pull', 'mkdir', 'rm', 'cd']
            send_data = input('>>: ').strip()
            if len(send_data) == 0: continue

            cmd_list = send_data.split()
            if cmd_list[0].upper() == 'HELP':
                print('''\033[32;0m
                --------- Help 帮助信息 ----------
                1. ls    浏览目录
                2. cd    切换目录
                3. rm    删除文件
                4. mkdir 新建文件夹
                5. put   上传文件
                6. pull  下载文件
                \033[0m''')
                continue
            if cmd_list[0] not in task_list:
                print('不支持的命令！[Help]查看帮助...')
                continue

            task_types(s, cmd_list)

if __name__ == '__main__':
    main()


import sys
import face_recognition as fr
import numpy as np

storedPath = open(sys.argv[1], 'r')
uploadedPath = open(sys.argv[2], 'r')

storedString = storedPath.read().replace('[', '').replace(']', '').replace('\\n', '').replace('\n', '').replace('\"', '').replace(',', '')
uploadedString = uploadedPath.read().replace('[', '').replace(']', '').replace('\n', '')

storedImg = np.fromstring(storedString, dtype=np.float64, sep=' ')
uploadedImg = np.fromstring(uploadedString, dtype=np.float64, sep=' ')

enset = [storedImg, uploadedImg]
# ff = open('compresult.txt', 'w')
dist = fr.face_distance(enset, uploadedImg)[0]

if (0.4 > dist):
    print('True\n' + str(float(dist)))
else:
    print('False\n' + str(float(dist)))


import numpy as np

def edit_distance(s1, s2):
    m = len(s1) + 1 # source string
    n = len(s2) + 1 # target string
    print('m: {}, n: {}'.format(m, n))
    distance = [[0] * m for _ in range(n)] # distance[i][j]: i from 0 to n-1, j from 0 to m-1
#    t = np.array(distance)
#    print('distance shape: {}'.format(t.shape))

    for i in range(n):
        distance[i][0] = i

    for j in range(m):
        distance[0][j] = j

    for i, c1 in enumerate(s2, start=1):
        for j, c2 in enumerate(s1, start=1):
#            print('i: {}, j: {}, c1: {}, c2: {}'.format(i, j, c1, c2))
            if c1 == c2:
                distance[i][j] = distance[i-1][j-1]
            else:
                distance[i][j] = min(distance[i-1][j] + 1, distance[i][j-1] + 1, distance[i-1][j-1] + 1)

    return distance[n-1][m-1]

def improved_(s1, s2):
    """
    two arraies: distance distance_, each length is len(shorter string)
    distance: previous row
    distance_: current row
    """

    if len(s1) > len(s2):
        s1, s2 = s2, s1

    distance = range(len(s1) + 1)

    # i2: row, i1: col
    for i2, c2 in enumerate(s2, start=1):
        distance_ = [i2]
        for i1, c1 in enumerate(s1, start=1):
            if c1 == c2:
                distance_.append(distance[i1-1])
            else:
                distance_.append(1 + min(distance[i1-1], distance[i1], distance_[-1]))
        distance = distance_

    return distance[-1]

s1 = 'a'
s2 = 'abcdef'
print(edit_distance(s1, s2))
print(improved_(s1, s2))


import cv2
import numpy as np
import time

# IMG_PATH = 'images/cvtest.jpg'
VID_PATH = 'images/album_cover.mp4'

W = 1920
H = 1080

def process_frame(img):
    # gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    blur = cv2.GaussianBlur(img, (11, 11), 0)
    res = cv2.Canny(blur, 11, 11)
    # res = cv2.resize(canny, None, fx=0.125, fy=0.125, interpolation=cv2.INTER_CUBIC)
    # res = cv2.resize(img, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_CUBIC)
    # gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    # corners = cv2.goodFeaturesToTrack(gray, 25, 1, 10)
    # corners = np.int32(corners)
    # for i in corners:
    #     x,y = i.ravel()
    #     res = cv2.circle(img, (x,y), 10, 255, -1)
    cv2.imshow('img', res)
    # print(res.shape)

if __name__ == "__main__":
    # img = cv2.imread('images/cvtest.jpg')
    cap = cv2.VideoCapture(VID_PATH)
    frame_count = cap.get(cv2.CAP_PROP_FRAME_COUNT)
    cap.set(cv2.CAP_PROP_POS_FRAMES, 0)

    while cap.isOpened():
        ret, frame = cap.read()
        frame_number = cap.get(cv2.CAP_PROP_POS_FRAMES)
        print(frame_number)
        if ret == True:
            process_frame(frame)
            cv2.waitKey(0)
        else:
            cap.release()
            cv2.destroyAllWindows()


from sys import stdin, stdout

def longest_common_subsequence_solve(X, Y, len_x, len_y):

    mat = [[int(0) for y in range(len_x+1)] for x in range(len_y+1)]

    for i in range(1, len_y+1):
        for j in range(1, len_x+1):
            if Y[i-1] == X[j-1]:
                mat[i][j] = 1 + mat[i-1][j-1]
            else:
                mat[i][j] = max(mat[i-1][j], mat[i][j-1])
    print(mat)
    print(mat[len_y][len_x])

if __name__ == '__main__':
    X = "AGGTAB"
    Y = "GXTXAYB"
    len_x = len(X)
    len_y = len(Y)
    longest_common_subsequence_solve(X, Y, len_x, len_y)


'''
Created on Mar 7, 2015

@author: Adrian
'''
def getJson(url):
    import requests, json
    r = requests.get(url)
    obj = json.loads(r.content)
    results = obj['json']
    return results
def sendPost(url, obj):
    import requests
    import urllib
    payload = urllib.urlencode(obj,doseq=True)
    headers = {'Content-Type':'application/x-www-form-urlencoded'}
    r = requests.post(url, data=payload, headers=headers)
    return r

def getRemovedHostNames():
    results = getJson('http://localhost:3000/bad_links')

    removed_domains = []

    for result in results:
        removed_domains.append(result['DOMAIN_NAME'])

    return removed_domains


from django.http import HttpResponse
from django.shortcuts import render, redirect, get_object_or_404
from django.views.decorators.http import require_POST
from coupons.forms import CouponApplyForm
from Books.models import Book
from .cart import Cart
from .forms import CartAddBookForm

@require_POST
def cart_add(request, book_id):
    """
    add books to basket
    :param request:
    :param book_id:
    :return:
    """
    cart = Cart(request)
    book = get_object_or_404(Book, id=book_id)
    form = CartAddBookForm(request.POST, id)
    if book.inventory > 0:
        if form.is_valid():
            cd = form.cleaned_data
            if cd['quantity'] < book.inventory:

                cart.add(book=book,
                         quantity=cd['quantity'],
                         update_quantity=cd['update'])
            else:
                return HttpResponse("کتاب مورد نظر موجود نیست")
        return redirect('cart_detail')

    else:
        return HttpResponse("کتاب مورد نظر موجود نیست")

def cart_remove(request, book_id):
    """
        remove books from basket
        :param request:
        :param book_id:
        :return:
    """
    cart = Cart(request)
    book = get_object_or_404(Book, id=book_id)
    cart.remove(book)
    return redirect('cart_detail')

def cart_detail(request):
    """
        final basket view
        :param request:
        :param book_id:
        :return:
        """
    cart = Cart(request)
    for item in cart:
        item['update_count_form'] = CartAddBookForm(initial={'quantity': item['quantity'],
                                                             'update': True})
    discount_apply_form = CouponApplyForm()
    return render(request, 'cart/cart_detail.html', {'cart': cart, 'discount_apply_form': discount_apply_form})


def checkSameLetter(characters):
    for i, c in enumerate(characters):
        if i < len(characters) - 2:
            if c == characters[i+2]:
                return True
    return False

def checkPairOfLetters(characters):
    for i, c in enumerate(characters):
        if i < len(characters) - 1:
            pair = str(c+characters[i+1])
            if pair in characters[i+2:]:
                return True
    return False

with open('5/input.txt', 'r') as file:
    nice = 0
    for line in file:
        if checkSameLetter(line) and checkPairOfLetters(line):
            nice += 1
    print(nice)
file.close()


# -*- coding: utf-8 -*-
import os.path
import io
from math import *
from random import shuffle
import numpy as np
import sys
import time
from collections import defaultdict

reload(sys)
sys.setdefaultencoding('utf8')

idx_unk = 0
str_unk = "<unk>"

idx_pad = 1
str_pad = "<pad>"

class Embeddings():

    def __init__(self, file, voc):
        w2e = {}
        if file is not None:
            with io.open(file, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
                self.num, self.dim = map(int, f.readline().split())
                i = 0
                for line in f:
                    i += 1
                    if i%10000 == 0:
                        if i%100000 == 0: sys.stderr.write("{}".format(i))
                        else: sys.stderr.write(".")
                    tokens = line.rstrip().split(' ')
                    if voc.exists(tokens[0]): w2e[tokens[0]] = tokens[1:]
            sys.stderr.write('Read {} embeddings ({} missing in voc)\n'.format(len(w2e),len(voc)-len(w2e)))
        else:
            sys.stderr.write('Embeddings not used!\n')

        # i need an embedding for each word in voc
        # embedding matrix must have tokens in same order than voc 0:<unk>, 1:<pad>, 2:le, ...
        self.matrix = []
        for tok in voc:
            if tok == str_unk or tok == str_pad or not tok in w2e: ### random initialize these tokens
                self.matrix.append(np.random.normal(0, 1.0, self.dim))
            else:
                self.matrix.append(np.asarray(w2e[tok], dtype=np.float32))

        self.matrix = np.asarray(self.matrix, dtype=np.float32)
        self.matrix = self.matrix / np.sqrt((self.matrix ** 2).sum(1))[:, None]

class Vocab():

    def __init__(self, dict_file):
        curr_time = time.strftime("[%Y-%m-%d_%X]", time.localtime())
        self.tok_to_idx = {}
        self.idx_to_tok = []
        self.idx_to_tok.append(str_unk)
        self.tok_to_idx[str_unk] = len(self.tok_to_idx) #0
        self.idx_to_tok.append(str_pad)
        self.tok_to_idx[str_pad] = len(self.tok_to_idx) #1
        with io.open(dict_file, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
            nline = 0
            for line in f:
                nline += 1
                line = line.strip()
                self.idx_to_tok.append(line)
                self.tok_to_idx[line] = len(self.tok_to_idx)

        self.length = len(self.idx_to_tok)
        sys.stderr.write('Read vocab ({} entries)\n'.format(self.length))

    def __len__(self):
        return len(self.idx_to_tok)

    def exists(self, s):
        return s in self.tok_to_idx

    def __iter__(self):
        for tok in self.idx_to_tok:
            yield tok

    def get(self,s):
        if type(s) == int: ### I want the string
            if s < len(self.idx_to_tok): return self.idx_to_tok[s]
            else:
                sys.stderr.write('error: key \'{}\' not found in vocab\n'.format(s))
                sys.exit()
        ### I want the index
        if s not in self.tok_to_idx: return idx_unk
        return self.tok_to_idx[s]

class Replace():

    def __init__(self, file):
        curr_time = time.strftime("[%Y-%m-%d_%X]", time.localtime())
        self.max_length = -1
        self.min_length = -1
        self.pos_to_wrd = defaultdict(list)
        with io.open(file, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
            nline = 0
            for line in f:
                nline += 1
                if nline==1:
                    tok = line.split(",")
                    self.min_length = int(tok[0])
                    self.max_length = int(tok[1])
                    continue
                seqpos, seqwrd = line.strip().split('\t')
                self.pos_to_wrd[seqpos].append(seqwrd)

        sys.stderr.write('Read replace ({} pos entries)\n'.format(len(self.pos_to_wrd)))

    def get(self,seq,pos):
        seqpos = " ".join(pos) #string
        if "-" in seqpos: return []
        seqwrd = " ".join(seq) #string

#        print("\tseqpos: {}".format(seqpos))
        if not seqpos in self.pos_to_wrd: return []
        seqwrds = self.pos_to_wrd[seqpos] #is a list of strings

        indexs = [i for i in range(min(100,len(seqwrds)))]
        shuffle(indexs)
        for i in indexs:
            seqwrd2=seqwrds[i] #string
#            print("\t\ti={} {}".format(i,seqwrd2))
            seq2=seqwrd2.split(' ') #vector
            totally_different = True
            for k in range(len(seq2)):
                if seq2[k] == seq[k]:
#                    print("not totally different")
                    totally_different = False
                    break
            if totally_different: return seq2

        return []

class Dataset():

    def __init__(self, file, voc_src, voc_tgt, replace_src, seq_size, max_sents, do_shuffle, mode):
        if file is None:
            return
        self.voc_src = voc_src
        self.voc_tgt = voc_tgt
        self.file = file
        self.seq_size = seq_size
        self.max_sents = max_sents
        self.do_shuffle = do_shuffle
        self.mode = mode
        self.data = []
        self.replace_src = replace_src
        self.length = 0 ### length of the data set to be used (not necessarily the whole set)
        self.annotated = False
        with io.open(self.file, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
            first = True
            for line in f:
                if first:
                    tokens = line.strip().split('\t')
                    if len(tokens)==3 and (tokens[0]=="P" or tokens[0]=="U" or tokens[0]=="I" or tokens[0]=="R"):
                        self.annotated = True
                    first = False
                self.data.append(line)
                self.length += 1
        if self.max_sents > 0:
            self.length = min(self.length,self.max_sents)
        sys.stderr.write('({} contains {} examples)\n'.format(self.file,len(self.data)))

    def __iter__(self):
        nsent = 0
        self.nsrc = 0
        self.ntgt = 0
        self.nunk_src = 0
        self.nunk_tgt = 0
        self.ndiv_src = 0
        self.ndiv_tgt = 0
        ### every iteration i get shuffled data examples if do_shuffle
        do_delete = True
        do_source = True
        indexs = [i for i in range(len(self.data))]
        if self.do_shuffle: shuffle(indexs)
        for index in indexs:
            tokens = self.data[index].strip().split('\t')
            if len(tokens) < 2 or len(tokens) > 4 or len(tokens[0])==0 or len(tokens[1])==0:
                sys.stderr.write("error: bad data entry \'{}\' in line={} [skipped]\n".format(self.data[index],index+1))
                continue

            if len(tokens)==3 and (tokens[0]=="P" or tokens[0]=="U" or tokens[0]=="I" or tokens[0]=="R"):
                self.annotated = True ### should not be done at each line... :-(
                src = tokens[1].split(' ')
                tgt = tokens[2].split(' ')
                yield self.tagged_sentence(src,tgt)
                nsent += 1
                continue

            src = tokens[0].split(' ')
            tgt = tokens[1].split(' ')

            if self.seq_size > 0 and (len(src) > self.seq_size or len(tgt) > self.seq_size): continue # filter out examples with more than seq_size tokens
            if self.max_sents > 0 and nsent > self.max_sents: break # already generated max_sents examples

            if 'p' in self.mode:
                yield self.parallel_sentence(src, tgt)
                nsent += 1

            if len(self.data)>1:
                if 'u' in self.mode:
                    yield self.uneven_sentence(src, tgt, index)
                    nsent += 1

                if 'i' in self.mode:
                    yield self.insert_sentence(src, tgt, index)
                    nsent += 1

            if 'd' in self.mode and len(tokens)==3:
                yield self.delete_sentence(src, tgt, tokens[2])
                nsent += 1

            if 'r' in self.mode and self.replace_src is not None and len(tokens)==4:
                yield self.replace_sentence(src, tgt, tokens[2], tokens[3])
                nsent += 1

    def __len__(self):
        return self.length

    def debug_sentence(self,tag,orig_src,src,isrc,src_tag,orig_tgt,tgt,itgt,tgt_tag):
        print(tag)
        print("\torig_src: {}".format(" ".join([s for s in orig_src])))
        print("\tsrc     : {}".format(" ".join([s for s in src])))
        print("\tisrc    : {}".format(" ".join([str(s) for s in isrc])))
        print("\tsrc_tag : {}".format(" ".join([str(s) for s in src_tag])))
        print("\torig_tgt: {}".format(" ".join([t for t in orig_tgt])))
        print("\ttgt     : {}".format(" ".join([t for t in tgt])))
        print("\titgt    : {}".format(" ".join([str(t) for t in itgt])))
        print("\ttgt_tag : {}".format(" ".join([str(t) for t in tgt_tag])))
#        print("   {}\nKK:{}\t{}\t{}\t{}".format(" ".join([s for s in orig_src])," ".join([s for s in src])," ".join([t for t in orig_tgt])," ".join([str(s) for s in src_tag])," ".join([str(t) for t in tgt_tag])))

    def tagged_sentence(self,src,tgt):
        isrc = []
        src_tag =[]
        for s in range(len(src)):
            if src[s][0]=="@":
                src_tag.append(1.0)
                src[s]=src[s][1:]
            else:
                src_tag.append(-1.0)
            isrc.append(self.voc_src.get(src[s]))

        itgt = []
        tgt_tag =[]
        for t in range(len(tgt)):
            if tgt[t][0]=="@":
                tgt_tag.append(1.0)
                tgt[t]=tgt[t][1:]
            else:
                tgt_tag.append(-1.0)
            itgt.append(self.voc_tgt.get(tgt[t]))
        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("TAGGED",src,src,isrc,src_tag,tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def original_words(self,src,tgt):
        isrc = []
        src_tag = []
        for s in src:
            isrc.append(self.voc_src.get(s))
            src_tag.append(-1.0) #positive example (non divergent)
        itgt = []
        tgt_tag = []
        for t in tgt:
            itgt.append(self.voc_tgt.get(t))
            tgt_tag.append(-1.0) #positive example (non divergent)

        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def keep_records(self, src_tag, tgt_tag, isrc, itgt):
        self.ndiv_src += sum(1 for s in src_tag if s==1.0)
        self.ndiv_tgt += sum(1 for t in tgt_tag if t==1.0)
        self.nunk_src += sum(1 for s in isrc if s==idx_unk)
        self.nunk_tgt += sum(1 for t in itgt if t==idx_unk)
        self.nsrc += len(src_tag)
        self.ntgt += len(tgt_tag)

    def parallel_sentence(self,src,tgt):
        orig_src = src
        orig_tgt = tgt
        isrc, itgt, src, tgt, src_tag, tgt_tag = self.original_words(src,tgt)
        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("PARALLEL",orig_src,src,isrc,src_tag,orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def uneven_sentence(self,src,tgt,ind):
        orig_src = src
        orig_tgt = tgt
        if ind%2==0: #replace tgt sentence
            correct_ratio = False
            while not correct_ratio:
                index = np.random.randint(0,len(self.data))
                while ind==index:
                    index = np.random.randint(0,len(self.data))
                tgt = self.data[index].strip().split('\t')[1].split(' ')
                ratio = len(tgt)/float(len(src))
                correct_ratio = (ratio>=0.5 and ratio<=2.0)
        else: #replace src sentence
            correct_ratio = False
            while not correct_ratio:
                index = np.random.randint(0,len(self.data))
                while ind==index:
                    index = np.random.randint(0,len(self.data))
                src = self.data[index].strip().split('\t')[0].split(' ')
                ratio = len(tgt)/float(len(src))
                correct_ratio = (ratio>=0.5 and ratio<=2.0)
        isrc, itgt, src, tgt, src_tag, tgt_tag = self.original_words(src,tgt)
        for i in range(len(src_tag)): src_tag[i] = 1.0 ### converting positive (-1.0) into negative examples (+1.0)
        for i in range(len(tgt_tag)): tgt_tag[i] = 1.0 ### converting positive (-1.0) into negative examples (+1.0)
        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("UNEVEN",orig_src,src,isrc,src_tag,orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def insert_sentence(self,src,tgt,ind):
        orig_src = src
        orig_tgt = tgt
        ### base sentence
        isrc, itgt, src, tgt, src_tag, tgt_tag = self.original_words(src,tgt)
        where_src = -1
        where_tgt = -1
        ###
        ### add words in source side (src,isrc), mark these source words as divergent (src_tag)
        ###
        if len(src)<=len(tgt):
            ### choose randomly the added src sentence in data
            i = np.random.randint(0,len(self.data))
            while ind==i:
                i = np.random.randint(0,len(self.data))
            add_source = self.data[i].strip().split('\t')[0].split(' ')
            #i will consider only the initial/final nsrc words of the new add_source sentence
            nsrc = np.random.randint(1,len(tgt)+1)
            if nsrc>len(add_source): nsrc=len(add_source)
            if ind%2==0: #add in the begining (take the last nsrc words)
                add_src = add_source[len(add_source)-nsrc:]
                j=0
            else: #add in the end (take the initial nsrc words)
                add_src = add_source[:nsrc]
                j=len(src)
            where_src=j
            add_src_tag = [1.0] * len(add_src) # divergent
            add_isrc = []
            for s in add_src: add_isrc.append(self.voc_src.get(s))
            prev = src[:j]
            post = src[j:]
            src = prev + add_src + post
            prev = isrc[:j]
            post = isrc[j:]
            isrc = prev + add_isrc + post
            prev = src_tag[:j]
            post = src_tag[j:]
            src_tag = prev + add_src_tag + post
        ###
        ### add words in target side (tgt,itgt), mark these target words as divergent (tgt_tag)
        ###
        else:
            j = np.random.randint(0,len(self.data))
            while ind==j:
                j = np.random.randint(0,len(self.data))
            add_target = self.data[j].strip().split('\t')[1].split(' ')
            #i will consider only the initial/final ntgt words of the new add_target sentence
            ntgt = np.random.randint(1,len(src)+1)
            if ntgt>len(add_target): ntgt=len(add_target)
            if ind%2==0: #add in the begining (take the last ntgt words)
                add_tgt = add_target[len(add_target)-ntgt:]
                i=0
            else: #add in the end (take the initial ntgt words)
                add_tgt = add_target[:ntgt]
                i=len(tgt)
            where_tgt=i
            add_tgt_tag = [1.0] * len(add_tgt) # divergent
            add_itgt = []
            for t in add_tgt: add_itgt.append(self.voc_tgt.get(t))
            ### choose where to include the add_tgt sequence
            prev = tgt[:i]
            post = tgt[i:]
            tgt = prev + add_tgt + post
            prev = itgt[:i]
            post = itgt[i:]
            itgt = prev + add_itgt + post
            prev = tgt_tag[:i]
            post = tgt_tag[i:]
            tgt_tag = prev + add_tgt_tag + post

        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("INSERT where_src={} where_tgt={}".format(where_src,where_tgt),orig_src,src,isrc,src_tag,orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def replace_sentence(self,src,tgt,align,srcpos):
        orig_src = list(src)
        orig_tgt = list(tgt)
        if len(src)<3: return self.parallel_sentence(src,tgt)
        src_from, src_to, tgt_from, tgt_to, replace_by = self.find_replacement(src, tgt, align, srcpos)
        if len(replace_by)==0 or src_from<0 or src_to<src_from or src_to>=len(src) or (src_from==0 and src_to==len(src)-1): return self.parallel_sentence(src,tgt)

        isrc, itgt, src, tgt, src_tag, tgt_tag = self.original_words(src,tgt)
        ###
        ### replace words in source side
        ###
        for s in range(src_from,src_to+1):
            src[s] = replace_by[s-src_from]
            isrc[s] = self.voc_src.get(src[s])
            src_tag[s] = 1.0
        ###
        ### keep words in target side except those aligned with the replaced in the source side
        ###
        for t in range(tgt_from,tgt_to+1):
            tgt_tag[t] = 1.0

        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("REPLACE[{},{}] ({}=>{}) [{},{}]".format(src_from,src_to," ".join(orig_src[src_from:src_to+1])," ".join(src[src_from:src_to+1]),tgt_from,tgt_to), orig_src,src,isrc,src_tag, orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def find_replacement(self,src,tgt,align,srcpos):
        if len(src) < self.replace_src.min_length*2:
            return -1, -1, -1, -1, []
        ali = align.split(" ")
        if len(ali)==0: ### there must be at leat one alignment
            return -1, -1, -1, -1, []
        pos = srcpos.split(" ")
        if len(pos)!=len(src):
            return -1, -1, -1, -1, []

        #build alignment structures for fast access
        s2t_minmax = [[None, None] for i in range(len(src))]
        t2s_minmax = [[None, None] for i in range(len(tgt))]
        for a in ali:
            s, t = map(int, a.split('-'))
            if s2t_minmax[s][0] is None:
                s2t_minmax[s][0] = t
                s2t_minmax[s][1] = t
            else:
                if t < s2t_minmax[s][0]: s2t_minmax[s][0] = t
                if t > s2t_minmax[s][1]: s2t_minmax[s][1] = t
            if t2s_minmax[t][0] is None:
                t2s_minmax[t][0] = s
                t2s_minmax[t][1] = s
            else:
                if s < t2s_minmax[t][0]: t2s_minmax[t][0] = s
                if s > t2s_minmax[t][1]: t2s_minmax[t][1] = s

        #find the largest replacement that does not exceeds len(src)/2
        replace_by = []
        src_from = -1
        src_to = -1
        tgt_from = -1
        tgt_to = -1
        indexs = [i for i in range(len(src))]
        shuffle(indexs)
        for src_from in indexs:
            #for src_to in range(src_from+self.replace_src.min_length-1,src_from+self.replace_src.max_length):
            for src_to in range(src_from+self.replace_src.max_length-1,src_from+self.replace_src.min_length-2,-1):
                if src_to>=len(src): continue
                if src_to-src_from+1 > len(src)//2: continue
                tgt_from, tgt_to, replace_by = self.replacement(src,tgt,src_from,src_to,pos,s2t_minmax,t2s_minmax)
                if len(replace_by)>0:
                    return src_from, src_to, tgt_from, tgt_to, replace_by
        return src_from, src_to, tgt_from, tgt_to, replace_by

    def replacement(self,src,tgt,src_from,src_to,pos,s2t_minmax,t2s_minmax):
        src_seq = src[src_from:src_to+1]
        pos_seq = pos[src_from:src_to+1]
        src_rep = self.replace_src.get(src_seq,pos_seq)
        if len(src_rep)!=len(src_seq): return -1, -1, []

        tgt_from = 9999
        tgt_to = -1
        ### first i collect the [tgt_from,tgt_to] words aligned to [src_from, src_to]
        for s in range(src_from, src_to+1):
            tgt_min = s2t_minmax[s][0]
            tgt_max = s2t_minmax[s][1]
            if tgt_min is not None and tgt_min < tgt_from: tgt_from = tgt_min
            if tgt_max is not None and tgt_max > tgt_to:   tgt_to   = tgt_max
        if tgt_from == 9999 or tgt_to == -1: return -1, -1, []

        ### second i make sure that any target word in [tgt_from, tgt_to] is not aligned outside [src_from,src_to]
        for t in range(tgt_from, tgt_to+1):
            src_min = t2s_minmax[t][0]
            src_max = t2s_minmax[t][1]
            if src_min is not None and src_min < src_from: return -1, -1, []
            if src_max is not None and src_max > src_to:   return -1, -1, []

        return tgt_from, tgt_to, src_rep

    def delete_sentence(self,src,tgt,ali):
        orig_src = src
        orig_tgt = tgt
        src_from, src_to, tgt_from, tgt_to = self.find_self_aligned_sequence(src, tgt, ali)
        if src_from<0 or src_to<src_from or src_to>=len(src) or (src_from==0 and src_to==len(src)-1): return self.parallel_sentence(src, tgt)
        ### delete on the side with more words
        ### remove words in source side, mark divergence in target side
        ###
        if len(orig_src)>=len(orig_tgt):
            new_src = []
            isrc = []
            src_tag = []
            for s in range(len(src)):
                if s < src_from or s > src_to: ### keep the word
                    new_src.append(src[s])
                    isrc.append(self.voc_src.get(src[s]))
                    src_tag.append(-1.0) # positive example
            src = new_src
            #tgt remains as it was
            itgt = []
            tgt_tag = []
            for t in range(len(tgt)):
                itgt.append(self.voc_tgt.get(tgt[t]))
                if t < tgt_from or t > tgt_to: tgt_tag.append(-1.0) ### positive example (link is aligned)
                else: tgt_tag.append(1.0) ### negative example (link is deleted)
        ###
        ### remove words in target side, mark divergence in source side
        ###
        else:
            new_tgt = []
            itgt = []
            tgt_tag = []
            for t in range(len(tgt)):
                if t < tgt_from or t > tgt_to: ### keep the word
                    new_tgt.append(tgt[t])
                    itgt.append(self.voc_tgt.get(tgt[t]))
                    tgt_tag.append(-1.0) # positive example
            tgt = new_tgt
            #src remains as it was
            isrc = []
            src_tag = []
            for s in range(len(src)):
                isrc.append(self.voc_src.get(src[s]))
                if s < src_from or s > src_to: src_tag.append(-1.0) ### positive example (link is aligned)
                else: src_tag.append(1.0) ### negative example (link is deleted)

        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("DELETE is_source={} [{},{}] [{},{}]".format(len(orig_src)>=len(orig_tgt),src_from,src_to,tgt_from,tgt_to), orig_src,src,isrc,src_tag, orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def find_self_aligned_sequence(self, src, tgt, align):
        ali = align.split(" ")
        if len(ali)==0: ### there must be at leat one alignment
            return -1, -1, -1, -1
        #0 : build alignment structures for fast access
        s2t_minmax = [[None, None] for i in range(len(src))]
        t2s_minmax = [[None, None] for i in range(len(tgt))]
        for a in ali:
            s, t = map(int, a.split('-'))
            if s2t_minmax[s][0] is None:
                s2t_minmax[s][0] = t
                s2t_minmax[s][1] = t
            else:
                if t < s2t_minmax[s][0]: s2t_minmax[s][0] = t
                if t > s2t_minmax[s][1]: s2t_minmax[s][1] = t
            if t2s_minmax[t][0] is None:
                t2s_minmax[t][0] = s
                t2s_minmax[t][1] = s
            else:
                if s < t2s_minmax[t][0]: t2s_minmax[t][0] = s
                if s > t2s_minmax[t][1]: t2s_minmax[t][1] = s
        #1 : find a target sequence [imin, imax] with at least one alignment
        max_size = 9
        while True:
            imin = np.random.randint(0,len(tgt))
            imax = imin + np.random.randint(0,min(max_size,len(tgt)-imin))
            jmin, jmax = self.MinMax(t2s_minmax,imin,imax)
            if jmin is not None and jmax is not None: break
        length_sequences = (imax-imin+1) + (jmax-jmin+1)
        j_or_i = 1
        #2 : find the minimum sequence of source/target words that contains i and are not aligned outside the sequence
        while True:
            if j_or_i == 0: jmin, jmax = self.MinMax(t2s_minmax,imin,imax)
            else:           imin, imax = self.MinMax(s2t_minmax,jmin,jmax)
            if length_sequences == (imax-imin+1) + (jmax-jmin+1): break #no added words stop iterating
            length_sequences = (imax-imin+1) + (jmax-jmin+1)
            j_or_i = (j_or_i + 1)%2
        #extend with unaligned words ???
        #filter bad examples: all words, too many tokens, etc.
        max_ratio = 0.5 #maximum ratio of deleted words
        if jmin==0 and jmax==len(src)-1: return -1, -1, -1, -1
        if imin==0 and imax==len(tgt)-1: return -1, -1, -1, -1
        if ((imax-imin+1) + (jmax-jmin+1))/(len(src)+len(tgt)) > max_ratio: return -1, -1, -1, -1
        return jmin,jmax,imin,imax

    def MinMax(self, a2b_minmax, amin, amax):
        bmin=None
        bmax=None
        for a in range(amin, amax+1):
            if a2b_minmax[a][0] is not None:
                if bmin is None or a2b_minmax[a][0] < bmin:
                    bmin = a2b_minmax[a][0]
            if a2b_minmax[a][1] is not None:
                if bmax is None or a2b_minmax[a][1] > bmax:
                    bmax = a2b_minmax[a][1]
        return bmin, bmax

def minibatches(data, minibatch_size, npad_src, npad_tgt):
    SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG = [], [], [], [], [], []
    max_src, max_tgt = 0, 0
    for (src, tgt, raw_src, raw_tgt, src_tag, tgt_tag) in data:
        if len(SRC) == minibatch_size:
            yield build_batch(SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG, max_src, max_tgt, npad_src, npad_tgt)
            SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG = [], [], [], [], [], []
            max_src, max_tgt = 0, 0
        if len(src) > max_src: max_src = len(src)
        if len(tgt) > max_tgt: max_tgt = len(tgt)
        SRC.append(src)
        TGT.append(tgt)
        RAW_SRC.append(raw_src)
        RAW_TGT.append(raw_tgt)
        SRC_TAG.append(src_tag)
        TGT_TAG.append(tgt_tag)

    if len(SRC) != 0:
        yield build_batch(SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG, max_src, max_tgt, npad_src, npad_tgt)

def build_batch(SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG, max_src, max_tgt, npad_src, npad_tgt):
    src_batch, tgt_batch, raw_src_batch, raw_tgt_batch, sign_src_batch, sign_tgt_batch, sign_batch, len_src_batch, len_tgt_batch = [], [], [], [], [], [], [], [], []
    ### build: src_batch, pad_src_batch sized of max_src
    batch_size = len(SRC)
    for i in range(batch_size):
        # add padding to have max_src/max_tgt words in all examples of current batch
        sign_src = SRC_TAG[i]
        sign_tgt = TGT_TAG[i]
        sign = max(max(sign_src),max(sign_tgt)) ### sign is divergent (+1) if there is any divergent word
        while len(sign_src) < max_src: sign_src.append(-1.0) ### continue filling up to max_tgt cells
        while len(sign_tgt) < max_tgt: sign_tgt.append(-1.0) ### continue filling up to max_tgt cells
        src = list(SRC[i])
        tgt = list(TGT[i])
        while len(src) < max_src: src.append(idx_pad) #<pad>
        while len(tgt) < max_tgt: tgt.append(idx_pad) #<pad>
        raw_src = list(RAW_SRC[i])
        raw_tgt = list(RAW_TGT[i])
        len_src = len(SRC[i])
        len_tgt = len(TGT[i])
        #add padding at the begining/end of the sequences for convolutions (npad = k1+k2/2 + 1)
        for _ in range(npad_src):
            src.append(idx_pad)
            src.insert(0,idx_pad)
        for _ in range(npad_tgt):
            tgt.append(idx_pad)
            tgt.insert(0,idx_pad)
        ### add to batches
        sign_src_batch.append(sign_src)
        sign_tgt_batch.append(sign_tgt)
        sign_batch.append(sign)
        src_batch.append(src)
        tgt_batch.append(tgt)
        raw_src_batch.append(raw_src)
        raw_tgt_batch.append(raw_tgt)
        len_src_batch.append(len_src)
        len_tgt_batch.append(len_tgt)

    return src_batch, tgt_batch, raw_src_batch, raw_tgt_batch, sign_src_batch, sign_tgt_batch, sign_batch, len_src_batch, len_tgt_batch


#Exercise 7 for Coursera Bioinformatic algorithms (partI)

def skew(seq):
	'''takes a seq and returns skew number for each position'''
	ccount=0
	gcount=0
	skew=[0]
	print(skew,end=' ')
	for i in seq:
		if i=='C':
			ccount+=1
		elif i=='G':
			gcount+=1
		skew.append(gcount-ccount)
	return skew

def minskew(seq):
	skewlist=skew(seq)
	minskew=min(skewlist)
	for i in range(len(seq)):
		if skewlist[i]==minskew:
			print (i, end=' ')


#This program opens a text file and reads the data.
#It then displays the data in a line graph
import matplotlib.pyplot as plt
def main():
    #open text file and store data in list
    text_file=open("1994_Weekly_Gas_Averages.txt","r")
    data = [float(line[:-1]) for line in text_file]
    text_file.close()
    #customize the plot to have a line graph of the data
    #line graph will have labeled axis and title
    plt.plot(data)
    plt.title("1994 Weekly Gas Averages")
    plt.xlabel("Intervals")
    plt.ylabel("Gas Averages")
    plt.show()

main()


# Imports
from fipy import Variable, FaceVariable, CellVariable, Grid1D, ExplicitDiffusionTerm, TransientTerm, DiffusionTerm, Viewer, ImplicitSourceTerm
from fipy.tools import numerix

# Setting up a mesh
nx = 50
dx = 0.02
L = nx*dx
mesh = Grid1D(nx=nx, dx=dx)

# Defining the cell variable
c = CellVariable(mesh= mesh, name=r"$c$")

# Initialise c with a funky profile
x = mesh.cellCenters[0]
c.value=0.0
c.setValue((0.3*numerix.sin(2.0*x*numerix.pi)) + 0.5)

# Setting up the no-flux boundary conditions
c.faceGrad.constrain(0.0, where=mesh.facesLeft)
c.faceGrad.constrain(0.0, where=mesh.facesRight)

# Provide value for diffusion and reaction coefficient
D = 1.0
k = -2.0

# Specifying our alpha value. We will only use backwards Euler going forward
alpha = 1

# Defining the equation with the reaction term.
eq = TransientTerm() == DiffusionTerm(coeff=  D)  + ImplicitSourceTerm(coeff=k)

# We can use a much larger dt now since we are using an implicit method.
dt = 0.001

# We might not want to see the output from every single timestep, so we define a stride parameter
time_stride = 10
timestep = 0
run_time = 0.5
t = timestep * dt

# We first initialise the default viewer to get things started:

if __name__ == "__main__":
    viewer = Viewer(vars=(c), datamin=0., datamax=1.)

while t < run_time:
    t += dt
    timestep += 1
    eq.solve(var=c, dt=dt)
    if (timestep % time_stride ==0):
        print ("Beep")
        if __name__ == '__main__':
            viewer.plot()
if __name__ == '__main__':
    input("Press <return> to proceed...")