from django.urls import path, include, re_path
from . import views, view_staff, view_chief, view_admin
from sci_dashboard_app.dash_apps import simpleexsample
from django.conf.urls import url

urlpatterns = [

    path('admin_home/', view_admin.home, name = 'admin_home'),
    path('staff_home/', view_staff.home, name = 'staff_home'),
    path('chief_home/', view_chief.home, name = 'chief_home'),
    path('dashboard/', views.dashboard, name = 'dashboard'),
    # path('dashboard/(?P<start_date>[0-9][\w-]+)/(?P<end_date>[0-9][\w-]+)/$', views.dashboard, name = 'dashboard'),
    url(r'^dashboard/(?P<start_date>[0-9][\w-]+)/(?P<end_date>[0-9][\w-]+)/$', views.dashboard, name = 'dashboard'),
    path('logout/', views.logoutUser, name = 'logout'),
    path('report/by-status/', views.reportbystatus, name = 'reportbystatus'),
    path('report/by-status/start', views.start, name = 'start'),
    path('report/by-status/success', views.success, name = 'success'),
    path('report/by-status/pending', views.pending, name = 'pending'),
    path('report/by-status/received', views.received, name = 'received'),
    path('report/by-location/', views.reportbylocation, name = 'reportbylocation'),
    path('report/by-location/<int:building_id>/', views.location, name = 'location'),
    path('report/by-location/<int:building_id>/type-<int:type_id>', views.locationtype, name = 'locationtype'),
    path('detailofficer/<int:pk>', views.detailofficer, name = 'detailofficer'),
    path('roomcaretaker/', views.roomcaretaker, name = 'roomcaretaker'),
    path('ITstaff/', views.ITstaff, name = 'ITstaff'),
    path('maintenance/', views.maintenance, name = 'maintenance'),
    path('supervisor/', views.supervisor, name = 'supervisor'),
    path('manager-view/', views.managerview, name = 'manager-view'),
    path('detailofficer/edit-profile/<str:pk>', views.editProfile, name = 'edit-profile'),
    path('board-manager-view/', views.boardmanager, name = 'boardmanager'),

]


import face_recognition
import secrets
import random
import faker
import simpy
import json

FOTOS_DOS_CANDIDATOS = [
    "/home/yara/Área de trabalho/ssc_cotas/faces/angela1.jpeg",
    "/home/yara/Área de trabalho/ssc_cotas/faces/tyrell1.jpeg",
    "/home/yara/Área de trabalho/ssc_cotas/faces/price1.jpg",

    "/home/yara/Área de trabalho/ssc_cotas/faces/darlene1.jpg",
    "/home/yara/Área de trabalho/ssc_cotas/faces/darlene2.jpg",
    "/home/yara/Área de trabalho/ssc_cotas/faces/darlene3.jpg",

    "/home/yara/Área de trabalho/ssc_cotas/faces/elliot1.jpg",
    "/home/yara/Área de trabalho/ssc_cotas/faces/elliot2.jpg",
    "/home/yara/Área de trabalho/ssc_cotas/faces/elliot3.jpg"
]
#Utilizei o mesmo banco de imagem da aula, visto que o face_recognition não faz reconhecimento de cor da pele

ARQUIVO_CONFIGURACAO = "/home/yara/Área de trabalho/ssc_cotas/configuracao.json"

TEMPO_CANDIDATO = 150
TEMPO_ESCOLARIDADE = 115
TEMPO_RENDA = 165
TEMPO_COTA = 120

def preparar():
    global configuracao

    configuracao = None
    with open(ARQUIVO_CONFIGURACAO, "r") as arquivo_configuracao:
        configuracao = json.load(arquivo_configuracao)
        if configuracao:
            print("Avaliação de bolsas de estudos iniciada.")

    global candidatos_reconhecidos
    candidatos_reconhecidos = {}

    global candidatos_perfil1
    candidatos_perfil1 = {}

    global candidatos_perfil2
    candidatos_perfil2 = {}

    global gerador_dados_falsos
    gerador_dados_falsos = faker.Faker(locale="pt_BR")

def simular_visita():
    visitante = {
        "foto": random.choice(FOTOS_DOS_CANDIDATOS),
        "ficha": None
    }

    return visitante

def reconhecer(visitante):
    global configuracao
    global gerador_dados_falsos

    print("Iniciando reconhecimento de candidato.")
    foto_visitante = face_recognition.load_image_file(visitante["foto"])
    encoding_foto_visitante = face_recognition.face_encodings(foto_visitante)[0]

    reconhecido = False
    for ficha in configuracao["candidatos"]:
        fotos_banco = ficha["fotos"]
        total_reconhecimentos = 0

        for foto in fotos_banco:
            foto_banco = face_recognition.load_image_file(foto)
            encoding_foto_banco = face_recognition.face_encodings(foto_banco)[0]

            foto_reconhecida = face_recognition.compare_faces([encoding_foto_visitante], encoding_foto_banco)[0]
            if foto_reconhecida:
                total_reconhecimentos += 1

        if total_reconhecimentos/len(fotos_banco) > 0.7:
            reconhecido = True

            visitante["ficha"] = {}
            visitante["ficha"]["nome"] = gerador_dados_falsos.name()
            visitante["ficha"]["idade"] = random.randint(7, 100)
            visitante["ficha"]["cor"] = random.choice(["Preta", "Parda", "Indigena"])
            visitante["ficha"]["escolaridade"] = random.choice(["Particular", "Publica"])
            visitante["ficha"]["renda"] = random.choice(["1212", "2456", "2985"])
            visitante["ficha"]["endereco"] = gerador_dados_falsos.address()

    return reconhecido, visitante

def imprimir_ficha(candidato):
    print("Nome Completo:", candidato["ficha"]["nome"])
    print("Idade:", candidato["ficha"]["idade"])
    print("Cota Racial: Pessoa", candidato["ficha"]["cor"])
    print("Escolaridade:", candidato["ficha"]["escolaridade"])
    print("Renda Familiar:", candidato["ficha"]["renda"])

def reconhecer_candidato(env):
    global candidatos_reconhecidos

    while True:
        print("Reconhecendo um candidato em ciclo/tempo: ", env.now)

        visitante = simular_visita()
        reconhecido, candidato = reconhecer(visitante)
        if reconhecido:
            id_candidato = secrets.token_hex(nbytes=16).upper()
            candidatos_reconhecidos[id_candidato] = candidato

            print("Um candidato foi reconhecido. Ficha:")
            imprimir_ficha(candidato)
        else:
            print("Candidato não foi reconhecido")

        yield env.timeout(TEMPO_CANDIDATO)

def verificar_escolaridade(env):
    global candidatos_reconhecidos
    global candidatos_perfil2
    global candidatos_perfil1

    while True:
        if len(candidatos_reconhecidos):
            print("Verificando Escolaridade do candidato em ciclo/tempo", env.now)

            for id_candidato, candidato in list(candidatos_reconhecidos.items()):

                if((candidato["ficha"]["escolaridade"] == "Publica")):
                    candidatos_perfil2[id_candidato] = candidato
                    candidatos_reconhecidos.pop(id_candidato)
                    print("Candidato", candidato["ficha"]["nome"], "APROVACAO COM 100% pela ESCOLARIDADE:",candidato["ficha"]["escolaridade"])

                if(candidato["ficha"]["escolaridade"] == "Particular"):
                    candidatos_perfil1[id_candidato] = candidato
                    candidatos_reconhecidos.pop(id_candidato)
                    print("Candidato", candidato["ficha"]["nome"], "APROVACAO COM 50% pela ESCOLARIDADE:",candidato["ficha"]["escolaridade"])

            yield env.timeout(TEMPO_ESCOLARIDADE)
        else:
            yield env.timeout(1)

def verificar_cota_racial(env):
    global candidatos_reconhecidos
    global candidatos_perfil2
    global candidatos_perfil1

    while True:
        if len(candidatos_reconhecidos):
            print("Verificando Cota Racial do candidato em ciclo/tempo", env.now)

            for id_candidato, candidato in list(candidatos_reconhecidos.items()):

                if((candidato["ficha"]["cor"] == "Preta")):
                    candidatos_perfil2[id_candidato] = candidato
                    candidatos_reconhecidos.pop(id_candidato)
                    print("Candidato", candidato["ficha"]["nome"], "APROVADO COM 100% pela COTA RACIAL: Pessoa",candidato["ficha"]["cor"])

                if(candidato["ficha"]["cor"] == "Indigena"):
                    candidatos_perfil1[id_candidato] = candidato
                    candidatos_reconhecidos.pop(id_candidato)
                    print("Candidato", candidato["ficha"]["nome"], "APROVADO COM 20% pela COTA RACIAL: Pessoa",candidato["ficha"]["cor"])

            yield env.timeout(TEMPO_COTA)
        else:
            yield env.timeout(1)

def verificar_renda_familiar(env):
    global candidatos_reconhecidos
    global candidatos_perfil2
    global candidatos_perfil1

    while True:
        if len(candidatos_reconhecidos):
            print("Verificando Renda familiar do candidato em ciclo/tempo", env.now)

            for id_candidato, candidato in list(candidatos_reconhecidos.items()):

                if((candidato["ficha"]["renda"] == "1212")):
                    candidatos_perfil2[id_candidato] = candidato
                    candidatos_reconhecidos.pop(id_candidato)
                    print("Candidato", candidato["ficha"]["nome"], "APROVADO COM 100% pela RENDA FAMILIAR:",candidato["ficha"]["renda"])

                if(candidato["ficha"]["renda"] == "2456"):
                    candidatos_perfil1[id_candidato] = candidato
                    candidatos_reconhecidos.pop(id_candidato)
                    print("Candidato", candidato["ficha"]["nome"], "APROVADO COM 50% pela RENDA FAMILIAR:",candidato["ficha"]["renda"])

            yield env.timeout(TEMPO_RENDA)
        else:
            yield env.timeout(1)

if __name__ == "__main__":
    preparar()

    env = simpy.Environment()
    env.process(reconhecer_candidato(env))
    env.process(verificar_renda_familiar(env))
    env.process(verificar_cota_racial(env))
    env.process(verificar_escolaridade(env))
    env.run(until=3000)


def stock_list(listOfArt, listOfCat):
	ls=[]
	if(len(listOfCat)==0 or len(listOfArt)==0):
		string=""
	else:
		for i in listOfCat:
			value = 0
			for j in listOfArt:
				if(i==j[0]):
					s=j.split()
					value=value + int(s[1])
			ins="("+i+" : "+str(value)+")"
			ls.append(ins)
		string=" - ".join(ls)
	return string
a=stock_list(["ABART 20","CDXEF 50","BKWRK 25","BTSQZ 89","DRTYM 60"],["A","B","C","W"])
print(a)


import json
import sys
import os

# Class that can parse a JSON Schema. I'm not entirely certain that this is correct,
# and furthermore, it is in some ways tailored specifically to read JSON Schemas for
# EUCDM (e.g. when finding description for a field).
class JSONSchemaParser():
    # Args:
    # filename is the name of the JSON Schema file.
    def __init__(self, filename):
        self.filename = filename
        self.path = os.path.dirname(filename)
        self.nodeclass = None       # Node class, in case the result should be stored a node class, rather than JSON structure.
        self.fullstructure = None   # Full JSON structure.
        self.js = None              # The structure under the top level 'properties' This is what I will parse,
                                    # to avoid going into a subtree used by $ref. This might not be present, but if it is,
                                    # it should not be processed independently.

        self.fullstructure = self.readJSON(filename)

        for k in self.fullstructure:
            if k == 'properties':
                self.js = self.fullstructure[k]

        self.keywords = ['enum', 'description', 'additionalProperties', 'minimum', 'maximum', 'required', 'pattern', 'maxLength', 'minLength', 'maxItems', 'minItems', 'type', 'multipleOf']

    def setNodeclass(self, nodeclass):
        self.nodeclass = nodeclass

    def readJSON(self, filename):
        with open(filename) as f:
            return json.load(f)

    def parseJSD(self):
        result = self._parseJSD(self.js)
        return result

    # Returns all keys in json object, with duplicates.
    # Currently this uses a json structure to record the result, but should really use a graph structure,
    # e.g. EUCDMNode.
    def _parseJSD(self, js):
        if isinstance(js, dict):
            result = {}

            for k in js:
                if k in ['properties', 'items']:
                    return self._parseJSD(js[k])
                elif isinstance(js[k], dict):
                    cards = self.getCardinalities(js[k])
                    descr = self.getDescription(js[k]).replace('>', '#62')
                    key = '>'.join([k]+[descr]+cards)# Primitive way of storing name, description and cardinalities as the key.
                    result[key] = self._parseJSD(js[k])
                elif k in self.keywords:
                    pass
                    #result[k] = js[k]
                elif k == '$ref':
                    refvalue = js[k].lstrip(' #')
                    if refvalue.startswith('/'):
                        refvalue = refvalue[1:]
                        # Search for reference in self.fullstructure, which is the full json structure:
                        refstruct = self.findPath(self.fullstructure, refvalue)
                        result = self._parseJSD(refstruct)
                    elif refvalue.startswith('file:'):
                        refvalue = refvalue[4:]
                        refvalue = self.path + '/' + refvalue.lstrip(':/')
                        result = self._parseJSD(self.readJSON(refvalue))
                else:
                    result[k] = self._parseJSD(js[k])
        elif isinstance(js, list):
            result = []
            for i in range(len(js)):
                result.append(self._parseJSD(js[i]))
        else:
            result = None # { js: 'Nuffink' }

        return result

    # Look for the given key in all of jsonstruct.
    # Returns list of all the corresponding values.
    # Intended to work with finding string values, e.g.
    # the vlues for key '$ref'.
    def findOccurrences(self, key, jsonstruct):
        list = []

        for k in jsonstruct:
            if k == key and isinstance(jsonstruct[k], str):
                list.append(jsonstruct[k])
            elif isinstance(jsonstruct[k], dict):
                list.extend(self.findOccurrences(key, jsonstruct[k]))

        return list

    # Example path: Root/Elem2/elem3
    # Returns the structure at json['Root']['Elem2']['Elem3'],
    # if each part of the path is found in json.
    def findPath(self, json, path):
        names = path.split('/')
        for name in names:
            if name in json:
                json = json[name]
            else:
                return None
        return json

    def getCardinalities(self, jsonstruct):
        mincard = 1
        maxcard = 1

        if 'type' in jsonstruct and jsonstruct['type'] == 'array':
            if 'minItems' in jsonstruct:
                mincard = jsonstruct['minItems']
            if 'maxItems' in jsonstruct:
                maxcard = jsonstruct['maxItems']
        return [str(mincard), str(maxcard)]

    def getDescription(self, jsonstruct):
        if 'description' in jsonstruct:
            return jsonstruct['description']
        elif '$ref' in jsonstruct:
            return self.getDescriptionInReference(jsonstruct['$ref'])
        elif 'items' in jsonstruct and '$ref' in jsonstruct['items']:
            return self.getDescriptionInReference(jsonstruct['items']['$ref'])
        else:
            return ''

    def getDescriptionInReference(self, jref):
        if jref.startswith('#/'):
            refvalue = jref[2:]
            refstruct = self.findPath(self.fullstructure, refvalue)
            if 'description' in refstruct:
                return refstruct['description']
            else:
                return ''
        else:
            return ''

if __name__ == "__main__":
    filename = sys.argv[1]

    jp = JSONSchemaParser(filename)

    res = jp.parseJSD()
    print(json.dumps(res, indent=4))


import inspect
import pickle

from copy import copy

from pytest import raises
from hypothesis import given, assume
from hypothesis.strategies import integers

import segpy
from segpy.header import field, Header, are_equal
from segpy.field_types import Int32, NNInt32
from segpy.datatypes import LIMITS, SegYType

from test.predicates import check_balanced

class ExampleHeader(Header):

    field_a = field(
        Int32, offset=1, default=0, documentation=
        """Field A. This is field A.""")

    field_b = field(
        NNInt32, offset=5, default=42, documentation=
        "Field B. This is field B.")

    field_c = field(
        Int32, offset=9, default=-1, documentation=
        "Field C. This is field C.")

class TestHeader:

    def test_initialize_with_defaults(self):
        h = ExampleHeader()
        assert h.field_a == 0
        assert h.field_b == 42
        assert h.field_c == -1

    def test_initialize_with_positional_arguments(self):
        h = ExampleHeader(14, 22, 8)
        assert h.field_a == 14
        assert h.field_b == 22
        assert h.field_c == 8

    def test_initialize_with_keyword_arguments(self):
        h = ExampleHeader(field_a=14, field_b=22, field_c=8)
        assert h.field_a == 14
        assert h.field_b == 22
        assert h.field_c == 8

    def test_initialize_with_positional_and_keyword_arguments(self):
        h = ExampleHeader(14, 22, field_c=8)
        assert h.field_a == 14
        assert h.field_b == 22
        assert h.field_c == 8

    def test_out_of_range_field_values_raises_value_error(self):
        with raises(ValueError):
            ExampleHeader(14, -1, field_c=8)

    def test_illegal_keyword_argument_raises_type_error(self):
        with raises(TypeError):
            ExampleHeader(14, 1, field_x=8)

    def test_ordered_field_names(self):
        assert ExampleHeader.ordered_field_names() == ('field_a', 'field_b', 'field_c')

    @given(a=integers(*LIMITS[SegYType.INT32]),
           b=integers(*LIMITS[SegYType.NNINT32]),
           c=integers(*LIMITS[SegYType.INT32]))
    def test_copy(self, a, b, c):
        h1 = ExampleHeader(a, b, c)
        h2 = copy(h1)
        assert h1 is not h2
        assert h1.field_a == h2.field_a
        assert h1.field_a == h2.field_a
        assert h1.field_a == h2.field_a

    @given(a=integers(*LIMITS[SegYType.INT32]),
           b=integers(*LIMITS[SegYType.NNINT32]),
           c=integers(*LIMITS[SegYType.INT32]))
    def test_repr(self, a, b, c):
        r = repr(ExampleHeader(a, b, c))
        assert str(a) in r
        assert str(b) in r
        assert str(c) in r
        assert 'field_a' in r
        assert 'field_b' in r
        assert 'field_c' in r
        assert 'ExampleHeader' in r
        assert check_balanced(r)

    @given(a=integers(*LIMITS[SegYType.INT32]),
           b=integers(*LIMITS[SegYType.NNINT32]),
           c=integers(*LIMITS[SegYType.INT32]))
    def test_equality(self, a, b, c):
        lhs = ExampleHeader(a, b, c)
        rhs = ExampleHeader(a, b, c)
        assert are_equal(lhs, rhs)

    @given(a=integers(*LIMITS[SegYType.INT32]),
           b=integers(*LIMITS[SegYType.NNINT32]),
           c=integers(*LIMITS[SegYType.INT32]))
    def test_inequality(self, a, b, c):
        assume(a != 0)
        lhs = ExampleHeader(-a, b, c)
        rhs = ExampleHeader(a, b, c)
        assert not are_equal(lhs, rhs)

    def test_inequality_different_type(self):
        h = ExampleHeader(1, 2, 3)
        assert not are_equal(h, 42)

    def test_read_illegal_attribute_raises_attribute_error(self):
        h = ExampleHeader(1, 2, 3)
        with raises(AttributeError):
            _ = h.field_x

    @given(a=integers(*LIMITS[SegYType.INT32]),
           b=integers(*LIMITS[SegYType.NNINT32]),
           c=integers(*LIMITS[SegYType.INT32]))
    def test_pickle_roundtrip(self, a, b, c):
        h1 = ExampleHeader(a, b, c)
        s = pickle.dumps(h1)
        h2 = pickle.loads(s)
        assert are_equal(h1, h2)

    @given(a=integers(*LIMITS[SegYType.INT32]),
           b=integers(*LIMITS[SegYType.NNINT32]),
           c=integers(*LIMITS[SegYType.INT32]))
    def test_pickle_versioning_mismatch_raises_type_error(self, a, b, c):
        h1 = ExampleHeader(a, b, c)
        s = pickle.dumps(h1)
        s = s.replace(segpy.__version__.encode('ascii'), b'xxxxx')
        with raises(TypeError):
            pickle.loads(s)

    def test_delete_field_raises_attribute_error(self):
        h1 = ExampleHeader(1, 2, 3)
        with raises(AttributeError):
            del h1.field_a

class TestNamedField:

    def test_name(self):
        assert ExampleHeader.field_a.name == 'field_a'

    def test_value_type(self):
        assert ExampleHeader.field_a.value_type == Int32

    def test_offset(self):
        assert ExampleHeader.field_a.offset == 1

    def test_default(self):
        assert ExampleHeader.field_a.default == 0

    def test_doc(self):
        assert inspect.getdoc(ExampleHeader.field_a) == "Field A. This is field A."

    def test_repr(self):
        r = repr(ExampleHeader.field_a)
        assert 'FieldAField' in r
        assert 'name' in r
        assert 'value_type' in r
        assert 'default' in r
        assert 'field_a' in r
        assert 'Int32' in r
        assert '1' in r
        assert '0' in r
        assert check_balanced(r)


#!/usr/bin/env python3

# ffmpeg wrapper
import multiprocessing
from os.path import isdir, isfile
import ffmpy

# argument parsing
import argparse

# multiprocessing stuff
from multiprocessing import Pool
from multiprocessing import cpu_count

# executing some commands
import subprocess

# parsing json output of loudnorm
import json

# file/directory handling
import os

# most recent starttime for program
import time

from random import randint

from typing import Any, Optional

"""

"""

# FIXME
# inputfile = (
#    '/home/marc/Downloads/FalKKonE - 01 Aria (From "Berserk: The Golden Age Arc").flac'
# )
# inputfile = "/home/marc/Downloads/test441.opus"
# outputfile = "/home/marc/Downloads/test441_out.opus"

# srcfolder = "/home/marc/Downloads/MusikRaw"
# destfolder = "/home/marc/Downloads/Musik"

musicfile_extensions = (".flac", ".wav", ".mp3", ".m4a", ".aac", ".opus")

def get_format(inputfile) -> str:
    # get codec format
    # https://stackoverflow.com/a/29610897
    # this shows the codecs of all audio streams present in the file, which shouldn't matter unless you have more than one stream
    ff = ffmpy.FFprobe(
        inputs={inputfile: None},
        global_options=(
            "-v quiet",
            "-select_streams a",
            "-show_entries stream=codec_name",
            "-of default=noprint_wrappers=1:nokey=1",
        ),
    )
    # print(ff.cmd)
    proc = subprocess.Popen(ff.cmd, shell=True, stdout=subprocess.PIPE)
    # NOTE read output from previous command
    # rstrip: remove trailing newline
    # decode: convert from binary string to normal string
    format: str = (
        proc.stdout.read()  # pyright: ignore[reportOptionalMemberAccess]
        .rstrip()
        .decode("utf8")
    )
    # print(format)
    return format

def remove_picture(inputfile):
    """
    This function makes sure no image is attached to the audio stream.
    An image might cause problems for the later conversion to opus.

    Parameters:
        inputfile (str): Path to file
    """
    tmpfile = os.path.splitext(inputfile)[0] + ".tmp" + os.path.splitext(inputfile)[1]
    ff = ffmpy.FFmpeg(
        inputs={inputfile: None},
        outputs={tmpfile: "-vn -c:a copy"},
        global_options=("-v error"),
    )
    ff.run()
    os.remove(inputfile)
    os.rename(tmpfile, inputfile)

def loudness_info(inputfile) -> dict[str, str]:
    print("Measuring loudness of ", os.path.basename(inputfile))
    # get format from file
    # inputformat = get_format(inputfile)
    # NOTE format is actually unnecessary here
    ff = ffmpy.FFmpeg(
        inputs={inputfile: None},
        outputs={"/dev/null": "-pass 1 -filter:a loudnorm=print_format=json -f null"},
        global_options=("-y"),
    )

    # print(ff.cmd)
    proc = subprocess.Popen(
        ff.cmd, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE
    )
    # NOTE get loudness info from subprocess
    # rstrip: remove trailing newline
    # decode: convert from binary string to utf8
    # splitlines: list of lines (only 12 last ones, length of the output json)
    # join: reassembles the list of lines and separates with "\n"
    loudness_json: str = "\n".join(
        proc.stdout.read().rstrip().decode("utf8").splitlines()[-12:]
    )
    # decode json to dict
    loudness: dict[str, str] = json.loads(loudness_json)
    # print(loudness_json)
    # print(ff.cmd)
    return loudness

def convert(inputfile, outputfile, loudness) -> Optional[list[Any]]:
    print("Working on ", os.path.basename(inputfile))
    # coverpath = os.path.join(os.path.dirname(inputfile), "cover.jpg")
    # NOTE including covers into ogg/opus containers currently doesn't work
    # https://trac.ffmpeg.org/ticket/4448
    inputcmd = {inputfile: None}
    outputcmd = {
        outputfile: "-pass 2"
        " "
        "-filter:a"
        " "
        "loudnorm=I=-26.0:"
        "LRA=9.0:"
        "measured_I={input_i}:"
        "measured_LRA={input_lra}:"
        "measured_tp={input_tp}:measured_thresh={input_thresh}:"
        "print_format=json"
        " "
        "-c:a libopus"
        " "
        "-b:a 320k".format(
            input_i=loudness["input_i"],
            input_lra=loudness["input_lra"],
            input_tp=loudness["input_tp"],
            input_thresh=loudness["input_thresh"],
        )
    }

    ff = ffmpy.FFmpeg(
        inputs=inputcmd,
        outputs=outputcmd,
        # global_options=("-y", "-v error"),
        global_options=("-y"),
    )
    # ff.run()
    proc = subprocess.Popen(
        ff.cmd, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE
    )
    # NOTE get loudness info from subprocess
    # rstrip: remove trailing newline
    # decode: convert from binary string to utf8
    # splitlines: list of lines (only 12 last ones, length of the output json)
    # join: reassembles the list of lines and separates with "\n"
    loudness_json: str = "\n".join(
        proc.stdout.read().rstrip().decode("utf8").splitlines()[-12:]
    )
    # decode json to dict
    loudness_new: dict[str, str] = json.loads(loudness_json)
    if loudness_new["normalization_type"] != "linear":
        nonlinear: list[Any] = [inputfile, loudness_new]
        return nonlinear

def main(inputfile: str) -> Optional[list[Any]]:
    """
    Main program loop

    Parameters:
        inputfile (str): Path to input file
    """
    # set output folder to parent path + "normalized"
    outputfolder = os.path.join(os.path.dirname(inputfile), "normalized")
    # NOTE create output folder
    # because multiple parallel processes are at work here,
    # there might be conflicts with one trying to create the directory although it already exists
    # this while loop makes sure the directory does exist
    # the try/except block ensures the error is caught and (hopefully) doesn't happen again just after with random sleep
    # there's very likely a better way to do this, idk
    while not os.path.isdir(outputfolder):
        try:
            os.mkdir(outputfolder)
        except:
            time.sleep(randint(0, 4))

    # output file path
    noext_infile: str = os.path.splitext(os.path.basename(inputfile))[0]
    outputfile: str = os.path.join(outputfolder, noext_infile + ".opus")

    # print(inputfile)
    # print(os.path.dirname(inputfile))
    # print(os.path.basename(inputfile))
    # print(outputfile)

    # remove_picture(inputfile=inputfile)
    loudness = loudness_info(inputfile=inputfile)
    nonlinear: Optional[list[Any]] = convert(
        inputfile=inputfile,
        outputfile=outputfile,
        loudness=loudness,
    )

    return nonlinear

if __name__ == "__main__":
    # start time of program
    starttime = time.time()

    parser = argparse.ArgumentParser(description="")

    # Input directory
    parser.add_argument(
        "-i", "--input-dir", required=True, type=str, help="Input source directory"
    )

    # number of cpus/threads to use, defaults to all available
    parser.add_argument(
        "-c",
        "--cpu-count",
        required=False,
        type=int,
        help="Number of cpu cores",
        default=multiprocessing.cpu_count(),
    )

    # in case you wanted to rerun the conversion for everything
    parser.add_argument(
        "-r",
        "--reset",
        required=False,
        action="store_true",
        help="Rerun conversion for all files",
    )

    args = parser.parse_args()

    srcfolder = args.input_dir

    cpu = args.cpu_count

    reset = args.reset

    # file where last run timestamp is stored
    timefile = os.path.join(srcfolder, "run.time")

    # list of non-linear normalizations
    nonlinear_all: Optional[list[Any]] = []

    # get time of previous run
    if reset:
        timeprev = 0
    elif os.path.isfile(timefile):
        with open(timefile, "r") as file:
            timeprev = file.read()
    else:
        timeprev = 0

    # print(timeprev)

    musicfiles: list[str] = []
    for root, dirs, files in os.walk(srcfolder):
        # ignore the "normalized" subfolder
        dirs[:] = [d for d in dirs if d not in ["normalized"]]
        for file in files:
            if file.endswith(musicfile_extensions):
                filepath = os.path.join(root, file)
                # only file newer than the last run are added
                if os.path.getmtime(filepath) >= float(timeprev):
                    musicfiles.append(os.path.join(root, file))

    # print(musicfiles)

    with Pool(cpu) as p:
        nonlinear_all: Optional[list[Any]] = p.map(main, musicfiles)

    # write this run's time into file
    with open(timefile, "w") as file:
        file.write(str(starttime))

    print("Dynamically normalized music:")
    for i in nonlinear_all:
        # NOTE ignore empty and "None" values
        if i:
            print(i)


from __future__ import division, print_function, unicode_literals
import os
import io
import json
import random
import time
import math
import numpy as np
import scipy as sp
from scipy.special import gammaln
from scipy.special import gamma
from numpy.random import random as randfloat

class CountMaster(object):
    def __init__(self, num_topics, alpha, beta):
        self.num_topics = num_topics
        self.alpha = alpha
        self.beta = beta
        self.types_to_numbers = {}
        self.numbers_to_types = []
        self.type_counts = []
        self.total_tokens = 0
        # these are populated once the vocabulary stops growing
        self.topic_totals = [0 for t in xrange(num_topics)]
        self.topics = []

        # stores 3-tuples, format: (total topics, tokens, token numbers, token topics)
        # types (list, list, list, list)
        self.document_topic_counts = []
        self.document_token_numbers = []
        self.document_token_topics = []

    def add_tokens(self, tokens):
        tokens = tokens
        numbers = []
        token_topics = []
        doc_topics = [0 for t in xrange(self.num_topics)]
        for token in tokens:
            if token not in self.types_to_numbers:
                next_key = len(self.numbers_to_types)
                self.numbers_to_types.append(token)
                self.type_counts.append(0)
                self.types_to_numbers[token] = next_key
            self.type_counts[self.types_to_numbers[token]] += 1
            numbers.append(self.types_to_numbers[token])
            topic_assignment = random.randint(0, self.num_topics-1)
            token_topics.append(topic_assignment)
            doc_topics[topic_assignment] += 1

        self.document_topic_counts.append(doc_topics)
        self.document_token_numbers.append(numbers)
        self.document_token_topics.append(token_topics)
        self.total_tokens += len(tokens)
        #~ print(doc_topics)
        #~ print(tokens)
        #~ print(numbers)
        #~ print(token_topics)
        #~ print()
        #~ print(self.topics)
        #~ print()

    def done_adding_tokens(self):
        self.topics = [[0 for i in xrange(len(self.type_counts))] for t in xrange(self.num_topics)]
        for d in xrange(len(self.document_topic_counts)):
            token_numbers = self.document_token_numbers[d]
            token_topics = self.document_token_topics[d]
            for i, tok_num in enumerate(token_numbers):
                topic = token_topics[i]
                self.topic_totals[topic] += 1
                self.topics[topic][tok_num] += 1
        self.check_counts()

    def check_counts(self):
        # check that the counts are adding up properly
        sum_tokens = 0
        sum_tokens2 = 0
        for t in xrange(self.num_topics):
            sum_tokens += self.topic_totals[t]
            for t_count in self.topics[t]:
                sum_tokens2 += t_count
        assert sum_tokens == sum_tokens2 and sum_tokens == self.total_tokens

    def get_num_documents(self):
        return len(self.document_topic_counts)

    def get_num_tokens(self, doc_index):
        return len(self.document_token_numbers[doc_index])

    def get_pi_vector(self, doc_index, tok_index):
        pi = []

        token_number = self.document_token_numbers[doc_index][tok_index]
        token_topic = self.document_token_topics[doc_index][tok_index]
        vocab_beta = len(self.types_to_numbers)*self.beta

        old_topic = self.document_token_topics[doc_index][tok_index]
        self.document_topic_counts[doc_index][old_topic] -= 1
        self.topic_totals[old_topic] -= 1
        self.topics[old_topic][token_number] -= 1

        total = 0
        for j in xrange(self.num_topics):
            try:
                #~ top_left = self.topics[j][token_number] + self.beta
                #~ bottom_left = self.topic_totals[j] + vocab_beta
                #~ top_right = document[CountMaster.DOCUMENT_TOPIC_COUNTS][j] + self.alpha
                #~ bottom_right = self.get_num_tokens(doc_index) + topic_alpha
                #~ if j == token_topic:
                    #~ bottom_left -= 1
                    #~ top_right -= 1
                #~ left = top_left/bottom_left
                #~ right = top_right/bottom_right
                #~ pi_j = left*right
                #~ pi.append(pi_j)
                #~ total += pi_j
                top_left = self.topics[j][token_number] + self.beta
                bottom_left = self.topic_totals[j] + vocab_beta
                top_right = self.document_topic_counts[doc_index][j] + self.alpha
                pi_j = (top_left/bottom_left)*top_right
                pi.append(pi_j)
                total += pi_j
            except:
                print()
                print(top_left)
                print(bottom_left)
                print(self.topics[j][0])
                print(self.alpha)
                print(self.beta)
                print(vocab_beta)
                print()
                raise

        return pi, total

    def select_new_topic(self, pi, total):
        r = random.random()*total
        total_num = 0
        for topic, num in enumerate(pi):
            assert num > 0
            total_num += num
            if r < total_num:
                break
        return topic

    def update_token_topic_assignment(self, doc_index, tok_index, new_topic):
        token_number = self.document_token_numbers[doc_index][tok_index]
        self.document_token_topics[doc_index][tok_index] = new_topic
        self.document_topic_counts[doc_index][new_topic] += 1
        self.topic_totals[new_topic] += 1
        self.topics[new_topic][token_number] += 1

    def print_topic(self, topic, top_n):
        t = {}
        for token_number, count in enumerate(self.topics[topic]):
            t[token_number] = count
        s = sorted(t, cmp=lambda x,y: y-x, key=lambda x: t[x])
        s = s[0:top_n]
        print('Topic %s: '%(str(topic)), end='')
        for token_number in s:
            print('%s(%s) '%(str(self.numbers_to_types[token_number]), str(t[token_number])), end='')

    def log_beta(self, alphas, constant):
        s = gammaln(alphas[0] + constant)
        for i in xrange(1, len(alphas)):
            if alphas[i]+constant == 0:
                print(alphas[i])
            s += gammaln(alphas[i] + constant)
        return s - gammaln(sum(alphas)+len(alphas)*constant)

    def log_likelihood(self):
        #~ def gammaln(num):
            #~ return math.log(gamma(num))
        V = len(self.numbers_to_types)
        T = self.num_topics
        topics = self.topics
        B = self.beta
        VB = V*B
        A = self.alpha
        TA = T*A
        D = self.get_num_documents()

        #~ ll = T*(gammaln(VB) - V*gammaln(B))
        #~ for j in xrange(T):
            #~ temp_topic_ll = -gammaln(self.topic_totals[j] + VB)
            #~ for w_count in topics[j]:
                #~ if w_count > 0:
                    #~ temp_topic_ll += gammaln(w_count + B)
            #~ ll += temp_topic_ll

        ll = 0
        for d in xrange(D):
            ll += self.log_beta(self.document_topic_counts[d], A)
        for j in xrange(T):
            ll += self.log_beta(self.topics[j], B)
            #~ j_total = self.topic_totals[j]
            #~ for w_count in topics[j]:
                #~ if w_count > 0:
                    #~ ll += math.log(w_count) - math.log(j_total)

        #~ V = len(self.numbers_to_types)
        #~ T = self.num_topics
        #~ topics = self.topics
        #~ B = self.beta
        #~ VB = V*B
        #~ A = self.alpha
        #~ TA = T*A
        #~ D = self.get_num_documents()
                #~
        #~ ll = T*(gammaln(VB + 1) - V*gammaln(B + 1))
        #~ for j in xrange(T):
            #~ temp_topic_ll = -gammaln(self.topic_totals[j] + VB + 1)
            #~ for w_count in topics[j]:
                #~ if w_count > 0:
                    #~ temp_topic_ll += gammaln(w_count + B + 1)
            #~ ll += temp_topic_ll

        #~ ll += D*(gammaln(TA) - T*gammaln(A))
        #~ for d in xrange(D):
            #~ temp_doc_ll = -gammaln(self.get_num_tokens(d) + TA)
            #~ for doc_topic_count in self.document_token_topics[d]:
                #~ temp_doc_ll += gammaln(doc_topic_count + A)
            #~ ll += temp_doc_ll

        return ll

class LDA(object):
    def __init__(self, num_topics, alpha=0.1, beta=0.1, burn=10, length=10):
        self.alpha = alpha
        self.beta = beta
        self.num_topics = num_topics
        self.burn = burn
        self.length = length
        self.total_tokens = 0
        self.log_likelihoods = []

    def run_analysis(self, document_iterator, seed=0):
        #~ random.seed(seed)
        self.cm = cm = CountMaster(self.num_topics, self.alpha, self.beta)

        print('----------Initializing----------')
        # preprocess documents
        for tokens in document_iterator:
            cm.add_tokens(tokens)

        cm.done_adding_tokens()

        print("Vocab Size:", len(cm.numbers_to_types))
        print("Dataset Size:", cm.get_num_documents())

        self.log_likelihoods.append(cm.log_likelihood())

        print('----------Sampling----------')
        for time_step in xrange(self.burn+self.length):
            print('Time Step: %s'%(str(time_step)), end='')
            start_time = time.time()
            for doc_index in xrange(cm.get_num_documents()):
                for tok_index in xrange(cm.get_num_tokens(doc_index)):
                    # get pi vector
                    pi, total = cm.get_pi_vector(doc_index, tok_index)
                    #~ print(pi)
                    # select topic assignment
                    new_topic = cm.select_new_topic(pi, total)
                    # update counts
                    cm.update_token_topic_assignment(doc_index, tok_index, new_topic)
            log_likelihood = cm.log_likelihood()
            print('; Log Likelihood: %s'%(str(log_likelihood)), end='')
            self.log_likelihoods.append(log_likelihood)
            print('; Seconds: %s'%(str(time.time()-start_time)))

        print('----------Done Sampling-----------')

    def print_topics(self, top_n):
        for t in xrange(self.num_topics):
            self.cm.print_topic(t, top_n)
            print()


import socket
from ip2locationio.common import urlencode, httprequest, is_valid_ip

# Windows version of Python does not provide it
# for compatibility with older versions of Windows.
if not hasattr(socket, 'inet_pton'):
    def inet_pton(t, addr):
        import ctypes
        a = ctypes.WinDLL('ws2_32.dll')
        in_addr_p = ctypes.create_string_buffer(b(addr))
        if t == socket.AF_INET:
            out_addr_p = ctypes.create_string_buffer(4)
        elif t == socket.AF_INET6:
            out_addr_p = ctypes.create_string_buffer(16)
        n = a.inet_pton(t, in_addr_p, out_addr_p)
        if n == 0:
            raise ValueError('Invalid address')
        return out_addr_p.raw
    socket.inet_pton = inet_pton

class IPGeolocation:

    def __init__(self,configuration):
        # print('apikey: ' + configuration.getapikey())
        self.apikey = configuration.getapikey()
        self.moduleversion = configuration.getmoduleversion()

    def lookup(self,ip,language=''):
        '''This function will look the given IP address up in IP2Location web service.'''
        parameters = urlencode((("key", self.apikey), ("ip", ip), ("format", "json"), ("lang", language), ("source", "sdk-python-iplio"), ("source_version", self.moduleversion)))
        response = httprequest("api.ip2location.io", "/?", parameters)
        if (response == None):
            # return False
            raise IP2LocationIOAPIError('IPGeolocation lookup error.')
        if ('error' in response):
        # if (('response' in response) and (response['response'] != 'OK')):
            raise IP2LocationIOAPIError(response['error']['error_message'])
            # return(response['response'])
        return response

class IP2LocationIOAPIError(Exception):
    """Raise for IP2Location API Error Message"""


from math import sqrt, sin, cos, exp
import numpy as np
from numba import njit
from ops.core.rotations import get_orientations

@njit(cache=True)
def diffNRV(rv, Mij):
    """ Differentiate orientations wrt rotation vectors. """
    N = rv.shape[0]
    Aaj = np.zeros((4, 3))
    Bia = np.zeros((3, 4))
    eye = np.array([[0.0, 1.0, 0.0, 0.0],
                    [0.0, 0.0, 1.0, 0.0],
                    [0.0, 0.0, 0.0, 1.0]])
    for i in range(N):
        r0 = rv[i, 0]
        r1 = rv[i, 1]
        r2 = rv[i, 2]

        u = sqrt(r0 * r0 + r1 * r1 + r2 * r2)
        a = 0.5 * u
        ca = cos(a)
        sau = sin(a) / u

        q0 = ca
        q1 = sau * r0
        q2 = sau * r1
        q3 = sau * r2

        Aaj[0, 0] = 2 * q2
        Aaj[0, 1] = -2 * q1
        Aaj[0, 2] = 2 * q0
        Aaj[1, 0] = 2 * q3
        Aaj[1, 1] = -2 * q0
        Aaj[1, 2] = -2 * q1
        Aaj[2, 0] = 2 * q0
        Aaj[2, 1] = 2 * q3
        Aaj[2, 2] = -2 * q2
        Aaj[3, 0] = 2 * q1
        Aaj[3, 1] = 2 * q2
        Aaj[3, 2] = 2 * q3

        Bia[0, 0] = -0.5 * sau * r0
        Bia[1, 0] = -0.5 * sau * r1
        Bia[2, 0] = -0.5 * sau * r2

        fac = (0.5 * ca - sau) / u**2
        for l in range(3):
            for m in range(1, 4):
                Bia[l, m] = sau * eye[l, m] + fac * rv[i, l] * rv[i, m-1]

        for l in range(3):
            for n in range(3):
                Mij[i, l, n] = 0.0
                for m in range(4):
                    Mij[i, l, n] += Bia[l, m] * Aaj[m, n]

@njit(cache=True)
def energyAndJacobian(x, conn, fvk=0.01, re=1.0, a=4.620981204):
    """ Calculate energy and jacobian of the energy of the OPS shell. """
    # Prepare data arrays
    energy = 0.0
    NP = int(len(x) / 6)  # Number of particles
    X = x.reshape((-1, 3))
    jac_arr = np.zeros(NP*6)
    jac = jac_arr.reshape((-1, 3))
    jacX = jac[:NP, :]
    jacR = jac[NP:, :]

    # Get the particle positions, rotation vectors and derivatives
    pos = X[:NP, :]
    rot = X[NP:, :]
    ori = np.zeros_like(rot)
    get_orientations(rot, ori)
    dNdr = np.zeros((NP, 3, 3))
    diffNRV(rot, dNdr)

    rij = np.zeros(3)
    rn = np.zeros(3)
    m = np.zeros(3)
    n = np.zeros(3)
    dMdr = np.zeros(3)
    dCdr = np.zeros(3)
    dPhi_cVi = np.zeros(3)
    dPhi_cVj = np.zeros(3)
    dPhi_nVi = np.zeros(3)
    dPhi_nVj = np.zeros(3)

    # Iterate over all edges from connectivity and calculate
    for z in range(conn.shape[0]):

        # Get the point ids
        i = conn[z, 0]
        j = conn[z, 1]

        # Get the positions and orientations and derivatives of normals
        for k in range(3):
            rij[k] = pos[j, k] - pos[i, k]
            m[k] = ori[i, k] - ori[j, k]
            n[k] = ori[i, k] + ori[j, k]

        r = sqrt(rij[0] * rij[0] + rij[1] * rij[1] + rij[2] * rij[2])
        inv_r = 1.0 / r

        n_dot_rn = 0.0
        m_dot_m = 0.0
        for k in range(3):
            rn[k] = rij[k] * inv_r
            n_dot_rn += n[k] * rn[k]
            m_dot_m += m[k] * m[k]

        # Morse potential and Co-circularity
        exp_1 = exp(-a*(r - re))
        exp_2 = exp_1**2

        # Morse and Co-circularity derivatives
        fac = 2*a*(exp_1 - exp_2)
        fac2 = 2 * n_dot_rn * inv_r
        for k in range(3):
            dMdr[k] = fac * rn[k]
            dCdr[k] = fac2 * (n[k] - n_dot_rn * rn[k])

        # Matrix vector scalar multiplications
        #  Co-normality derivatives
        #  Co-circularity potential derivatives
        fac = 2 * n_dot_rn
        for p in range(3):
            dPhi_nVi[p] = 0.0
            dPhi_nVj[p] = 0.0
            dPhi_cVi[p] = 0.0
            dPhi_cVj[p] = 0.0
            for q in range(3):
                dPhi_nVi[p] += 2.0 * dNdr[i, p, q] * m[q]
                dPhi_nVj[p] -= 2.0 * dNdr[j, p, q] * m[q]
                dPhi_cVi[p] += fac * dNdr[i, p, q] * rn[q]
                dPhi_cVj[p] += fac * dNdr[j, p, q] * rn[q]

        # Total energy
        inv_fvk = 1.0 / fvk
        energy += (exp_2 - 2*exp_1) + (m_dot_m + n_dot_rn**2) * inv_fvk

        # Total Derivatives of energy wrt xi, vi, vj
        for k in range(3):
            jacX[i, k] -= dMdr[k] + dCdr[k] * inv_fvk
            jacX[j, k] += dMdr[k] + dCdr[k] * inv_fvk
            jacR[i, k] += (dPhi_nVi[k] + dPhi_cVi[k]) * inv_fvk
            jacR[j, k] += (dPhi_nVj[k] + dPhi_cVj[k]) * inv_fvk

    return energy, jac_arr

@njit(cache=True)
def energy(x, conn, fvk=0.01, re=1.0, a=4.620981204):
    """ Calculate the more, normality and circularity energy. """
    # Prepare data arrays
    NP = int(len(x) / 6)  # Number of particles
    morse = 0
    normality = 0
    circularity = 0
    X = x.reshape((-1, 3))

    # Get the particle positions, rotation vectors and derivatives
    pos = X[:NP, :]
    rot = X[NP:, :]
    ori = np.zeros_like(rot)
    get_orientations(rot, ori)

    # Iterate over all edges from connectivity and calculate
    for z in range(conn.shape[0]):

        # Get the point ids
        i, j = conn[z]

        # Get the positions and orientations and derivatives of normals
        rij = pos[j] - pos[i]
        r = np.linalg.norm(rij)
        rn = rij / r

        # Morse potential and derivatives
        exp_1 = np.exp(-a*(r - re))
        morse += exp_1*(exp_1 - 2)

        # Co-normality potential and derivatives
        m = ori[i] - ori[j]
        normality += np.dot(m, m)/fvk

        # Co-circularity potential and derivatives
        n_dot_rn = np.dot(ori[i] + ori[j], rn)
        circularity += (n_dot_rn**2) / fvk

    return morse, normality, circularity


# -*- coding: utf-8 -*-

import time
import json
import datetime
import socket
from urllib.parse import urljoin
from personalitics.items import PersonaliticsItem
from personalitics.utils import get_topic
from personalitics.utils.xpath_lookup import SixteenpXpath
from personalitics.utils.lua_lookup import lua_dict
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose, TakeFirst
from scrapy.http import HtmlResponse
from scrapy.http import FormRequest, Request
from scrapy.utils.python import to_bytes
from scrapy.signalmanager import SignalManager
from scrapy import signals
from scrapy_splash import SplashRequest, SplashJsonResponse
from scrapy_splash import SplashTextResponse

from selenium import webdriver
from tqdm import tqdm
import numpy as np

class SixteenPSpider(CrawlSpider, SixteenpXpath):
    name = 'sixteen_p'
    allowed_domains = ['16personalities.com']
    start_urls = ['https://www.16personalities.com/personality-types']

    xpath_rules = SixteenpXpath.XPATHS['rules']
    xpath_comment_section = SixteenpXpath.XPATHS['comment_section']
    render_js = lua_dict['render_js']

    def start_requests(self):
        lua_script = lua_dict['login']

        yield SplashRequest(
                url='https://www.16personalities.com/personality-types',
                endpoint='execute',
                cache_args=['lua_source'],
                session_id="foo",
                callback=self.parse_types,
                args= {'wait': 0.5, 'lua_source': lua_script}
            )

    # rules = [
    #         Rule(LinkExtractor(restrict_xpaths=xpath_rules['types']), follow=True, ),
    #         Rule(LinkExtractor(restrict_xpaths=xpath_rules['type_explorer']), callback='parse_comment', process_request='use_splash', follow=True),
    #         Rule(LinkExtractor(restrict_xpaths=xpath_rules['next_comment_section']), callback='parse_comment', process_request='use_splash', follow=True)
    #     ]

    def parse_types(self, response):
        # print(response.xpath('//div[@class="info"]//div[@class="name"]//text()').extract_first().strip())

        # Parse Type List
        type_selector = response.xpath(self.xpath_rules['types'] + '/@href')
        for url in type_selector.extract():
            yield SplashRequest(urljoin('https://www.16personalities.com/', url),
                                endpoint='execute',
                                session_id="foo",
                                args={'lua_source': self.render_js},
                                callback=self.parse_explorer)

    def parse_explorer(self, response):
        # print(response.xpath('//div[@class="info"]//div[@class="name"]//text()').extract_first().strip())

        # Parse Explorer List
        explore_selector = response.xpath(self.xpath_rules['type_explorer'] + '/@href')
        for url in explore_selector.extract():
            yield SplashRequest(urljoin('https://www.16personalities.com/', url),
                                endpoint='execute',
                                session_id="foo",
                                # headers = response.data['headers'],
                                args={'lua_source': self.render_js},
                                callback=self.parse_comment)

    def parse_comment(self, response):
        self.pbar.update()
        user_comments = response.xpath(self.xpath_comment_section['user_comments'])
        print(response.xpath('//div[@class="info"]//div[@class="name"]//text()').extract_first().strip())

        # Parse next comment page
        next_selector = response.xpath(self.xpath_rules['next_comment_section'] + '/@href')
        for url in next_selector.extract():
            yield SplashRequest(urljoin('https://www.16personalities.com/', url),
                                endpoint='execute',
                                session_id="foo",
                                # headers = response.data['headers'],
                                args={'lua_source': self.render_js},
                                callback=self.parse_comment)

        # Loop over the comment list
        for element in user_comments:
            loader = ItemLoader(item=PersonaliticsItem())

            # Main fields
            loader.add_value('topic', get_topic(element.xpath(self.xpath_comment_section['topic']).extract_first()))
            loader.add_value('user_id', element.xpath(self.xpath_comment_section['user_id']).extract_first())
            loader.add_value('user_type', element.xpath(self.xpath_comment_section['user_type']).extract_first())
            loader.add_value('parent_text', element.xpath(self.xpath_comment_section['parent_text']).extract_first())
            loader.add_value('child_text', element.xpath(self.xpath_comment_section['child_text']).extract_first())
            loader.add_value('date', element.xpath(self.xpath_comment_section['date']).extract_first())

            # House keeping fields
            loader.add_value('source', response.url)
            loader.add_value('project', self.settings.get('BOT_NAME'))
            loader.add_value('spider', self.name)
            loader.add_value('server', socket.gethostname())
            loader.add_value('created_at', datetime.datetime.now())
            yield loader.load_item()

    # Source https://github.com/scrapy-plugins/scrapy-splash/issues/92
    def _requests_to_follow(self, response):
        if not isinstance(
                response,
                (HtmlResponse, SplashJsonResponse, SplashTextResponse)):
            return
        seen = set()
        for n, rule in enumerate(self._rules):
            links = [lnk for lnk in rule.link_extractor.extract_links(response)
                     if lnk not in seen]
            if links and rule.process_links:
                links = rule.process_links(links)
            for link in links:
                seen.add(link)
                r = self._build_request(n, link)
                yield rule.process_request(r)

    def use_splash(self, request):
        request.meta.update(splash={
            'session_id': "foo",
            'args': {
                'wait': 1,
            },

            'endpoint': 'render.html',
        })
        return request

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(SixteenPSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_opened, signals.spider_opened)
        crawler.signals.connect(spider.spider_closed, signals.spider_closed)
        return spider

    def spider_opened(self, spider):
        self.pbar = tqdm()
        self.pbar.clear()
        self.pbar.write('Opening {} spider'.format(spider.name))

    def spider_closed(self, spider):
        self.pbar.clear()
        self.pbar.write('Closing {} spider'.format(spider.name))
        self.pbar.close()


from flask import Flask,render_template
from config_param import COS_Config_param

app = Flask(__name__)

@app.route('/')
@app.route('/<prefix>')
def navigate(prefix=''):
    from qcloud_cos import CosConfig
    from qcloud_cos import CosS3Client
    import sys
    import logging
    logging.basicConfig(level=logging.INFO, stream=sys.stdout)
    secret_id, secret_key, region, token, scheme, bucket, cos_domain, viewer_domain = COS_Config_param()
    config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme)

    client = CosS3Client(config)

    response = client.list_objects(
        Bucket=bucket,
        Prefix=prefix.replace('$','/')
    )

    files_list = []

    for rec in response['Contents']:
        key = rec['Key']
        loc = key.split('/')
        files_list.append(loc)

    current_list = []
    if prefix == '':
        for files in files_list:
            if len(files) == 2:
                if files[1] == '': current_list.append(files[0])
            if len(files) == 1:
                current_list.append(files[0])
    else:
        compat = prefix.split('$')
        deep = len(compat)
        for files in files_list:
            if files[:deep] == compat and files[deep] != '':
                current_list.append(files[deep])
    current_list = list(set(current_list))

    webdir = []
    for dirs in current_list:
        if '.' in dirs:
            webdir.append(cos_domain + prefix.replace('$','/') + '/' +dirs)
        else:
            webdir.append(viewer_domain + ((prefix + '$') if prefix else '') + dirs)

    return render_template('list.html',abbrlist = list(zip(current_list,webdir)))

if __name__ == '__main__':
    app.run()


# -*- coding: utf-8 -*-
"""
Created on Sun Mar 24 21:46:48 2019

@author: Tushar
"""

import numpy as np
import matplotlib.pyplot as plt
#import time

from sklearn import linear_model

#INITIALIZE
npos = 200
nneg = 200
mu1 = np.array([1,1])
mu2 = np.array([10,10])
sig1 = np.array([[1,0],[0,1]])
sig2 = np.array([[1,0],[0,1]])
max_step = 2e5
eta = 1e-4
tol = 1e-6

print(eta,tol)
#randomly generated data for classifier
pos = np.random.multivariate_normal(mean=mu1, cov=sig1, size=npos)
neg = np.random.multivariate_normal(mean=mu2, cov=sig2, size=nneg)
ones = np.ones((npos+nneg, 1))
dat = np.r_[pos,neg]
dat = np.c_[ones, dat]

#def logsig(wg):
#    lg = 1/(1+np.exp(-wg.T*x))
#    return lg

#write gradient and hessian function
def grad(wg):
    wtxsq = np.square(np.matmul(x,wg))
    sqrtwtxsq = np.sqrt(1+wtxsq)
    g = (x*(2*y - 1)*sqrtwtxsq-np.matmul(x,wg))/(1+wtxsq)
    g = np.matrix(np.sum(g, axis=0)).T
    return(np.asarray(g))

#def gradlg(wg):
#    p = logsig(wg)
#    P = np.diag(np.multiply(p,(1 - p)))
#    #E = np.diag(y-p)
#    #J = P*x
#    #X' * P * (y - p)
#    g = x * P * (y-p)
#    return (g)

x = dat
y = np.ones((npos,1))
y = np.r_[y,(np.zeros((nneg,1)))]

#initial solution using OLS
wi = np.matmul(np.matmul(np.linalg.inv(np.matmul(x.T,x)),x.T),y)

# PLOT function
def xyplot(beta):
    mn = - beta[0] / beta[1] - beta[2] / beta[1] * min(x[:, 2])
    mx = - beta[0] / beta[1] - beta[2] / beta[1] * max(x[:, 2])
    x1 = np.linspace(mn,mx,10);
    x2 = - beta[0] / beta[2] - beta[1] / beta[2] * x1
    colors = np.squeeze(np.asarray(y))
    plt.scatter(dat[:,1],dat[:,2], c = colors, alpha=0.7)
    plt.plot(x1,x2, "r--")
    plt.show()
    plt.clf()
    plt.cla()
    plt.close()

def xyplot1(beta1,beta2):
    mn1 = - beta1[0] / beta1[1] - beta1[2] / beta1[1] * min(x[:, 2])
    mx2 = - beta1[0] / beta1[1] - beta1[2] / beta1[1] * max(x[:, 2])
    x11 = np.linspace(mn1,mx2,20);
    x21 = - beta1[0] / beta1[2] - beta1[1] / beta1[2] * x11
    mn3 = - beta2[0] / beta2[1] - beta2[2] / beta2[1] * min(x[:, 2])
    mx4 = - beta2[0] / beta2[1] - beta2[2] / beta2[1] * max(x[:, 2])
    x12 = np.linspace(mn3,mx4,20);
    x22 = - beta2[0] / beta2[2] - beta2[1] / beta2[2] * x12
    #mn5 = - beta3[0] / beta3[1] - beta3[2] / beta3[1] * min(x[:, 2])
    #mx6 = - beta3[0] / beta3[1] - beta3[2] / beta3[1] * max(x[:, 2])
    #x13 = np.linspace(mn5,mx6,20);
    #x23 = - beta3[0] / beta3[2] - beta3[1] / beta3[2] * x13
    colors = np.squeeze(np.asarray(y))
    plt.scatter(dat[:,1],dat[:,2], c = colors, alpha=0.7)
    plt.plot(x11,x21, "r-", color = "green", linewidth = 0.7)
    plt.plot(x12,x22, "r--", label = "", linewidth = 0.7)
    plt.legend(["Given Function","Logistic Function"])
    #plt.plot(x13,x23, "r--", color = "blue", linewidth=0.5)
    plt.show()
    plt.clf()
    plt.cla()
    plt.close()

xyplot(wi)
print(wi)

#iteratively solve for optimum W parameters
i=0
w=wi
eps = tol
while i < max_step and eps >= tol:
    w_old = w/sum(abs(w))
    #w_old = w/sum(w)
    w = w+(eta*grad(w))
    eps = sum(abs(w_old - w / sum(abs(w))))
    #eps = sum(w_old - w / sum(w))
    #print(eps)
    #xyplot(w)
    i = i+1

xyplot(w)
print(i,eps)

print(w)

clf = linear_model.SGDClassifier(loss = 'log',
                                 max_iter = 1e5,
                                 learning_rate = 'constant',
                                 eta0 = 1e-4,
                                 tol = 1e-6,
                                 fit_intercept = False,
                                 shuffle=False).fit(x,np.squeeze(y))

xyplot(clf.coef_.T)
print(clf.n_iter_)
print(clf.coef_.T)

xyplot1(w,clf.coef_.T)


from django.urls import path

from . import views

urlpatterns = [
    path('about/', views.about_view, name='about'),

    path('categories/', views.CategoryListView.as_view(), name='category_list'),
    path('categories/<str:name>', views.category_detail_view, name='category_detail'),

    path('posts/<int:year>/<int:month>/<slug:slug>/',
         views.postView.as_view(), name='post_detail'),
    path('posts/', views.index.as_view(), name="posts"),
    path('posts/<int:year>/<int:month>/', views.index.as_view(), name='posts'),
    path('posts/<int:year>/', views.index.as_view(), name='posts'),

    path('tags/', views.TagsListView.as_view(), name='tags_list'),
    path('tags/<str:name>/', views.tag_detail_view, name='tag_detail'),

    path('', views.index.as_view(), name='index'),
]


# fDE

#mutation
a, b, c = pop[np.random.choice(idxs, 3, replace=False)]                       # random choice among popsize excluding current index
#selection
fitness[j] , target[j] = selection(f , fitness[j] , target[j] , trial , Fn)

# fNRAND1

#mutation
all_sum = []
for n in idxs:
    indv_sum =  sum((target[j][i] - target[n][i])**2  for i in range(D))
    all_sum.append(indv_sum)
ind = np.argmin(all_sum)
a = target[ind]                                                    # here only two position vector are randomly selected in nature
b, c = target[np.random.choice(idxs, 2, replace=False)]
#selection
fitness[j] , target[j] = selection(f , fitness[j] , target[j] , trial , Fn)

#fINRAND1

#mutation
all_sum = []

if (j == 0 or j == popsize-1):
    idxs = [idx for idx in range(popsize) if idx != j]

    for n in idxs:
        indv_sum =  sum((target[j][i] - target[n][i])**2  for i in range(D))
        all_sum.append(indv_sum)
    ind = np.argmin(all_sum)

else:

    for n in [j-1,j+1]:
        indv_sum =  sum((target[j][i] - target[n][i])**2  for i in range(D))
        all_sum.append(indv_sum)

    ind = np.where(np.argmin(all_sum) == 0 , j-1 , j+1)

a = target[ind]                                                    # three different vectors are generated.
b, c = target[np.random.choice(idxs, 2, replace=False)]            # here only two position vector are randomly selected in nature

#selection
fitness[j] , target[j] = selection(f , fitness[j] , target[j] , trial , Fn)

# fCDE

 # process to find the closest target vector for this iteration trial vector
all_sum = []
for n in np.arange(popsize):

    indv_sum = sum((trial[i] - target[n][i])**2  for i in range(D))
    all_sum.append(indv_sum)

ind  = np.argmin(all_sum)
x_n  = target[ind]
f_xn = fitness[ind]
# Selection                                                        # Constrain_implementation
fitness[j] , target[j] = selection(f , f_xn , x_n , trial , Fn)    # domination check between trial vector and closest vector

#fNCDE


def color_nodes(graph):
  graphs_nodes = sorted(list(graph.keys()), key=lambda x: len(graph[x]), reverse=True)
  color_dict = {}
  for node in graphs_nodes:
    graph_colors = [True] * len(graphs_nodes)
    for adjacent in graph[node]:
      if adjacent in color_dict:
        color = color_dict[adjacent]
        graph_colors[color] = False
    for color, available in enumerate(graph_colors):
      if available:
        color_dict[node] = color
        break
  return color_dict

nodes_alphabets=['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']

graph={}
for i in nodes_alphabets:
    graph[i]=[]
for i in nodes_alphabets:
    for j in nodes_alphabets:
        if i!=j:
            graph[i].append(j)
print(graph)
print("Colors : \n",color_nodes(graph))

UE20CS323 GTA/UNIT2 CODING ASSIGNMENTS/welsh_powell.py


import json
from joblib import Parallel, delayed
import os, sys
from _utils import load_clients, get_non_iid_clusters_topk
from utils import blockPrint, enablePrint
sys.path.append('/'.join(sys.path[0].split('/')[:-1]))

from tqdm import tqdm
from exp_generate_words import load_words

from FA_ import FA_cluster, std_out_err_redirect_tqdm, distance

DATA_PATH ="dataset/steps/"
CURR_DIR = os.path.dirname(os.path.abspath(__file__))

def FA_aggregation(clients: list, k: int, varepsilon: float = 2, m=64, iterations=32, stop_iter=-1):
    """_summary_

    Args:
        clients (list[list]): list of clusters of clients
        k (int): top k
        varepsilon (float, optional): LDP privacy parameter. Defaults to 2.
        m (int, optional): maximum length of binary strings. Defaults to 64.
        iterations (int, optional): number of iterations/trie's height. Defaults to 32.

    Returns:
        [list]: list of top k
    """

    clusters = len(clients)
    with std_out_err_redirect_tqdm() as orig_stdout:
        # tqdm needs the original stdout
        # and dynamic_ncols=True to autodetect console width
        results = Parallel(n_jobs=clusters)(delayed(FA_cluster)(
            clients[i], k, m=m, iterations=iterations, varepsilon=varepsilon, stop_iter=stop_iter ) for i in tqdm(range(clusters)))

    candidates_among_clusters = {}

    for i in range(len(results)):
        result_i = json.loads(results[i])
        predict_hh:dict = result_i['predict_hh']
        threshold = k if k < list(predict_hh.values())[0] else 0
        for hh in predict_hh:
            x = predict_hh[hh]
            if x <= threshold:
                continue
            incre = x/sum(predict_hh.values())
            candidates_among_clusters[hh] = candidates_among_clusters.get(hh, 0) + incre

        # aggregate results
    candidates_among_clusters = sorted(candidates_among_clusters.items(),
                           key=lambda x: x[1], reverse=True)

    candidates_n = len(candidates_among_clusters)

    good_hhs = []
    bad_hh = []
    for i in range(candidates_n):
        x = int(candidates_among_clusters[i][0])
        if x in bad_hh:
            continue
        for j in range(i+1, candidates_n):

            y = int(candidates_among_clusters[j][0])
            threshold = (min(len(bin(x)), len(bin(y))) - 2)/2
            dis = distance(x, y)
            if dis <= threshold and candidates_n - len(bad_hh) > k:
                bad_hh.append(y)
                continue
            # print(f"{x} {y}: distance = {dis}")
        good_hhs.append(x)
        if len(good_hhs) == k: break
    return good_hhs

def load_clusters(k=5):

    clusters_ = []

    for n in range(5):
        filename = f"week_{n}"
        cluster = list(map(int,load_words(f"{DATA_PATH}{filename}.txt")))

        clusters_.append(cluster)

    return clusters_

if __name__ == '__main__':

    m = 48
    k = 1

    varepsilon = 2.5

    iterations = 24
    stop_iter = 18

    runs = 20

    clients = load_clusters(k)
    results = {}

    cumulate = 0
    enablePrint()
    for rnd in range(runs):
        cumulate += FA_aggregation(clients, k, varepsilon=varepsilon,\
             m=m, iterations=iterations, stop_iter=stop_iter)[0]
    print(cumulate/runs)

# with open(f"{CURR_DIR}/steps.json", "w") as f:
#     json.dump(results, f)


from os.path import join
import uuid

import numpy as np

from rastervision2.pipeline.file_system import (make_dir)
from rastervision2.core.data.label import SemanticSegmentationLabels
from rastervision2.core.utils.misc import save_img
from rastervision2.core.data_sample import DataSample
from rastervision2.pytorch_backend.pytorch_learner_backend import (
    PyTorchLearnerSampleWriter, PyTorchLearnerBackend)

class PyTorchSemanticSegmentationSampleWriter(PyTorchLearnerSampleWriter):
    def write_sample(self, sample: DataSample):
        """
        This writes a training or validation sample to
        (train|valid)/img/{scene_id}-{ind}.png and
        (train|valid)/labels/{scene_id}-{ind}.png
        """
        split_name = 'train' if sample.is_train else 'valid'
        label_arr = sample.labels.get_label_arr(sample.window).astype(np.uint8)

        img_dir = join(self.sample_dir, split_name, 'img')
        labels_dir = join(self.sample_dir, split_name, 'labels')
        make_dir(img_dir)
        make_dir(labels_dir)

        img_path = join(img_dir, '{}-{}.png'.format(sample.scene_id,
                                                    self.sample_ind))
        labels_path = join(
            labels_dir, '{}-{}.png'.format(sample.scene_id, self.sample_ind))
        save_img(sample.chip, img_path)
        save_img(label_arr, labels_path)

        self.sample_ind += 1

class PyTorchSemanticSegmentation(PyTorchLearnerBackend):
    def get_sample_writer(self):
        output_uri = join(self.pipeline_cfg.chip_uri, '{}.zip'.format(
            str(uuid.uuid4())))
        return PyTorchSemanticSegmentationSampleWriter(
            output_uri, self.pipeline_cfg.dataset.class_config, self.tmp_dir)

    def predict(self, chips, windows):
        if self.learner is None:
            self.load_model()

        batch_out = self.learner.numpy_predict(chips, raw_out=False)
        labels = SemanticSegmentationLabels()
        for out, window in zip(batch_out, windows):
            labels.set_label_arr(window, out)

        return labels


#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
Date: 2020/7/17 17:06
Desc: 中国银行保险监督管理委员会-首页-政务信息-行政处罚-银保监分局本级-XXXX行政处罚信息公开表
http://www.cbirc.gov.cn/cn/view/pages/ItemList.html?itemPId=923&itemId=4115&itemUrl=ItemListRightList.html&itemName=%E9%93%B6%E4%BF%9D%E7%9B%91%E5%88%86%E5%B1%80%E6%9C%AC%E7%BA%A7&itemsubPId=931&itemsubPName=%E8%A1%8C%E6%94%BF%E5%A4%84%E7%BD%9A#2
提取 具体页面 html 页面的 json 接口
http://www.cbirc.gov.cn/cn/static/data/DocInfo/SelectByDocId/data_docId=881446.json
2020新接口
"""
import requests
import numpy as np
import pandas as pd

from mssdk.bank.cons import cbirc_headers_without_cookie_2020

item_id_list = {
    '机关': '4113',
    '本级': '4114',
    '分局本级': '4115',
}

def bank_fjcf_total_num(item: str = "分局本级") -> int:
    """
    首页-政务信息-行政处罚-银保监分局本级 总页数
    http://www.cbirc.gov.cn/cn/view/pages/ItemList.html?itemPId=923&itemId=4115&itemUrl=ItemListRightList.html&itemName=%E9%93%B6%E4%BF%9D%E7%9B%91%E5%88%86%E5%B1%80%E6%9C%AC%E7%BA%A7&itemsubPId=931
    :return: 总页数
    :rtype: int
    """
    cbirc_headers = cbirc_headers_without_cookie_2020.copy()
    main_url = "http://www.cbirc.gov.cn/cbircweb/DocInfo/SelectDocByItemIdAndChild"
    params = {
        "itemId": item_id_list[item],
        "pageSize": "18",
        "pageIndex": "1",
    }
    res = requests.get(main_url, params=params, headers=cbirc_headers)
    return int(res.json()["data"]["total"])

def bank_fjcf_total_page(item: str = "分局本级", begin: int = 1) -> int:
    """
    获取  首页-政务信息-行政处罚-银保监分局本级 总页数
    http://www.cbirc.gov.cn/cn/view/pages/ItemList.html?itemPId=923&itemId=4115&itemUrl=ItemListRightList.html&itemName=%E9%93%B6%E4%BF%9D%E7%9B%91%E5%88%86%E5%B1%80%E6%9C%AC%E7%BA%A7&itemsubPId=931
    :return: 总页数
    :rtype: int
    """
    cbirc_headers = cbirc_headers_without_cookie_2020.copy()
    main_url = "http://www.cbirc.gov.cn/cbircweb/DocInfo/SelectDocByItemIdAndChild"
    params = {
        "itemId": item_id_list[item],
        "pageSize": "18",
        "pageIndex": str(begin),
    }
    res = requests.get(main_url, params=params, headers=cbirc_headers)
    if res.json()["data"]["total"] / 18 > int(res.json()["data"]["total"] / 18):
        total_page = int(res.json()["data"]["total"] / 18) + 1
    return total_page

def bank_fjcf_page_url(page: int = 5, item: str = "分局本级", begin: int = 1) -> pd.DataFrame:
    """
    获取 首页-政务信息-行政处罚-银保监分局本级-每一页的 json 数据
    :param page: 需要获取前 page 页的内容, 总页数请通过 bank_fjcf_total_page() 获取
    :type page: int
    :return: 需要的字段
    :rtype: pandas.DataFrame
    """
    cbirc_headers = cbirc_headers_without_cookie_2020.copy()
    main_url = "http://www.cbirc.gov.cn/cbircweb/DocInfo/SelectDocByItemIdAndChild"
    temp_df = pd.DataFrame()
    for i_page in range(begin, page+begin):
        print(i_page)
        params = {
            "itemId": item_id_list[item],
            "pageSize": "18",
            "pageIndex": str(i_page),
        }
        res = requests.get(main_url, params=params, headers=cbirc_headers)
        temp_df = temp_df.append(pd.DataFrame(res.json()["data"]["rows"]))
    return temp_df[["docId", "docSubtitle", "publishDate", "docFileUrl", "docTitle", "generaltype"]]

def bank_fjcf_table_detail(page: int = 5, item: str = "分局本级", begin: int = 1) -> pd.DataFrame:
    """
    获取 首页-政务信息-行政处罚-银保监分局本级-XXXX行政处罚信息公开表 数据
    :param page: 需要获取前 page 页的内容, 总页数请通过 bank_fjcf_total_page() 获取
    :type page: int
    :return: 返回所有行政处罚信息公开表的集合, 按第一页到最后一页的顺序排列
    :rtype: pandas.DataFrame
    """
    id_list = bank_fjcf_page_url(page=page, item=item, begin=begin)["docId"]
    big_df = pd.DataFrame()
    for item in id_list:
        print(item)
        url = f"http://www.cbirc.gov.cn/cn/static/data/DocInfo/SelectByDocId/data_docId={item}.json"
        res = requests.get(url)
        # print(res.json()["data"]["docClob"])
        try:
            table_list = pd.read_html(res.json()["data"]["docClob"])[0]
            table_list = table_list.iloc[:, 3:].values.tolist()
            # 部分旧表缺少字段，所以填充
            if len(table_list) == 7:
                table_list.insert(2, np.nan)
                table_list.insert(3, np.nan)
                table_list.insert(4, np.nan)
            elif len(table_list) == 8:
                table_list.insert(1, np.nan)
                table_list.insert(2, np.nan)
            elif len(table_list) == 9:
                table_list.insert(2, np.nan)
            elif len(table_list) == 11:
                table_list = table_list[2:]
                table_list.insert(2, np.nan)
            # 部分会变成嵌套列表, 这里还原
            table_list = [item[0] if isinstance(
                item, list) else item for item in table_list]
            table_list.append(str(item))
            table_list.append(res.json()["data"]["publishDate"])
            table_df = pd.DataFrame(table_list)
            table_df.columns = ["内容"]
            big_df = big_df.append(table_df.T, ignore_index=True)
            # 解决有些页面缺少字段的问题, 都放到 try 里面
        except:
            print(f"{item} is not table, it will be skip")
            continue
    big_df.columns = [
        "行政处罚决定书文号",
        "姓名",
        "单位",  # 20200108新增
        "单位名称",
        "主要负责人姓名",
        "主要违法违规事实（案由）",
        "行政处罚依据",
        "行政处罚决定",
        "作出处罚决定的机关名称",
        "作出处罚决定的日期",
        "处罚ID",
        "处罚公布日期",
    ]
    return big_df

if __name__ == '__main__':
    bank_fjcf_table_detail_df = bank_fjcf_table_detail(page=2)
    print(bank_fjcf_table_detail_df)


# from crypten import mpc
import copy
from torch.utils.data import DataLoader
from tqdm import tqdm

import accuracy
import config
import posion_info
import tools
from Data import model_and_data
from client import create_model
from client import local_train
# from private_filter import fun_test
from filter import *
from util import *

def run(args, dev):
    # 加载并划分数据集
    train_dataset, test_dataset, train_iter_list, test_iter_list, poison_test_iter = model_and_data(args)
    train_dataloader = DataLoader(train_dataset, batch_size=args['batchsize'], shuffle=True, num_workers=4)
    test_dataloader = DataLoader(test_dataset, batch_size=args['batchsize'], shuffle=True, num_workers=4)
    # 实例化模型
    global_net = create_model(args).to(dev)
    # global_net = torch.nn.DataParallel(global_net)  # 多块gpu共同训练

    local_net = copy.deepcopy(global_net)

    global_weight = tools.shape_to_1dim(global_net, single=True)

    num_in_comm = int(max(args['num_of_clients'] * args['cfraction'], 1))  # 每轮选择聚合的客户端个数

    # 训练
    print("_________________START TRAIN________________________")
    main_acc_local, main_acc_global = [], []
    backdoor_acc = [[] for _ in range(5)]
    true_positive_rate_list = []
    true_negative_rate_list = []
    train_acc_global = []
    all_trigger_lst, trigger_of_acs = posion_info.creat_trigger(args)
    # epoch_under_attack = list(np.random.choice(range(args['num_comm']), int(args['attack_epoch_num']), replace=False))
    epoch_under_attack = list(range(args["attack_epoch_in"], args["num_comm"]))
    acs = list(range(args['num_of_attack']))

    for epoch in tqdm(range(args['num_comm']), desc='Training: ', colour='green'):
        # 挑选恶意参与方以及良性参与方
        if_ATTACK = True if epoch in epoch_under_attack else False
        benigns = list(
            np.random.choice(
                range(args['num_of_attack'], args['num_of_clients']),
                num_in_comm - args['num_of_attack'],
                replace=False
            )
        )
        local_choose = acs + benigns

        local_choose.sort()

        # 开始训练
        local_update_lts = []
        global_weight = tools.shape_to_1dim(global_net, single=True)
        for i in range(num_in_comm):
            local = local_choose[i]
            tools.hand_out(global_net, local_net)
            net, train_iter = local_net, train_iter_list[local]

            if local in acs and if_ATTACK:  # 毒化
                # INFO_LOG.logger.info("Attack")
                poison_info = trigger_of_acs[local]
                opt = torch.optim.SGD(
                    net.parameters(),
                    lr=poison_info['poison_lr'],
                    momentum=0.9,
                    weight_decay=args['weight_decay']
                )
                scheduler = torch.optim.lr_scheduler.MultiStepLR(
                    opt,
                    milestones=[0.2 * args['local_poison_epoch'], 0.8 * args['local_poison_epoch']],
                    gamma=0.1
                )
                local_train(
                    net, train_iter, opt, device=dev,
                    num_epochs=args['local_poison_epoch'],
                    poison_info=poison_info,
                    scheduler=scheduler,
                    dataset_name=args['dataset']
                )
            else:
                opt = torch.optim.SGD(
                    net.parameters(),
                    lr=args['learning_rate'], momentum=0.9,
                    weight_decay=args['weight_decay']
                )
                local_train(
                    net, train_iter, opt, device=dev,
                    num_epochs=args['epoch'],
                    dataset_name=args['dataset']
                )
            # local_update_lts.shape = (10, 2797610)
            local_update_lts.append(tools.shape_to_1dim(net, single=True))  # update full model

        if not if_ATTACK:

            admitted_index = local_choose
            # local_net_params.shape = (10, 62)
            local_net_params = tools.shape_back(local_update_lts, local_net)

            sum_params = np.array(local_net_params).sum(axis=0)
            for g_p, l_sum_p in zip(global_net.parameters(), sum_params):
                l_sum_p /= len(admitted_index)  # FedAvg
                l_sum_p = l_sum_p.to(dev)
                g_p.data = l_sum_p.data
        else:
            with torch.no_grad():
                # 模型过滤层
                print("clients index: ", local_choose)
                # print(local_update_lts)
                admitted_index_in_wlst = model_filtering_layer(local_update_lts, args)
                # print("cluster result:", admitted_index_in_wlst, hamming_index_in_wlst)

                admitted_index = []
                for i in admitted_index_in_wlst:
                    admitted_index.append(local_choose[i])
                admitted_index.sort()
                print("choose model: ", admitted_index)

                # 自适应裁剪
                if args['clip']:
                    local_update_lts, St = adaptive_clipping(global_weight, local_update_lts, admitted_index_in_wlst)

                # filter
                benign_w = []
                for i in admitted_index_in_wlst:
                    benign_w.append(local_update_lts[i])
                local_net_params = tools.shape_back(benign_w, local_net)

                # 聚合更新全局模型
                sum_params = np.array(local_net_params).sum(axis=0)
                # sum_params = local_net_params[0]
                # for i in range(1, len(local_net_params)):
                #     for s, l in zip(sum_params, local_net_params[i]):
                #         s.data += l.data

                for g_p, l_sum_p in zip(global_net.parameters(), sum_params):
                    l_sum_p /= len(admitted_index)  # FedAvg
                    l_sum_p = l_sum_p.to(dev)
                    g_p.data = l_sum_p.data

                # 自适应加噪
                if args['noise']:
                    alpha = args['λ'] * St
                    adaptive_noising(global_net, alpha)

        if not if_ATTACK:
            true_negative_rate, true_positive_rate = 1.0, 1.0
        else:
            true_positive_rate = get_true_positive_rate(admitted_index, acs)
            true_negative_rate = get_true_negative_rate(admitted_index, acs)

        true_positive_rate_list.append(true_positive_rate)
        true_negative_rate_list.append(true_negative_rate)

        INFO_LOG.logger.info(f'true_positive_rate: {true_positive_rate_list}')
        INFO_LOG.logger.info(f'true_negative_rate: {true_negative_rate_list}')

        # 全局模型对主任务的精度

        train_acc = accuracy.evaluate_accuracy(
            train_dataloader,
            global_net,
            dataset_name=args['dataset']
        )
        train_acc_global.append(train_acc)

        test_acc = accuracy.evaluate_accuracy(
            test_dataloader,
            global_net,
            dataset_name=args['dataset']
        )
        main_acc_global.append(test_acc)

        # 全局模型对后门任务的精度

        for i in range(5):
            acc = 0.0
            if if_ATTACK:
                acc = accuracy.evaluate_accuracy(
                    poison_test_iter,
                    global_net,
                    poison_info=all_trigger_lst[i],
                    dataset_name=args['dataset']
                )
            backdoor_acc[i].append(acc)

        # 记录每轮全局模型的后门精度
        INFO_LOG.logger.info(f'train_acc: {train_acc_global}')
        INFO_LOG.logger.info(f'main_take_acc: {main_acc_global}')
        INFO_LOG.logger.info(f'global_trigger_acc: {backdoor_acc[0]}')

    # save result

    save_path = f"tmp/flame_result/{args['IID_rate']}-{args['dataset']}-poison_rate-{trigger_of_acs[0]['poison_rate']}-{getNowTime()}-{args['learning_rate']}.pth"
    INFO_LOG.logger.info(f'save result in file:\n {save_path}')

    torch.save(
        [
            train_acc_global,
            main_acc_global,
            backdoor_acc,
            true_positive_rate_list,
            true_negative_rate_list
        ],
        save_path
    )

if __name__ == '__main__':

    parser = config.agg_f()
    args = parser.parse_args()
    args = args.__dict__
    INFO_LOG.logger.info(args)
    import os
    os.environ['CUDA_VISIBLE_DEVICES'] = args['gpu']
    dev = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    run(args, dev)
    INFO_LOG.logger.info(args)


from piglow import PiGlow
from time import sleep

piglow = PiGlow()

def pata(min, max):
    i = min
    while i <= max:
        if i > min:
            piglow.led(i - 1, 0)
        piglow.led(i, 1)
        sleep(0.3)
        i += 1

def estrella():
    piglow.all(1)
    sleep(0.5)
    i = 6
    j = 12
    k = 18

    while i >= 1:
        piglow.led(i, 0)
        piglow.led(j, 0)
        piglow.led(k, 0)
        sleep(0.3)

        i -= 1
        j -= 1
        k -= 1

try:
    while True:
        piglow.all(0)
        pata(1, 6)
        estrella()
        pata(7, 12)
        estrella()
        pata(13, 18)
        estrella()

except KeyboardInterrupt:
    piglow.all(0)


#!/usr/bin/python3
""" a python script that fetch all states """
from model_state import Base, State
from sys import argv
from sqlalchemy import (create_engine)
from sqlalchemy.orm import sessionmaker

def add_state():
    """ funtion that access database and print states """
    engine = create_engine('mysql+mysqldb://{}:{}@localhost/{}'.format(
        argv[1], argv[2], argv[3]), pool_pre_ping=True)
    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)
    session = Session()
    state = State(name="Louisiana")
    session.add(state)
    session.commit()
    print(state.id)
    session.close()

if __name__ == "__main__":
    add_state()


#
#
#      0===================================0
#      |    TP2 Iterative Closest Point    |
#      0===================================0
#
#
#------------------------------------------------------------------------------------------
#
#      Script of the practical session
#
#------------------------------------------------------------------------------------------
#
#      Hugues THOMAS - 17/01/2018
#

#------------------------------------------------------------------------------------------
#
#          Imports and global variables
#      \**********************************/
#

# Import numpy package and name it "np"
import numpy as np

# Import library to plot in python
from matplotlib import pyplot as plt

# Import functions from scikit-learn
from sklearn.neighbors import KDTree

# Import functions to read and write ply files
from ply import write_ply, read_ply
from visu import show_ICP

import sys

#------------------------------------------------------------------------------------------
#
#           Functions
#       \***************/
#
#
#   Here you can define usefull functions to be used in the main
#

def best_rigid_transform(data, ref):
    '''
    Computes the least-squares best-fit transform that maps corresponding points data to ref.
    Inputs :
        data = (d x N) matrix where "N" is the number of points and "d" the dimension
         ref = (d x N) matrix where "N" is the number of points and "d" the dimension
    Returns :
           R = (d x d) rotation matrix
           T = (d x 1) translation vector
           Such that R * data + T is aligned on ref
    '''

    # YOUR CODE
    #R = np.eye(data.shape[0])
    #T = np.zeros((data.shape[0],1))

    data_mean = data.mean(axis=1, keepdims=True)
    ref_mean = ref.mean(axis=1, keepdims=True)

    Q_data = data - data_mean
    Q_ref = ref - ref_mean
    H = Q_data @ Q_ref.T

    U, S, V = np.linalg.svd(H)

    R = V.T @ U.T
    if np.linalg.det(R) < 0:
        U[:,-1] *= -1
        R = V.T @ U.T

    T = ref_mean - R @ data_mean

    return R, T

def icp_point_to_point(data, ref, max_iter, RMS_threshold):
    '''
    Iterative closest point algorithm with a point to point strategy.
    Inputs :
        data = (d x N_data) matrix where "N_data" is the number of points and "d" the dimension
        ref = (d x N_ref) matrix where "N_ref" is the number of points and "d" the dimension
        max_iter = stop condition on the number of iterations
        RMS_threshold = stop condition on the distance
    Returns :
        data_aligned = data aligned on reference cloud
        R_list = list of the (d x d) rotation matrices found at each iteration
        T_list = list of the (d x 1) translation vectors found at each iteration
        neighbors_list = At each iteration, you search the nearest neighbors of each data point in
        the ref cloud and this obtain a (1 x N_data) array of indices. This is the list of those
        arrays at each iteration

    '''
    RMS = lambda data_, ref_: np.sqrt( np.mean( np.sum(np.power(data_ - ref_, 2), axis=0) ) )

    # Variable for aligned data
    data_aligned = np.copy(data)

    # Initiate lists
    d, n = data.shape
    R_list = [np.eye(d)]
    T_list = [np.zeros((d,1))]
    neighbors_list = []
    RMS_list = []

    kdtree = KDTree(ref.T)
    for i in range(max_iter):
        neighbors = kdtree.query(data_aligned.T, k=1, return_distance = False)
        neighbors = neighbors.squeeze()
        neighbors_list.append(neighbors)

        R, T = best_rigid_transform(data_aligned, ref[:,neighbors])
        data_aligned = R @ data_aligned + T

        T_list.append(R @ T_list[-1] + T)
        R_list.append(R @ R_list[-1])

        rms = RMS(data_aligned, ref[:,neighbors])
        RMS_list.append(rms)

        if rms < RMS_threshold:
            return data_aligned, R_list[1:], T_list[1:], neighbors_list, RMS_list

    return data_aligned, R_list[1:], T_list[1:], neighbors_list, RMS_list

def icp_point_to_point_fast(data, ref, max_iter, RMS_threshold, n_samples):
    '''
    Iterative closest point algorithm with a point to point strategy.
    Inputs :
        data = (d x N_data) matrix where "N_data" is the number of points and "d" the dimension
        ref = (d x N_ref) matrix where "N_ref" is the number of points and "d" the dimension
        max_iter = stop condition on the number of iterations
        RMS_threshold = stop condition on the distance
    Returns :
        data_aligned = data aligned on reference cloud
        R_list = list of the (d x d) rotation matrices found at each iteration
        T_list = list of the (d x 1) translation vectors found at each iteration
        neighbors_list = At each iteration, you search the nearest neighbors of each data point in
        the ref cloud and this obtain a (1 x N_data) array of indices. This is the list of those
        arrays at each iteration

    '''
    RMS = lambda data_, ref_: np.sqrt( np.mean( np.sum(np.power(data_ - ref_, 2), axis=0) ) )

    # Variable for aligned data
    data_aligned = np.copy(data)

    # Initiate lists
    d, n = data.shape
    R_list = [np.eye(d)]
    T_list = [np.zeros((d,1))]
    neighbors_list = []
    RMS_list = []

    kdtree = KDTree(ref.T)

    for i in range(max_iter):
        rand_idx = np.random.choice(n, n_samples, replace=False)
        neighbors = kdtree.query(data_aligned[:,rand_idx].T, k=1, return_distance = False)
        neighbors = neighbors.squeeze()
        R, T = best_rigid_transform(data_aligned[:,rand_idx], ref[:,neighbors])
        data_aligned = R @ data_aligned + T

        T_list.append(R @ T_list[-1] + T)
        R_list.append(R @ R_list[-1])

        rms = RMS(data_aligned[:,rand_idx], ref[:,neighbors])
        RMS_list.append(rms)

        if rms < RMS_threshold:
            return data_aligned, R_list[1:], T_list[1:], neighbors_list, RMS_list

    return data_aligned, R_list[1:], T_list[1:], neighbors_list, RMS_list

#------------------------------------------------------------------------------------------
#
#           Main
#       \**********/
#
#
#   Here you can define the instructions that are called when you execute this file
#

if __name__ == '__main__':

    # Transformation estimation
    # *************************
    #

    # If statement to skip this part if wanted
    if True:

        # Cloud paths
        bunny_o_path = '../data/bunny_original.ply'
        bunny_r_path = '../data/bunny_returned.ply'

		# Load clouds
        bunny_o_ply = read_ply(bunny_o_path)
        bunny_r_ply = read_ply(bunny_r_path)
        bunny_o = np.vstack((bunny_o_ply['x'], bunny_o_ply['y'], bunny_o_ply['z']))
        bunny_r = np.vstack((bunny_r_ply['x'], bunny_r_ply['y'], bunny_r_ply['z']))

        # Find the best transformation
        R, T = best_rigid_transform(bunny_r, bunny_o)

        # Apply the tranformation
        bunny_r_opt = R.dot(bunny_r) + T

        # Save cloud
        write_ply('../bunny_r_opt', [bunny_r_opt.T], ['x', 'y', 'z'])

        # Compute RMS
        distances2_before = np.sum(np.power(bunny_r - bunny_o, 2), axis=0)
        RMS_before = np.sqrt(np.mean(distances2_before))
        distances2_after = np.sum(np.power(bunny_r_opt - bunny_o, 2), axis=0)
        RMS_after = np.sqrt(np.mean(distances2_after))

        print('Average RMS between points :')
        print('Before = {:.3f}'.format(RMS_before))
        print(' After = {:.3f}'.format(RMS_after))
        print(RMS_after)

    # Test ICP and visualize
    # **********************
    #

    # If statement to skip this part if wanted
    if False:

        # Cloud paths
        ref2D_path = '../data/ref2D.ply'
        data2D_path = '../data/data2D.ply'

        # Load clouds
        ref2D_ply = read_ply(ref2D_path)
        data2D_ply = read_ply(data2D_path)
        ref2D = np.vstack((ref2D_ply['x'], ref2D_ply['y']))
        data2D = np.vstack((data2D_ply['x'], data2D_ply['y']))

        # Apply ICP
        data2D_opt, R_list, T_list, neighbors_list, RMS_list = icp_point_to_point(data2D, ref2D, 10, 1e-4)

        # Show ICP
        show_ICP(data2D, ref2D, R_list, T_list, neighbors_list)

        # Plot RMS
        plt.plot(RMS_list)
        plt.show()

    # If statement to skip this part if wanted
    if False:

        # Cloud paths
        bunny_o_path = '../data/bunny_original.ply'
        bunny_p_path = '../data/bunny_perturbed.ply'

        # Load clouds
        bunny_o_ply = read_ply(bunny_o_path)
        bunny_p_ply = read_ply(bunny_p_path)
        bunny_o = np.vstack((bunny_o_ply['x'], bunny_o_ply['y'], bunny_o_ply['z']))
        bunny_p = np.vstack((bunny_p_ply['x'], bunny_p_ply['y'], bunny_p_ply['z']))

        # Apply ICP
        bunny_p_opt, R_list, T_list, neighbors_list, RMS_list = icp_point_to_point(bunny_p, bunny_o, 25, 1e-4)

        # Show ICP
        show_ICP(bunny_p, bunny_o, R_list, T_list, neighbors_list)

        # Plot RMS
        plt.plot(RMS_list)
        plt.show()

    # If statement to skip this part if wanted
    if True:

        # Cloud paths
        bunny_o_path = '../data/Notre_Dame_Des_Champs_1.ply'
        bunny_p_path = '../data/Notre_Dame_Des_Champs_2.ply'

        # Load clouds
        bunny_o_ply = read_ply(bunny_o_path)
        bunny_p_ply = read_ply(bunny_p_path)
        bunny_o = np.vstack((bunny_o_ply['x'], bunny_o_ply['y'], bunny_o_ply['z']))
        bunny_p = np.vstack((bunny_p_ply['x'], bunny_p_ply['y'], bunny_p_ply['z']))
        n_samples = [1000, 10000, 50000, 100000]
        rms_list = []
        for n in n_samples:
        # Apply ICP
            bunny_p_opt, R_list, T_list, neighbors_list, RMS_list = icp_point_to_point_fast(bunny_p, bunny_o, 100, 1e-4, 10000)
            rms_list.append(RMS_list)
        #write_ply('../Notre_Dame_Des_Champs_ICP', [bunny_p_opt.T], ['x', 'y', 'z'])
        # Show ICP
        #show_ICP(bunny_p, bunny_o, R_list, T_list, neighbors_list)

        # Plot RMS
        for sl, rms in zip(n_samples, rms_list):
            plt.plot(rms, label="{} samples".format(sl))
        plt.title("Evolution of the RMS at each step of fast ICP")
        plt.legend()
        plt.show()


import random
import numpy as np
import os
import matplotlib.pyplot as plt
import keras
import csv
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import SGD
from keras.losses import categorical_crossentropy
import keras.backend as k
from keras.layers import Dropout, Flatten, Dense,Input
from keras.preprocessing.image import ImageDataGenerator
from sklearn.utils import shuffle
from keras import optimizers
from keras.callbacks import ModelCheckpoint
from keras.applications.imagenet_utils import preprocess_input

all=[]
train=[]
test=[]
details={1:{'circle':[],'line':[],'square':[]},
         2:{'circle':[],'line':[],'square':[]},
         3:{'circle':[],'line':[],'square':[]},
         4:{'circle':[],'line':[],'square':[]},
         5:{'circle':[],'line':[],'square':[]}}
for r,d,f in os.walk('all'):
    for file in f:
        index=file.index('level')
        level=int(file[index+5])
        level_class='class'+file[index+5]
        all.append([file,level_class])
        if 'circle' in file:
            kind='circle'
        elif 'line' in file:
            kind='line'
        elif 'square' in file:
            kind='square'
        details[level][kind].append([file,level_class])

for level in [1,2,3,4,5]:
    for kind in ['circle','line','square']:
        test.extend(details[level][kind][:100])
        train.extend(details[level][kind][100:])

with open('all.csv', newline='',mode='w') as csvfile:
    csvwriter=csv.writer(csvfile)
    csvwriter.writerow(['filename','class'])
    for row in all:
        csvwriter.writerow(row)

with open('train.csv', newline='',mode='w') as csvfile:
    csvwriter=csv.writer(csvfile)
    csvwriter.writerow(['filename','class'])
    for row in train:
        csvwriter.writerow(row)

with open('test.csv', newline='',mode='w') as csvfile:
    csvwriter=csv.writer(csvfile)
    csvwriter.writerow(['filename','class'])
    for row in test:
        csvwriter.writerow(row)

shape=(480,640,3)
#Here we set the data generators for applying data augmentation methods
train_datagen = ImageDataGenerator(horizontal_flip=True,vertical_flip=True,zoom_range=0.05,rotation_range=360,width_shift_range=0.05,height_shift_range=0.05,shear_range=0.05,preprocessing_function=preprocess_input)
test_datagen = ImageDataGenerator(preprocessing_function=preprocess_input)
train_df =pd.read_csv('train.csv') #raed train csv file
train_df = shuffle(train_df) #Shuffle the train data
test_df =pd.read_csv('test.csv') #raed train csv file

#Create the generators
train_generator = train_datagen.flow_from_dataframe(
      dataframe=train_df,
      directory='all',
      x_col="filename",
      y_col="class",
      target_size=shape[:2],
      batch_size=30,
      class_mode='categorical',shuffle=True,)

test_generator = test_datagen.flow_from_dataframe(
        dataframe=test_df,
      directory='all',
      x_col="filename",
      y_col="class",
      target_size=shape[:2],
      batch_size=30,
      class_mode='categorical',shuffle=True)

input_tensor=Input(shape=shape)
k.clear_session() #Clear keras backend
try:
  os.mkdir('models') #create folder for saving the trained networks
except:
  pass
model=keras.applications.MobileNet(input_shape=None,include_top=True,weights=None,input_tensor=input_tensor,classes=5)
model.summary()
model.compile(optimizer=optimizers.Nadam(lr=0.0001), loss='categorical_crossentropy',metrics=['accuracy'])
filepath="models/model-{epoch:02d}-{val_accuracy:.4f}.hdf5" # Path to save the trained models
checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', save_best_only=True, mode='max') #creating checkpoint to save the best validation accuracy
callbacks_list = [checkpoint]
model.fit_generator(train_generator, epochs=20,validation_data=test_generator,shuffle=True,callbacks=callbacks_list)