import os
from os.path import join
import sys
import json
import numpy as np
# from .read_openpose import read_openpose
import utils.segms as segm_utils

def db_coco_extract(dataset_path, subset, out_path):

    # convert joints to global order
    joints_idx = [19, 20, 21, 22, 23, 9, 8, 10, 7, 11, 6, 3, 2, 4, 1, 5, 0]

    # bbox expansion factor
    scaleFactor = 1.2

    # structs we need
    imgnames_, scales_, centers_, parts_, smpl_2dkps_, dp_annot_ = [], [], [], [], [], []
    im_id_, id_ = [], []

    # subfolders for different subsets
    subfolders = {'train': 'train2014', 'minival': 'val2014', 'valminusminival': 'val2014', 'test': 'test2014'}

    # json annotation file
    json_path = os.path.join(dataset_path,
                             'annotations',
                             'densepose_coco_2014_{}.json'.format(subset))
    json_data = json.load(open(json_path, 'r'))

    imgs = {}
    for img in json_data['images']:
        imgs[img['id']] = img

    has_dp_count = 0
    no_dp_count = 0
    for annot in json_data['annotations']:
        im_id, id = annot['image_id'], annot['id']
        if 'dp_masks' not in annot.keys():
            # print('dp_masks not in annot')
            no_dp_count += 1
            continue
        # keypoints processing
        keypoints = annot['keypoints']
        keypoints = np.reshape(keypoints, (17, 3))
        keypoints[keypoints[:, 2] > 0, 2] = 1
        # if sum(keypoints[5:, 2] > 0) < 12:
        #     no_dp_count += 1
        #     continue
        has_dp_count += 1

        # check if all major body joints are annotated
        # if sum(keypoints[5:,2]>0) < 12:
        #     continue
        # create smpl joints from coco keypoints
        smpl_2dkp = kp_coco2smpl(keypoints.copy())
        # image name
        image_id = annot['image_id']
        img_name = str(imgs[image_id]['file_name'])
        img_name_full = join(subfolders[subset], img_name)
        # keypoints
        part = np.zeros([24, 3])
        part[joints_idx] = keypoints
        # scale and center
        bbox = annot['bbox']
        center = [bbox[0] + bbox[2]/2, bbox[1] + bbox[3]/2]
        scale = scaleFactor*max(bbox[2], bbox[3])/200

        dp_annot = {'bbox': annot['bbox'],
                    'dp_x': annot['dp_x'],
                    'dp_y': annot['dp_y'],
                    'dp_I': annot['dp_I'],
                    'dp_U': annot['dp_U'],
                    'dp_V': annot['dp_V'],
                    'dp_masks': annot['dp_masks']
                    }

        # store data
        imgnames_.append(img_name_full)
        centers_.append(center)
        scales_.append(scale)
        parts_.append(part)
        smpl_2dkps_.append(smpl_2dkp)
        dp_annot_.append(dp_annot)
        im_id_.append(im_id)
        id_.append(id)

    print('# samples with dp: {}; # samples without dp: {}'.format(has_dp_count, no_dp_count))

    # store the data struct
    if not os.path.isdir(out_path):
        os.makedirs(out_path)
    out_file = os.path.join(out_path, 'dp_coco_2014_{}.npz'.format(subset))
    np.savez(out_file, imgname=imgnames_,
                       center=centers_,
                       scale=scales_,
                       part=parts_,
                       smpl_2dkps=smpl_2dkps_,
                       dp_annot=dp_annot_,
                       im_id=im_id_,
                       id=id_)

def kp_coco2smpl(kps_coco):
    smpl2coco = [[1,  2,  4,  5,  7,  8,  16, 17, 18, 19, 20, 21],
                 [11, 12, 13, 14, 15, 16, 5,  6,  7,  8,  9,  10]]

    kps_smpl = np.zeros((24, 4))

    kps_smpl[smpl2coco[0], :2] = kps_coco[smpl2coco[1], :2]
    kps_smpl[smpl2coco[0], 3] = kps_coco[smpl2coco[1], 2] / 2.

    if all(kps_coco[[11, 12], 2] > 0):
        kps_smpl[0, :2] = np.mean(kps_coco[[11, 12], :2], axis=0)
        kps_smpl[0, 3] = 0.5

    if all(kps_coco[[5, 6], 2] > 0):
        kps_smpl[12, :2] = np.mean(kps_coco[[5, 6], :2], axis=0)
        kps_smpl[12, 3] = 0.5

    if kps_smpl[12, 3] > 0 and kps_coco[0, 2] > 0:
        kps_smpl[15, :2] = (kps_smpl[12, :2] + kps_coco[0, :2]) / 2.
        kps_smpl[15, 3] = 0.5

    if kps_smpl[0, 3] > 0 and kps_smpl[12, 3] > 0:
        kps_smpl[6, :2] = np.mean(kps_smpl[[0, 12], :2], axis=0)
        kps_smpl[9, :2] = kps_smpl[6, :2]
        kps_smpl[6, 3] = 0.5
        kps_smpl[9, 3] = 0.5

    if kps_smpl[0, 3] > 0 and kps_smpl[6, 3] > 0:
        kps_smpl[3, :2] = np.mean(kps_smpl[[0, 6], :2], axis=0)
        kps_smpl[3, 3] = 0.5

    if kps_smpl[9, 3] > 0 and kps_smpl[16, 3] > 0:
        kps_smpl[13, :2] = np.mean(kps_smpl[[9, 16], :2], axis=0)
        kps_smpl[13, 3] = 0.5

    if kps_smpl[9, 3] > 0 and kps_smpl[17, 3] > 0:
        kps_smpl[14, :2] = np.mean(kps_smpl[[9, 17], :2], axis=0)
        kps_smpl[14, 3] = 0.5

    hand_foot = [[7, 8, 20, 21], [10, 11, 22, 23]]
    for i in range(4):
        if kps_smpl[hand_foot[0][i], 3] > 0:
            kps_smpl[hand_foot[1][i], :2] = kps_smpl[hand_foot[0][i], :2]
            kps_smpl[hand_foot[1][i], 3] = 0.5

    kps_smpl[:, 2] = kps_smpl[:, 3]

    return kps_smpl[:, :3].copy()

if __name__ == '__main__':
    import path_config as cfg
    db_coco_extract(cfg.COCO_ROOT, 'train', 'notebooks/output/extras')
    # db_coco_extract(cfg.COCO_ROOT, 'minival', 'notebooks/output/extras')


# coding: utf-8
# 前端测试是否眨眼，用于活体检测 Front-end test blinks for biopsy
from scipy.spatial import distance as dist
from imutils import face_utils
import time
import dlib
import cv2

def eye_aspect_ratio(eye):
    # 计算两只眼睛之间的垂直欧式距离
    A = dist.euclidean(eye[1], eye[5])
    B = dist.euclidean(eye[2], eye[4])

    # 计算两眼之间的水平欧式距离
    C = dist.euclidean(eye[0], eye[3])

    # 计算眼睛纵横比
    ear = (A + B) / (2.0 * C)

    # return the eye aspect ratio
    return ear

def main():
    shape_predictor = "shape_predictor_68_face_landmarks.dat"
    EYE_AR_THRESH = 0.27  # 阈值
    EYE_AR_CONSEC_FRAMES = 33   # the number of consecutive frames the eye must be below the threshold

    # initialize the frame counters and the total number of blinks
    COUNTER = 0
    TOTAL = 0

    # initialize dlib's face detector (HOG-based) and then create
    # the facial landmark predictor
    print("[INFO] loading facial landmark predictor...")
    detector = dlib.get_frontal_face_detector()
    predictor = dlib.shape_predictor(shape_predictor)

    # grab the indexes of the facial landmarks for the left and right eye, respectively
    (lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
    (rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]

    # start the video stream thread
    print("[INFO] starting video stream thread...")
    print("[INFO] print q to quit...")

    vs = cv2.VideoCapture(1)
    time.sleep(1.0)

    # loop over frames from the video stream
    while True:
        # grab the frame from the threaded video file stream, resize it, and convert it to grayscale channels)
        _, frame = vs.read()
        frame = cv2.resize(frame, (0, 0), fx=0.75,fy=0.75)
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

        # detect faces in the grayscale frame
        rects = detector(gray, 0)

        # loop over the face detections
        for rect in rects:
            # determine the facial landmarks for the face region, then
            # convert the facial landmark (x, y)-coordinates to a NumPy array
            shape = predictor(gray, rect)
            shape = face_utils.shape_to_np(shape)

            # extract the left and right eye coordinates, then use the
            # coordinates to compute the eye aspect ratio for both eyes
            leftEye = shape[lStart:lEnd]
            rightEye = shape[rStart:rEnd]
            leftEAR = eye_aspect_ratio(leftEye)
            rightEAR = eye_aspect_ratio(rightEye)

            # average the eye aspect ratio together for both eyes
            ear = (leftEAR + rightEAR) / 2.0

            # compute the convex hull for the left and right eye, then visualize each of the eyes
            leftEyeHull = cv2.convexHull(leftEye)
            rightEyeHull = cv2.convexHull(rightEye)
            cv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1)
            cv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1)

            # check to see if the eye aspect ratio is below the blink
            # threshold, and if so, increment the blink frame counter
            if ear < EYE_AR_THRESH:
                COUNTER += 1

            # otherwise, the eye aspect ratio is not below the blink threshold
            else:
                # if the eyes were closed for a sufficient number of then increment the total number of blinks
                if COUNTER >= EYE_AR_CONSEC_FRAMES:
                    return 1

                # reset the eye frame counter
                COUNTER = 0

            # draw the total number of blinks on the frame along with the computed eye aspect ratio for the frame
            cv2.putText(frame, "Blinks: {}".format(TOTAL), (10, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
            cv2.putText(frame, "EAR: {:.2f}".format(ear), (300, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)

        cv2.imshow("Frame", frame)
        key = cv2.waitKey(1) & 0xFF

        # if the `q` key was pressed, break from the loop
        if key == ord("q"):
            break

    # do a bit of cleanup
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
'''
This file is part of Habitam.

Habitam is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of
the License, or (at your option) any later version.

Habitam is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public
License along with Habitam. If not, see
<http://www.gnu.org/licenses/>.

Created on Jul 21, 2013

@author: Stefan Guna
'''
from habitam.downloads.common import habitam_brand, signatures, MARGIN
from reportlab.lib import colors
from reportlab.lib.pagesizes import A4, landscape
from reportlab.lib.styles import getSampleStyleSheet
from reportlab.lib.units import cm
from reportlab.platypus.doctemplate import SimpleDocTemplate
from reportlab.platypus.flowables import Spacer
from reportlab.platypus.paragraph import Paragraph
from reportlab.platypus.tables import Table, TableStyle
import logging
import tempfile

logger = logging.getLogger(__name__)

__HEIGHT__ = A4[0]
__WIDTH__ = A4[1]

def __assets(building, day):
    result = {}

    # sold in casa si in banca
    for mt in ['cash', 'bank']:
        is_mt = lambda ac: ac.money_type == mt and ac.type == 'std'
        result[mt] = building.balance_by_type(is_mt, day)

    aps = building.apartments()
    ap_pending = 0
    penalties_pending = 0
    for ap in aps:
        ap_pending = ap_pending + ap.balance(day)
        tmp = ap.penalties(day)
        if tmp != None:
            penalties_pending = penalties_pending + ap.penalties(day)

    # lista curenta
    result['apartment_pending'] = ap_pending * -1
    # restante ale locatarilor
    result['penalties_pending'] = penalties_pending * -1

    # facturi de la furnizori si prestatori care urmeaza a fi repartizate pe liste.
    # everything is automatically distributed to the list
    result['outstanding_invoices'] = 0

    return result

def __liabilities(building, day):
    result = {}
    # fond rulment
    # fond de reparatii
    # fond de penalizari
    # fonduri speciale
    # facturi de la terti
    for t in ['rulment', 'repairs', 'penalties', 'special', '3rd party']:
        is_t = lambda ac:ac.type == t
        result[t] = building.balance_by_type(is_t, day)

    return result

def __balance_format(canvas, doc):
    canvas.saveState()
    canvas.setFontSize(16)
    t = u'Situația soldurilor elementelor de activ și pasiv pentru %s la %s'
    canvas.drawCentredString(__WIDTH__ / 2.0, __HEIGHT__ - 100,
                             t % (doc.habitam_data['building'].name, doc.habitam_data['day']))
    habitam_brand(canvas, __WIDTH__, __HEIGHT__)
    canvas.restoreState()

def __format_data(data):
    styles = getSampleStyleSheet()

    assets = data['assets']
    liabilities = data['liabilities']

    d = [['NR.\nCRT.', 'ELEMENTE DE ACTIV', 'VALORI\n(LEI)', 'ELEMENTE DE PASIV', 'VALORI\n(LEI)'],
         ['1.', Paragraph(u'Sold în casă', styles['Normal']), assets['cash'], Paragraph('Sold fond de rulment', styles['Normal']), liabilities['rulment']],
         ['2.', Paragraph(u'Sold conturi la bănci', styles['Normal']), assets['bank'], Paragraph(u'Sold fond de reparații', styles['Normal']), liabilities['repairs']],
         ['3.', Paragraph(u'Sume neachitate de proprietarii din asociație pentru lista de plată curentă', styles['Normal']), assets['apartment_pending'], Paragraph('Sold fond sume speciale', styles['Normal']), liabilities['special']],
         ['4.', Paragraph(u'Restanțe existente la data întocmirii acestei situații', styles['Normal']), assets['penalties_pending'], Paragraph('Soldul altor fonduri legal stabilite', styles['Normal']), '0'],
         ['5.', Paragraph(u'Debitori, alții decât mebrii asociației', styles['Normal']), '0', Paragraph('Furnizori pentru facturi neachitate', styles['Normal']), '0'],
         ['6.', Paragraph(u'Acte de plată pe luna în curs, nerepartizate proprietarilor', styles['Normal']), assets['outstanding_invoices'], Paragraph(u'Creditori diverși', styles['Normal']), liabilities['3rd party']],
         ['7.', Paragraph(u'Acte de plăți pentru cheltuielile aferente fondurilor de reparații, speciale, de penalizări care nu au fost încă scăzute din fondurile respective', styles['Normal']), '0', '', ''],
         ['', Paragraph(u'TOTAL PARTEA I', styles['Normal']), sum(assets.values()), Paragraph(u'TOTAL PARTEA II', styles['Normal']), sum(liabilities.values())]
        ]

    table = Table(d, colWidths=[1.3 * cm, 7 * cm, 4 * cm, 7 * cm, 4 * cm])
    table.setStyle(TableStyle([
                        ('FONT', (0, 0), (-1, 0), 'Helvetica-Bold'),
                        ('ALIGN', (0, 0), (-1, 0), 'CENTER'),
                        ('VALIGN', (0, 0), (-1, 0), 'MIDDLE'),
                        ('ALIGN', (0, 0), (0, -1), 'CENTER'),
                        ('VALIGN', (0, 0), (0, -1), 'MIDDLE'),
                        ('ALIGN', (2, 0), (2, -1), 'CENTER'),
                        ('VALIGN', (2, 0), (2, -1), 'MIDDLE'),
                        ('ALIGN', (4, 0), (4, -1), 'CENTER'),
                        ('VALIGN', (4, 0), (4, -1), 'MIDDLE'),
                        ('INNERGRID', (0, 0), (-1, -1), 0.25, colors.black),
                        ('BOX', (0, 0), (-1, -1), 0.25, colors.black)
                        ]))
    return table

def __to_pdf(temp, data):
    doc = SimpleDocTemplate(temp, pagesize=landscape(A4), leftMargin=MARGIN,
                            rightMargin=MARGIN, topMargin=MARGIN,
                            bottomMargin=MARGIN,
                            title=u'Situația activ/pasiv pentru %s' % data['building'].name,
                            author='www.habitam.ro')

    flowables = [Spacer(1, 6 * cm), __format_data(data), Spacer(1, cm), signatures()]
    doc.habitam_data = data
    doc.build(flowables, onFirstPage=__balance_format, onLaterPages=__balance_format)

def download_balance(building, day):
    data = {'building': building, 'day': day,
            'assets': __assets(building, day),
            'liabilities': __liabilities(building, day)}
    logger.debug('Balance is %s' % data)

    temp = tempfile.NamedTemporaryFile()
    __to_pdf(temp, data)
    return temp


from report import report_sxw
from osv import osv
import pooler
import time
from datetime import datetime
from dateutil.relativedelta import relativedelta
from tools.translate import _

class report_attendance_parser(report_sxw.rml_parse):
    def __init__(self, cr, uid, name, context):
        super(report_attendance_parser, self).__init__(cr, uid, name, context=context)
        self.localcontext.update({
            'get_employee':self._get_employee,
            'get_work_time':self._get_work_time,
        })

    def _get_employee(self,employee_ids):
        if employee_ids:
            return self.pool.get('hr.employee').browse(self.cr,self.uid,employee_ids)
        else:
            return False

    def _get_work_time(self,employee_id,date_print):
        sign_in='%s 00:00:00'%date_print
        sign_out='%s 23:59:59'%date_print

        if employee_id:
            employee=[]
            attendance_ids=self.pool.get('hr.attendance').search(self.cr,self.uid,[('employee_id','=',employee_id),('name','>=',sign_in),('name','<=',sign_out)],order="employee_id,name ASC")
            #print 'attendance_ids==================',attendance_ids
            attendance_datas=self.pool.get('hr.attendance').browse(self.cr,self.uid,attendance_ids)
            dummy={
                   'sign_in': False,
                   'sign_out':False,
                   }
            for y in attendance_datas:
                dummy={
                       'sign_in':dummy['sign_in']==False and y.action == 'sign_in' and y.name,
                       'sign_out':dummy['sign_out']==False and y.action == 'sign_out' and y.name,
                       }
                #print "===========>",dummy
                if dummy['sign_in'] and dummy['sign_out']:
                    employee.append(dummy)
                    dummy={
                           'sign_in': False,
                           'sign_out':False,
                           }
            return employee
        else:
            return False

report_sxw.report_sxw('report.report.hr.attendance', 'hr.attendance.report.wizard', 'addons/ad_hr_report/report/attendance_report.mako', parser = report_attendance_parser, header = True)


import os
import shutil
from pathlib import Path
from os import system
from shutil import rmtree
import shutil

mi_ruta = Path(Path.home(), '\Programacion-Cursos-Desarrollador\Python\Python-proyecto1\Dia7\Banco')

class Persona:

    def __init__(self, nombre, apellido):
        self.nombre = nombre
        self.apellido = apellido

class Cliente(Persona):

    def __init__(self, nombre, apellido, numero_cuenta, balance = 0):
        super().__init__(nombre, apellido) # heredado de class Persona
        self.numero_cuenta = numero_cuenta
        self.balance = balance

        ruta_balance = Path(mi_ruta, 'BalanceCuenta.txt')
        BalanceCuentaLeer = open(ruta_balance)
        contenidoBalanceCuentaLeer = BalanceCuentaLeer.read()
        # print("Prueba balance print: ", contenidoBalanceCuentaLeer)
        # print("Tipo de dato: ", type(contenidoBalanceCuentaLeer))
        balance_int = int(contenidoBalanceCuentaLeer)
        # print("Tipo de dato 2: ", type(contenidoBalanceCuentaLeer))
        self.balance = balance_int
        # print(type(self.balance))
        BalanceCuentaLeer.close()

    def __str__(self):
        return f"Cliente: { self.nombre } { self.apellido }\nNumero de Cuenta: { self.numero_cuenta }\nBalance: ${ self.balance }"

    def depositar(self, monto_deposito):

        self.balance += monto_deposito

        ruta_depositar = Path(mi_ruta, 'RegistroTransacciones.txt')
        ruta_balance = Path(mi_ruta, 'BalanceCuenta.txt')

        archivoRegistroTransacciones = open(ruta_depositar, "a")
        archivoRegistroTransacciones.write(f"{ monto_deposito } - INGRESO\n")
        archivoRegistroTransacciones.close()

        # Path.write_text(ruta_depositar, str(monto_deposito)) error, ya que no haria salto de linea \n
        Path.write_text(ruta_balance, str(self.balance))
        print("Deposito Aceptado")

    def retirar(self, monto_retiro):
        if self.balance >= monto_retiro:

            self.balance -= monto_retiro

            ruta_depositar = Path(mi_ruta, 'RegistroTransacciones.txt')
            ruta_balance = Path(mi_ruta, 'BalanceCuenta.txt')

            archivoRegistroTransacciones = open(ruta_depositar, "a")
            archivoRegistroTransacciones.write(f"{monto_retiro} - RETIRO\n")
            archivoRegistroTransacciones.close()

            Path.write_text(ruta_balance, str(self.balance))

            print("Retiro Realizado")
        else:
            print("Fondos Insuficientes Para el Retiro")

def crear_cliente():
    nombre_cl = input("Ingrese su Nombre: ")
    apellido_cl = input("Ingrese su Apellido: ")
    numero_cuenta = input("Ingrese Su Numero de Cuenta: ")
    cliente = Cliente(nombre_cl, apellido_cl, numero_cuenta) # class Cliente

    return cliente

def inicio():
    mi_cliente = crear_cliente()
    print(mi_cliente) # f"Cliente: { self.nombre } { self.apellido }\nNumero de Cuenta: { self.numero_cuenta }\nBalance: { self.balance }"
    opcion = 0

    while opcion != 'S': # 'S' salir
        print('Elija: Depositar (D), Retirar (R), Salir (S)')
        opcion = input()

        if opcion == 'D':
            monto_dep = int(input("Monto a Depositar: $"))
            mi_cliente.depositar(monto_dep)
        elif opcion == 'R':
            monto_dep = int(input("Monto a Retirar: $"))
            mi_cliente.retirar(monto_dep)
        print(mi_cliente) # mostrar como queda cliente despues de esas operaciones

    print("Gracias Por Su Visita - Hasta Luego")

inicio()

# proyecto: https://www.udemy.com/course/python-total/learn/lecture/28747292#questions
# convertir str a int: https://www.freecodecamp.org/news/python-convert-string-to-int-how-to-cast-a-string-in-python/#:~:text=To%20convert%2C%20or%20cast%2C%20a,int(%22str%22)%20.
# convertir int a str: https://es.stackoverflow.com/questions/364966/como-convertir-un-dato-int-a-string-en-python

Dia7/ProyectoDelDia7 - mejorada cuenta bancaria.py


import os
import re
import openai
from dotenv import load_dotenv

load_dotenv()
openai.api_key = os.environ['OPENAI_API_KEY']

def request_chatgpt(messages: list):
    return openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=messages,
    )

def analyze_toots(mastodon_timelines: list) -> tuple[list[int], str]:
    message_content = ''.join([f"id={toot.id} - {re.sub(re.compile('<.*?>'), '', toot.content)}\n" for toot in mastodon_timelines])

    response = request_chatgpt(
        messages=[
            {"role": "user", "content": message_content},
            {
                "role": "system",
                "content": """
                    You are a professional analyst.
                    Please output `id list of statements about programming` based on the following constraints and input statements.
                    # Constraints
                     - Output no more than 20 IDs.
                     - Use this format for output : `{id}, {id}, {id}, {id}, {id}`
                    I'll send you the input data.
                """,
            },
        ]
    )["choices"][0]["message"]["content"]

    relevant_toot_ids = [int(re.sub(r"\D", "", id)) for id in response.split(',') if re.search(r"\d", id)]

    return relevant_toot_ids, response


def count_substrings(string1, string2):
    ''' Get the number of subsctrings in two strings
    '''
    answer = 0
    for i in range(len(string1)):
        result = ''
        for j in range(i, len(string1)):
            result += string1[j]
            if string2.find(result) != -1:
                answer += 1
    return answer
print(count_substrings('aab', 'aaaab'))
print(count_substrings('duke', 'duke lester is here'))


#!/usr/bin/env python3

names= ["thiago",
        "joao",
        "rafael",
        "Rafaela",
        "ronaldo",
        "joana"]

## estilo funcional
print(*list(filter(lambda nome: nome[0].lower() == "r", names)),sep="\n")
print()

## estilo imperativo
def comeca_b(texto):
    return texto[0].lower() == "r"

filtro= filter(comeca_b,names)
filtro= list(filtro)
for i in filtro:
    print(i)


from collections import Counter

def get_hints(word: str, secret_word: str):
    word = word.lower()

    result = [""] * len(word)
    missing_indexes = []
    secrect_counter = Counter(secret_word)

    for idx, c in enumerate(word):
        if c == secret_word[idx]:
            result[idx] = "green"
            secrect_counter[c] -= 1
        else:
            missing_indexes.append(idx)

    for idx in missing_indexes:
        c = word[idx]
        if c in secrect_counter and secrect_counter[c] > 0:
            result[idx] = "yellow"
            secrect_counter[c] -= 1
        else:
            result[idx] = "black"

    if word == secret_word:
        return result
    else:
        return result


import sys
import os
import time
import traceback
import pandas as pd
import seaborn as sns
import pydotplus
import matplotlib.pyplot as plt
import numpy as np
from sklearn.tree import export_graphviz

from sklearn import tree

from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn import datasets
from sklearn.utils import shuffle
from sklearn.metrics import mean_squared_error
from sklearn.metrics import confusion_matrix, classification_report

sys.path.append('../')

from SOL4Py.ZApplicationView import *
from SOL4Py.ZLabeledComboBox import *
from SOL4Py.ZPushButton      import *
from SOL4Py.ZVerticalPane    import *
from SOL4Py.ZTabbedWindow    import *
from SOL4Py.ZMLModel import *
from SOL4Py.ZScalableScrolledFigureView import *
from SOL4Py.ZScalableScrolledDecisionTreeView import*

Iris         = 0
Digits       = 1
Wine         = 2
BreastCancer = 3

############################################################
# Classifier Model class

class DecisionTreeClassifierModel(ZMLModel):

  ##
  # Constructor
  def __init__(self, dataset_id, mainv):
    super(DecisionTreeClassifierModel, self).__init__(dataset_id, mainv)

  def run(self):
    self.write("====================================")
    self._start(self.run.__name__)
    try:
      # 1 Load dataset
      self.load_dataset()

      # 2 Load or create model
      if self.trained():
        # 2.1 if trained, load a trained model pkl file
        self.load()
      else:
        # 2.2 else create a model, and train and save it
        self.build()
        self.train()
        self.save()

      # 3 Predict for test_data
      self.predict()

      # 4 Visualize the prediction
      self.visualize()

    except:
      traceback.print_exc()
    self._end(self.run.__name__ )

  def load_dataset(self):
    self._start(self.load_dataset.__name__ )

    if self.dataset_id == Iris:
       self.dataset= datasets.load_iris()
       self.write("loaded iris dataset")

    if self.dataset_id == Digits:
       self.dataset= datasets.load_digits()
       self.write("loaded Digits dataset")

    if self.dataset_id == Wine:
       self.dataset= datasets.load_wine()
       self.write("loaded Wine dataset")

    if self.dataset_id == BreastCancer:
       self.dataset= datasets.load_breast_cancer()
       self.write("loaded BreastCancer dataset")

    attr = dir(self.dataset)
    self.write("dir:" + str(attr))

    if "feature_names" in attr:
      self.write("feature_names:" + str(self.dataset.feature_names))
    if "target_names" in attr:
      self.write("target_names:" + str(self.dataset.target_names))

    self.set_model_filename()
    self.view.description.setText(self.dataset.DESCR)

    X, y = self.dataset.data, self.dataset.target
    self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(X, y, test_size=0.3, random_state=42)

    self._end(self.load_dataset.__name__)

  def build(self):
    self._start(self.build.__name__)
    self.model = tree.DecisionTreeClassifier(random_state=0)
    self._end(self.build.__name__)

  def train(self):
    self._start(self.train.__name__)
    start = time.time()

    params = {'max_depth':range(3,20)}

    grid_search = GridSearchCV(self.model, param_grid=params, n_jobs=4)
    grid_search.fit(self.X_train, self.y_train)

    self.write("GridSearch BestParams " + str(grid_search.best_params_) )
    self.write("GridSearch BestScore "  + str(grid_search.best_score_))

    self.model = tree.DecisionTreeClassifier(**grid_search.best_params_)

    # Class fit method of the classifier
    self.model.fit(self.X_train, self.y_train)

    elapsed_time = time.time() - start
    elapsed = str("Train elapsed_time:{0}".format(elapsed_time) + "[sec]")
    self.write(elapsed)
    self._end(self.train.__name__)

  def predict(self):
    self._start(self.predict.__name__)
    self.pred_test  = self.model.predict(self.X_test)
    report = str (classification_report(self.y_test, self.pred_test) )
    self.write(report)

    self._end(self.predict.__name__)

  def visualize(self):
    cmatrix = confusion_matrix(self.y_test, self.pred_test)
    self.view.visualize(cmatrix, self.model)

############################################################
# Classifier View

class MainView(ZApplicationView):
  # Class variables

  # ClassifierView Constructor
  def __init__(self, title, x, y, width, height):
    super(MainView, self).__init__(title, x, y, width, height)
    self.font        = QFont("Arial", 10)
    self.setFont(self.font)

    # 1 Add a labeled combobox to top dock area
    self.add_datasets_combobox()

    # 2 Add a textedit to the left pane of the center area.
    self.text_editor = QTextEdit()
    self.text_editor.setLineWrapColumnOrWidth(600)
    self.text_editor.setLineWrapMode(QTextEdit.FixedPixelWidth)

    # 3 Add a tabbed_window to the right pane of the center area.
    self.tabbed_window = ZTabbedWindow(self, 0, 0, width/2, height)

    # 4 Add a description text edit.
    self.description = QTextEdit()
    self.description.setLineWrapColumnOrWidth(600)
    self.description.setLineWrapMode(QTextEdit.FixedPixelWidth)

    # 5 Add a figure_view to the right pane of the center area.
    self.figure_view = ZScalableScrolledFigureView(self, 0, 0, width/2, height)

    # 6 Add a figure_view to the right pane of the center area.
    self.tree_view = ZScalableScrolledDecisionTreeView(self, 0, 0, width/2, height)

    self.add(self.text_editor)
    self.add(self.tabbed_window)

    self.tabbed_window.add("Description",     self.description)
    self.tabbed_window.add("ConfusionMatrix", self.figure_view)
    self.tabbed_window.add("DecisionTree",    self.tree_view)

    self.figure_view.hide()
    self.tree_view.hide()

    self.show()

  def add_datasets_combobox(self):
    self.dataset_id = Iris
    self.datasets_combobox = ZLabeledComboBox(self, "Datasets", Qt.Horizontal)

    # We use the following datasets of sklearn to test DecisionTreeClassifier.
    self.datasets = {"Iris": Iris, "Digits": Digits, "Wine": Wine, "BreastCancer": BreastCancer}
    title = self.get_title()
    self.setWindowTitle( "Iris" + " - " + title)

    self.datasets_combobox.add_items(self.datasets.keys())
    self.datasets_combobox.add_activated_callback(self.datasets_activated)
    self.datasets_combobox.set_current_text(self.dataset_id)

    self.start_button = ZPushButton("Start", self)
    self.clear_button = ZPushButton("Clear", self)

    self.start_button.add_activated_callback(self.start_button_activated)
    self.clear_button.add_activated_callback(self.clear_button_activated)

    self.datasets_combobox.add(self.start_button)
    self.datasets_combobox.add(self.clear_button)

    self.set_top_dock(self.datasets_combobox)

  def write(self, text):
    self.text_editor.append(text)
    self.text_editor.repaint()

  def datasets_activated(self, text):
    self.dataset_id = self.datasets[text]
    title = self.get_title()
    self.setWindowTitle(text + " - " + title)

  def start_button_activated(self, text):
    self.model = DecisionTreeClassifierModel(self.dataset_id, self)
    self.start_button.setEnabled(False)
    self.clear_button.setEnabled(False)
    try:
      self.model.run()
    except:
      pass
    self.start_button.setEnabled(True)
    self.clear_button.setEnabled(True)

  def clear_button_activated(self, text):
    self.text_editor.setText("")
    self.description.setText("")
    self.figure_view.hide()
    self.tree_view.hide()
    if plt.gcf() != None:
      plt.close()

  def visualize(self, cmatrix, tree):
    # 1 Show figure view
    self.figure_view.show()
    if plt.gcf() != None:
      plt.close()

    sns.set()
    df = pd.DataFrame(cmatrix)
    sns.heatmap(df, annot=True, fmt="d")

    self.figure_view.set_figure(plt)

    # 2 Show tree view
    self.tree_view.show()

    feature_names = None
    try:
      feature_names = tree.dataset.feature_names
    except:
      pass

    target_names  = None
    try:
      target_names = tree.dataset.target_names
    except:
      pass

    self.tree_view.set_tree(tree, feature_names, target_names)

############################################################
#
if main(__name__):

  try:
    app_name  = os.path.basename(sys.argv[0])
    applet    = QApplication(sys.argv)

    main_view = MainView(app_name, 40, 40, 1000, 500)
    main_view.show ()

    applet.exec_()

  except:
    traceback.print_exc()


from decimal import Decimal
from random import random
from unittest.mock import ANY, Mock
from uuid import UUID, uuid4

from fastapi import FastAPI
from injector import InstanceProvider
from mockito import when
from pytest import fixture, mark

from currency import Currency
from ordering import Service as OrderingService
from ordering import commands, errors
from ordering.queries import BuyOrdersQueries
from tests.ordering.factories import BuyOrderFactory as BuyOrder

from .factories import ApiCreateBuyOrderRequestFactory as CreateBuyOrder

CREATE_ORDER_URL = "/orders/"

class TestCreateBuyOrderRequest:
    def test_after_creating_redirects_to_created_order(self, api_client):
        request = CreateBuyOrder()
        response = api_client.post(CREATE_ORDER_URL, json=request)
        assert response.status_code == 201
        order_url = response.headers["Location"]
        order = api_client.get(order_url).json()
        assert order["request_id"] == request["request_id"]

    def test_creating_order_is_idempotent(self, api_client):
        request = CreateBuyOrder()
        first = api_client.post(CREATE_ORDER_URL, json=request)
        second = api_client.post(CREATE_ORDER_URL, json=request)
        assert first.headers["Location"] == second.headers["Location"]

    @mark.parametrize(
        "request_id", ["ILLEGAL", uuid4().hex[:-3], "", random(), None]
    )
    def test_reject_when_no_uuid_id(self, api_client, request_id):
        request = CreateBuyOrder().update(request_id=request_id)
        response = api_client.post(CREATE_ORDER_URL, json=request)
        assert response.status_code == 422

    def test_reject_when_negative_amount(self, api_client):
        request = CreateBuyOrder().update(amount=-1)
        response = api_client.post(CREATE_ORDER_URL, json=request)
        assert response.status_code == 422

    def test_reject_when_amount_higher_than_1_000_000_000(self, api_client):
        request = CreateBuyOrder().update(amount=1_000_000_000)
        response = api_client.post(CREATE_ORDER_URL, json=request)
        assert response.status_code == 422

    @mark.parametrize("currency", ["PLN", "AUD", "XXX"])
    def test_reject_when_currency_not_eur_gbp_usd(self, api_client, currency):
        request = CreateBuyOrder().update(currency=currency)
        response = api_client.post(CREATE_ORDER_URL, json=request)
        assert response.status_code == 422

class TestCreateBuyOrderController:
    def test_201_when_created(
            self, app, api_client, create_buy_order, order_url,
    ):
        response = api_client.post(CREATE_ORDER_URL, json=create_buy_order)
        assert response.status_code == 201
        assert response.headers["Location"] == order_url

    def test_301_when_already_created(
            self, api_client, ordering, order_id, order_url,
    ):
        when(ordering).create_buy_order(...).thenRaise(
            errors.OrderAlreadyExists(order_id)
        )

        response = api_client.post(CREATE_ORDER_URL, json=CreateBuyOrder())

        assert response.status_code == 301
        assert response.headers["Location"] == order_url

    def test_409_when_order_limit_exceeded(self, api_client, ordering):
        when(ordering).create_buy_order(...).thenRaise(
            errors.BalanceLimitExceeded(Decimal(100))
        )

        response = api_client.post(CREATE_ORDER_URL, json=CreateBuyOrder())

        assert response.status_code == 409
        assert response.json()["detail"] == "Exceeded 100BTC ordering limit"

    @fixture
    def create_buy_order(self) -> dict:
        return CreateBuyOrder()

    @fixture
    def order_id(self) -> UUID:
        return uuid4()

    @fixture
    def order_url(self, app: FastAPI, order_id: UUID) -> str:
        return app.url_path_for("orders:get_order", order_id=str(order_id))

    @fixture(autouse=True)
    def ordering(
            self, container, create_buy_order, order_id,
    ) -> OrderingService:
        service = Mock(spec=OrderingService)
        container.binder.bind(OrderingService, to=InstanceProvider(service))
        queries = Mock(spec=BuyOrdersQueries)
        container.binder.bind(BuyOrdersQueries, to=InstanceProvider(queries))

        request_id = UUID(hex=create_buy_order["request_id"])
        when(queries).get_order_id(request_id).thenReturn(order_id)
        when(service).create_buy_order(
            commands.CreateBuyOrder.construct(
                id=request_id,
                amount=(
                    Decimal(create_buy_order["amount"])
                    .quantize(Decimal(10) ** -4)
                ),
                currency=Currency[create_buy_order["currency"]],
                timestamp=ANY,
            )
        ).thenReturn(order_id)
        return service

class TestGetBuyOrderController:
    def test_404_when_no_order(self, api_client):
        response = api_client.get(f"/orders/{uuid4()}")
        assert response.status_code == 404

    def test_order_data_when_order_exists(self, api_client, order):
        response = api_client.get(f"/orders/{order.id}")
        assert response.json() == {
            "id": str(order.id),
            "request_id": str(order.request_id),
            "bitcoins": float(order.bitcoins),
            "bought_for": float(order.bought_for),
            "currency": order.currency.name,
        }

    @fixture
    def order(self) -> BuyOrder:
        return BuyOrder()

    @fixture(autouse=True)
    def queries(self, container, order) -> BuyOrdersQueries:
        queries = Mock(BuyOrdersQueries)
        when(queries).get_order(...).thenReturn(None)
        when(queries).get_order(order.id).thenReturn(order)
        container.binder.bind(BuyOrdersQueries, to=InstanceProvider(queries))
        return queries


# -*- coding: utf-8 -*-
"""
Created on Fri Dec 11 14:34:04 2015

@author: 89965
fonctions de structurelles diverses
"""
import os
import re
import logging
import subprocess
from collections import defaultdict
import psutil

import pyetl.formats.formats as F
import pyetl.formats.mdbaccess as DB
from .outils import charge_mapping, remap, prepare_elmap, renseigne_attributs_batch

LOGGER = logging.getLogger('pyetl')

def map_struct(regle):
    """mappe la structure clef etrangeres et fonctions"""
    charge_mapping(regle, mapping=regle.schema.mapping)

def _map_schemas(regle, obj):
    '''essaye de trouver un mapping pour une classe'''
    if obj is None:
        if regle.getvar("schema_entree"):
            schema_origine = regle.stock_param.schemas[regle.getvar("schema_entree")]
            print('-------------------------mapping', schema_origine)
#        else:
#            return
#        if regle.params.val_entree.val:
#            schema2 = regle.stock_param.init_schema(regle.params.val_entree.val,
#                                                    modele=schema_origine, origine='B')
#        else:
        return
    else:
        schema_origine = obj.schema.schema
        if regle.params.val_entree.val:
            schema2 = regle.stock_param.init_schema(regle.params.val_entree.val,
                                                    modele=schema_origine, origine='B')
        else:
            schema2 = obj.schema.schema
    regle.schema = schema2
    if schema2.elements_specifiques:
        for i in schema2.elements_specifiques:
#            print('mapping specifique', i)
            spec = schema2.elements_specifiques[i]
            mapped = remap(spec, regle.elmap)
#            print('mapping specifique', i, len(spec), '->', len(mapped))
            schema2.elements_specifiques[i] = mapped
    else:
        LOGGER.info("pas d'elements specifiques")

#        print("-----------------------------pas d'elements specifiques")

    for i in schema_origine.classes:
        schema2.get_classe(i, modele=schema_origine.classes[i], cree=True)
    for i in list(schema_origine.classes.keys()):
#        print ('map_schemas ',schema_origine.nom,i,regle.mapping.get(i))
        if i in regle.mapping:
            schema2.renomme_classe(i, regle.mapping[i])

        # mapping foreign keys :

#    print("mapping effectue", len(schema2.classes))
    for clef in schema2.classes:
        if clef in regle.mapping_attributs:
            for orig, dest in regle.mapping_attributs[clef].items():
                schema2.classes[clef].rename_attribut(orig, dest)

def applique_mapping(regle):
    """gere les clefs etrangeres et les elements speciaux dans les mappings"""
    mapping = regle.schema.mapping
    regle.elmap = prepare_elmap(mapping)
    _map_schemas(regle, None)
    regle.nbstock = 0
    for i in mapping:
        for scl in regle.schema.classes.values():
            scl.renomme_cible_classe(i, mapping[i])

def h_map2(regle):
    """ prepare le mapping des structures"""
    regle.store = True
    regle.blocksize = 1
    regle.nbstock = 0
    regle.traite_stock = applique_mapping

def f_map2(regle, obj):
    '''#aide||mapping en fonction d'une creation dynamique de schema
  #aide_spec||parametres: mappe les structures particulieres
   #pattern2||;;;map;=#struct;;
    '''
    regle.schema = obj.schema.schema
    regle.nbstock = 1

def h_map(regle):
    ''' precharge le fichier de mapping et prepare les dictionnaires'''
    regle.dynlevel = 0 # les noms de mapping dependent ils des donnees d entree
    regle.mapping = None
    regle.schema = None
#    if regle.params.att_sortie.val == '#schema': # mapping d un schema existant
#        schema2 =
    regle.changeschema = True
    fich = regle.params.cmp1.val
    if "[F]" in fich:
        regle.dynlevel = 2
    elif "[C]" in fich:
        regle.dynlevel = 1
    if regle.dynlevel:
        regle.clefdyn = ""
    else:
        charge_mapping(regle)
        _map_schemas(regle, None)

def f_map(regle, obj):
    '''#aide||mapping en fonction d'un fichier
  #aide_spec||parametres: map; nom du fichier de mapping
 #aide_spec2||si #schema est indique les objets changent de schema
    #pattern||?=#schema;?C;;map;C;;
  #test||obj||^#schema;test;;map;%testrep%/refdata/map.csv;;||atv;toto;A
 #test2||obj||^#schema;test;;map+-;%testrep%/refdata/map.csv;;||cnt;2
    '''
#    print ("dans map ===============",obj)
    if regle.dynlevel: # attention la regle est dynamique
        clef_dyn = regle.stock_param.chemin_courant if regle.dynlevel == 1\
                   else regle.stock_param.fichier_courant
        if clef_dyn != regle.clef_dyn:
            charge_mapping(regle)
    if not regle.schema:
        _map_schemas(regle, obj)
    clef = obj.ident
    schema2 = regle.schema
    if clef in regle.mapping:
        nouv = regle.mapping.get(clef)
        obj.setident(nouv, schema2=schema2)
        if clef in regle.mapping_attributs:
            for orig, dest in regle.mapping_attributs[clef].items():
                try:
                    obj.attributs[dest] = obj.attributs[orig]
                    del obj.attributs[orig]
                except KeyError:
                    obj.attributs[dest] = ''

        return True
#    print ('====================== mapping non trouve', clef)
#    print ('definition mapping', '\n'.join([str(i)+':\t\t'+str(regle.mapping[i])
#                                            for i in sorted(regle.mapping)]))
    return False

def store_traite_stock(regle):
    ''' relache les objets '''
    store = regle.tmpstore

    reverse = regle.params.cmp2.val == 'rsort'
#    print ("tri inverse ",reverse)
    if isinstance(store, list):
        if regle.params.cmp2.val:
            keyval = lambda obj: "|".join(obj.attributs.get(i, '')
                                          for i in regle.params.att_entree.liste)
            store.sort(key=keyval, reverse=reverse)
        for obj in store:
#            print ('store: relecture objet ', obj, obj.schema.identclasse,obj.schema.info)
            regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["end:"])
    else:
        for clef in sorted(store.keys(), reverse=reverse) if regle.params.cmp2.val else store:
            obj = store[clef]
            regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["end:"])
    h_stocke(regle) # on reinitialise

def h_stocke(regle):
    '''marque la regle comme stockante'''
#    print ('stockage tmpstore ', regle.params.att_entree.liste)
    regle.store = True
    regle.stocke_obj = True # on stocke les objets et pas que la clef
    regle.nbstock = 0
    regle.traite_stock = store_traite_stock
    regle.tmpstore = dict() if regle.params.cmp1.val else list()
    # mode comparaison : le stock est reutilise ailleurs (direct_reuse)=False
    regle.direct_reuse = not 'cmp' in regle.params.cmp1.val
    regle.fold = regle.params.cmp1.val == 'cmpf'
    if regle.params.cmp2.val == 'clef':
        regle.stocke_obj = False
        regle.tmpstore = set()

def f_stocke(regle, obj):
    '''#aide||stockage temporaire d'objets pour assurer l'ordre dans les fichiers de sortie
  #aide_spec||liste de clefs,tmpstore;uniq;sort|rsort : stockage avec option de tri
  #aide_spec2||liste de clefs,tmpstore;cmp;nom : prechargement pour comparaisons
   #pattern1||;;?L;tmpstore;?=uniq;?=sort;||
   #pattern2||;;?L;tmpstore;?=uniq;?=rsort;||
   #pattern3||;;?L;tmpstore;=cmp;A;?=clef||
   #pattern4||;;?L;tmpstore;=cmpf;A;?=clef||
       #test||obj;point;4||^;;V0;tmpstore;uniq;rsort||^;;C1;unique||atv;V0;3;
      #test2||obj;point;4||^V2;;;cnt;-1;4;||^;;V2;tmpstore;uniq;sort||^;;C1;unique;||atv;V2;1;
    '''
#    regle.stock.append(obj)
    if obj.virtuel:
        return True
    if regle.direct_reuse:
        regle.nbstock += 1

    if regle.params.cmp1.val:
        if len(regle.params.att_entree.liste) > 1:
            clef = "|".join(obj.attributs.get(i, '') for i in regle.params.att_entree.liste)
        else:
            clef = obj.attributs.get(regle.params.att_entree.val, '')
        if regle.stocke_obj:
            regle.tmpstore[clef] = obj
        else:
            regle.tmpstore.add(obj)
        return True
#    print ('store: stockage objet ', obj, obj.schema.identclasse,obj.schema.info)
    regle.tmpstore.append(obj)
    return True

def h_uniq(regle):
    ''' stocke les clefs pour l'unicite '''
    regle.tmpstore = set()

def f_uniq(regle, obj):
    '''#aide||unicite de la sortie laisse passer le premier objet et filtre le reste
    #aide_spec||liste des attibuts devant etre uniques si #geom : test geometrique
    #pattern||;?=#geom;?L;unique;;;
    #test||obj;point;2||^;;C1;unique||+fail:;;;;;;;pass>;;||cnt;1
    #test2||obj;point;2||^;;C1;unique-||cnt;1
    #test3||obj;point;2||^;#geom;;unique-||cnt;1
    #test4||obj;point;2||^;#geom;C1;unique-||cnt;1
    '''
#    regle.stock.append(obj)
    clef = str(tuple(tuple(i) for i in obj.geom_v.coords))\
               if regle.params.val_entree.val == '#geom' else ''
    clef = clef + "|".join(obj.attributs.get(i, '') for i in regle.params.att_entree.liste)

#    print ('uniq ',clef, regle.params.att_entree.val )
    if clef in regle.tmpstore:
        return False
    regle.tmpstore.add(clef)
    return True

def h_uniqcnt(regle):
    ''' stocke les clefs pour l'unicite '''
    regle.maxobj = regle.params.cmp1.num if regle.params.cmp1.num else 1
    regle.cnt = regle.maxobj > 1
    regle.tmpstore = defaultdict(int)

def f_uniqcnt(regle, obj):
    '''#aide||unicite de la sortie laisse passer les N premiers objet et filtre le reste
    #pattern||A;?=#geom;?L;unique;?N;||sortie
     #schema||ajout_attribut
       #test||obj;point;4||^X;;C1;unique;2;||+fail:;;;;;;;pass>;;||cnt;2
      #test2||obj;point;4||^X;;C1;unique-;2;||cnt;2
      #test3||obj;point;4||^X;#geom;;unique-;2;||cnt;2
      #test4||obj;point;4||^X;#geom;C1;unique-;2;||cnt;2
      #test4||obj;point;4||V0;1;;;V0;2;;set;;;||^X;#geom;V0;unique>;1;;||cnt;1
    '''
#    regle.stock.append(obj)
    clef = str(tuple(tuple(i) for i in obj.geom_v.coords))\
           if regle.params.val_entree.val == '#geom' else ''
    clef = clef + "|".join(obj.attributs.get(i, '') for i in regle.params.att_entree.liste)
    regle.tmpstore[clef] += 1
    obj.attributs[regle.params.att_sortie.val] = str(regle.tmpstore[clef])
    if regle.tmpstore[clef] > regle.maxobj:
        return False
    return True

def sortir_traite_stock(regle):
    '''ecriture finale'''
    print('traite stock sortir', regle.final)
    if regle.final:
        regle.f_sortie.ecrire_objets(regle, True)
        regle.nbstock = 0
        return
    for groupe in list(regle.stockage.keys()):
        for obj in regle.recup_objets(groupe):
            regle.f_sortie.ecrire_objets_stream(obj, regle, False)
            regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["end:"])
    regle.nbstock = 0

def h_sortir(regle):
    '''preparation sortie'''

    if regle.params.att_sortie.val == "#schema": # on force les noms de schema pour l'ecriture
        regle.nom_fich_schema = regle.params.val_entree.val
    else:
        regle.nom_fich_schema = regle.params.cmp2.val
    regle.nom_base = os.path.basename(regle.params.cmp2.val
                                      if regle.params.cmp2.val else regle.nom_fich_schema)

    if regle.debug:
        print("nom de schema ", regle.nom_fich_schema)

    if '[' in regle.params.cmp1.val: # on a defini un fanout
        tmplist = regle.params.cmp1.val.find('[')
        #print("valeur ii ", regle.params.cmp1,ii)

        regle.setvar("fanout", regle.params.cmp1.val[tmplist+1:-1])
        regle.params.cmp1.val = regle.params.cmp1.val[:tmplist]
    regle.f_sortie = F.Writer(regle.params.cmp1.val) # tout le reste
#    print ('positionnement writer ',regle, regle.params.cmp1.val)
    if regle.f_sortie.nom_format == 'sql': # gestion des dialectes sql et du mode connecté
        destination = regle.f_sortie.writerparms.get('destination')
        dialecte = regle.f_sortie.writerparms.get('dialecte')
        regle.f_sortie.writerparms['reinit'] = regle.getvar('reinit')
        regle.f_sortie.writerparms['nodata'] = regle.getvar('nodata')
        if destination: # on va essayer de se connecter
            connection = DB.dbaccess(regle.stock_param, destination)
            if connection.valide:
                regle.f_sortie.gensql = connection.gensql # la on a une instance connectee
        elif dialecte:
            regle.f_sortie.gensql = dialecte.gensql()
#        print ('sortie',regle.ligne,regle.f_sortie.writerparms)
    elif regle.f_sortie.nom_format == 'file': #gestion de fichiers de texte generiques
        dialecte = regle.f_sortie.writerparms.get('dialecte')
        regle.ext = dialecte

    if regle.params.cmp2.val and regle.params.cmp2.val != "#print":
        rep_base = regle.getvar('_sortie', loc=0)
#   print('positionnement sortie', rep_base, os.path.join(rep_base, regle.params.cmp2.val))

        regle.setvar('_sortie', os.path.join(rep_base, regle.params.cmp2.val), loc=1)

    regle.fanout = regle.getvar("fanout", 'groupe')\
                   if regle.f_sortie.multiclasse else 'classe'
#    print("fanout de sortie",regle.fanout)
    regle.calcule_schema = regle.f_sortie.calcule_schema
    regle.memlimit = int(regle.getvar('memlimit', 0))
    regle.store = None
    regle.nbstock = 0
    regle.traite_stock = sortir_traite_stock
#    regle.liste_attributs = regle.params.att_entree.liste
    if regle.stock_param.debug:
        print('sortir :', regle.params.att_entree.liste)
    regle.final = True
    regle.menage = True
    #print ('icsv: sortir copy:',regle.copy,'stream:',regle.stock_param.stream)
    if regle.copy and regle.getvar("mode_sortie", "A") == "D":
    # cette regle consomme les objets sauf si on est en mode copie et streaming
        regle.final = False
        regle.copy = False
    regle.valide = True
#    print ('fin preparation sortie ',regle.f_sortie.writerparms)

def setschemasortie(regle, obj):
    '''positionne le schema de sortie pour l objet '''
    if regle.nom_fich_schema:# on copie le schema pour ne plus le modifier apres ecriture
        regle.change_schema_nom(obj, regle.nom_fich_schema)
    if obj.schema and obj.schema.amodifier(regle):
        obj.schema.setsortie(regle.f_sortie, os.path.join(regle.getvar('_sortie'),
                                                          os.path.dirname(regle.params.cmp1.val)))
        obj.schema.setminmaj(regle.f_sortie.minmaj)
    if regle.params.att_entree.liste:
        obj.liste_atttributs = regle.params.att_entree.liste

def f_sortir(regle, obj):
    '''#aide||sortir dans differents formats
  #aide_spec||parametres:?(#schema;nom_schema);?liste_attributs;sortir;format[fanout]?;?nom
    #pattern||?=#schema;?C;?L;sortir;?C;?C||sortie
       #test||redirect||obj||^Z;ok;;set||^;;;sortir;csv;#print||end
    '''
    if obj.virtuel: # on ne traite pas les virtuels
        return True
    listeref = obj.liste_attributs
    schemaclasse_ref = obj.schema

    setschemasortie(regle, obj)
    if regle.store is None: # on decide si la regle est stockante ou pas
        regle.store = regle.f_sortie.calcule_schema and\
            (not obj.schema or not obj.schema.stable)
        if regle.store: # on ajuste les branchements
            regle.setstore()

    if regle.store:
        regle.nbstock += 1
        groupe = obj.attributs["#groupe"]
#        print("stockage", obj.ido, groupe, regle)
        if groupe != "#poubelle":
            nom_base = regle.nom_base
            #regle.stock_param.nb_obj+=1
            if regle.stock_param.stream: #sortie classe par classe
                if groupe  not in regle.stockage:
                    regle.f_sortie.ecrire_objets(regle, False) # on sort le groupe precedent
                    regle.compt_stock = 0
            regle.endstore(nom_base, groupe, obj, regle.final,
                           geomwriter=regle.f_sortie.tmp_geom, nomgeom=regle.f_sortie.nom_fgeo)
            return True

    regle.f_sortie.ecrire_objets_stream(obj, regle, False)
    obj.schema = None

    if regle.final:
        return True
    # la on regenere l'objet et on l'envoie dans le circuit poutr la suite
    obj.setschema(schemaclasse_ref)
    obj.liste_attributs = listeref
        # on reattribue le schema pour la sortie en simulant une copie
    return True

def valreplace(chaine, obj):
    '''remplace les elements provenant de l objet '''
    vdef = r'\[(#?[a-zA-Z_][a-zA-Z0-9_]*)\]'
    repl = lambda x: obj.attributs.get(x.group(1), '')
    return re.sub(vdef, repl, chaine)

def preload(regle, obj):
    '''prechargement'''
    vrep = lambda x: regle.resub.sub(regle.repl, x)
    chaine_comm = vrep(regle.params.cmp1.val)
    regle.setvar('nocomp', False)
    process = psutil.Process(os.getpid())

    mem1 = process.memory_info()[0]
    if obj and regle.params.att_entree.val:
        entree = obj.attributs.get(regle.params.att_entree.val, regle.fich)
    else:
        entree = regle.entree if regle.entree else valreplace(regle.fich, obj)

    print('------- preload commandes:(', chaine_comm, ') f:', entree,
          'clef', regle.params.att_sortie.val)
    if chaine_comm: # on precharge via une macro
        nomdest = regle.params.cmp2.val if regle.params.cmp2.val.startswith('#') \
                                        else '#'+ regle.params.cmp2.val
        processor = regle.stock_param.getpyetl(chaine_comm, entree=entree, rep_sortie=nomdest)
        processor.process()
        renseigne_attributs_batch(regle, obj, processor.retour)

        print('------- preload ', processor.store)
        regle.stock_param.store.update(processor.store) # on rappatrie les dictionnaires de stockage
        regle.setvar('storekey', processor.retour)  # on stocke la clef

    else:
#        racine = regle.stock_param.racine
        chemin = os.path.dirname(entree)
        fichier = os.path.basename(entree)
        ext = os.path.splitext(fichier)[1]
        lecteur = regle.stock_param.reader(ext)
        regle.reglestore.tmpstore = dict()
        nb_total = 0
        try:
            nb_total = lecteur.lire_objets('', chemin, fichier, regle.stock_param,
                                           regle.reglestore)
            regle.stock_param.store[regle.params.cmp2.val] = regle.reglestore.tmpstore
        except FileNotFoundError:
            regle.stock_param.store[regle.params.cmp2.val] = None
            print('fichier inconnu', os.path.join(chemin, fichier))

    mem2 = process.memory_info()[0]
    mem = mem2-mem1
    print('------- preload ', nb_total, mem, '--------', int(mem/(nb_total+1)))

def h_preload(regle):
    '''prechargement'''
    obj = None
    mapper = regle.stock_param
    reglestore = mapper.interpreteur(";;;;;;"+regle.params.att_sortie.val+
                                     ";tmpstore;cmp;"+regle.params.cmp2.val, "", 99999)
    regle.reglestore = reglestore
    regle.repl = lambda x: obj.attributs.get(x.group(1), '')
    regle.resub = re.compile(r'\[(#?[a-zA-Z_][a-zA-Z0-9_]*)\]')
    fich = regle.params.val_entree.val
#    fich = fich.replace('[R]', regle.stock_param.racine)
    regle.fich = fich
    regle.dynlevel = 0
    if '[R]' in fich:
        regle.dynlevel = 1
    if "[F]" in fich:
        regle.dynlevel = 2
    elif "[G]" in fich:
        regle.dynlevel = 1
    elif "[" in fich:
        regle.dynlevel = 3
    regle.entree = None

    if regle.dynlevel == 0: # pas de selecteur on precharge avant de lire
        regle.entree = regle.params.val_entree.val
        regle.fich = regle.entree
        preload(regle, None)
        regle.valide = "done"

    print('==================h_preload===', regle.dynlevel, regle.valide)

def f_preload(regle, obj):
    '''#aide||precharge un fichier en appliquant une macro
  #aide_spec||parametres clef;fichier;attribut;preload;macro;nom
 #aide_spec1||les elements entre [] sont pris dans l objet courant
 #aide_spec2||sont reconnus[G] pour #groupe et [F] pour #classe pour le nom de fichier
    #pattern||A;?C;?A;preload;?C;C
    #!test||
    '''
    fich = regle.fich

    if regle.dynlevel > 0:
        fich = fich.replace('[G]', obj.attributs['#groupe'])
        fich = fich.replace('[R]', regle.stock_param.racine)
        fich = fich.replace('[F]', obj.attributs['#classe'])
        if fich != regle.entree:
            regle.entree = fich
            print('==================f_preload===', regle.stock_param.racine, regle.entree)

            preload(regle, obj)
#            print ('chargement ',regle.params.cmp2.val,
#                   regle.stock_param.store[regle.params.cmp2.val])
    return True

def compare_traite_stock(regle):
    """ sort les objets detruits"""
    for obj in regle.comp.values():
        obj.attributs[regle.params.att_sortie.val] = 'supp'
        obj.setident(regle.precedent)
        regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["supp:"])
    regle.comp = None
    regle.nbstock = 0

#def compare_traite_stock(regle):
#    """ sort les objets detruits"""
#    for clef, obj in regle.comp.items():
#        if obj.redirect is None:
#            obj.attributs[regle.params.att_sortie.val]='supp'
#            regle.stock_param.moteur.traite_objet(obj, regle.branchements.brch["supp:"])
#        regle.comp[clef] = None
#    regle.comp = None
#    regle.nbstock = 0

def h_compare(regle):
    """comparaison a une reference"""
    regle.branchements.addsortie('new:')
    regle.branchements.addsortie('supp:')
    regle.branchements.addsortie('diff:')
    regle.branchements.addsortie('orig:')
#    regle.taites = set()
    regle.store = True
    regle.nbstock = 0
    regle.comp = None
    regle.precedent = None
    regle.traite_stock = compare_traite_stock

def f_compare2(regle, obj):
    '''#aide||compare a un element precharge
  #aide_spec||parametres clef;fichier;attribut;preload;macro;nom
 #aide_spec2||sort en si si egal en sinon si different
 #aide_spec3||si les elements entre [] sont pris dans l objet courant
    #pattern||A;;?L;compare2;A;C
     #helper||compare
     #schema||ajout_attribut
    #!test||
    '''
    if regle.precedent != obj.ident:
        comp = regle.stock_param.store[regle.params.cmp2.val]
        if regle.comp and comp is not regle.comp:
            compare_traite_stock(regle)
            regle.nbstock = 1
        regle.comp = comp
        if regle.comp:
            if regle.params.att_entree.liste:
                regle.comp2 = {i:([i.attributs[j] for j in
                                   regle.params.att_entree.liste]) for i in regle.comp}
            else:
                regle.comp2 = {i:([i.attributs[j] for j in
                                   sorted([k for k in i.attributs if k[0] != "#"])])
                               for i in regle.comp}
#    print ('comparaison ', len(regle.comp), regle.comp)
    try:
        if len(regle.params.cmp1.liste) > 1:
            clef = "|".join(obj.attributs.get(i, '') for i in regle.params.att_entree.liste)
        else:
            clef = obj.attributs[regle.params.cmp1.val]
        ref = regle.comp2[clef]
        regle.ref.add(clef)
    except KeyError:
        obj.redirect = "new:"
        obj.attributs[regle.params.att_sortie.val] = 'new'
        return False
    if regle.params.att_entree.liste:
        compare = all([obj.attributs[i] == ref.attributs[i]
                       for i in regle.params.att_entree.liste])
    else:
        atts = {i for i in obj.attributs if i[0] != "#"}
        kref = {i for i in ref.attributs if i[0] != "#"}
    #    id_att = atts == kref
        compare = atts == kref and all([obj.attributs[i] == ref.attributs[i]
                                        for i in atts]) and obj.geom == ref.geom
    if compare:
        return True
    obj.redirect = "diff:"
    obj.attributs[regle.params.att_sortie.val] = 'diff'
    ref.attributs[regle.params.att_sortie.val] = 'orig'
    regle.stock_param.moteur.traite_objet(ref, regle.branchements.brch["orig:"])
    # on remet l'original dans le circuit
    return False

def f_compare(regle, obj):
    '''#aide||compare a un element precharge
  #aide_spec||parametres clef;fichier;attribut;preload;macro;nom
 #aide_spec2||sort en si si egal en sinon si different
 #aide_spec3||si les elements entre [] sont pris dans l objet courant
    #pattern||A;;?L;compare;A;C
     #schema||ajout_attribut
    #!test||
    '''
    if regle.precedent != obj.ident: # on vient de changer de classe
        if regle.comp:
            compare_traite_stock(regle)
            regle.nbstock = 1
        regle.comp = regle.stock_param.store[regle.params.cmp2.val]
        regle.precedent = obj.ident
#    print ('comparaison ', len(regle.comp), regle.comp)
    if regle.comp is None:
        return False

    try:
        if len(regle.params.cmp1.liste) > 1:
            clef = "|".join(obj.attributs.get(i, '')
                            for i in regle.params.att_entree.liste)
        else:
            clef = obj.attributs[regle.params.cmp1.val]
        ref = regle.comp.pop(clef)
    except KeyError:

        obj.redirect = "new:"
        obj.attributs[regle.params.att_sortie.val] = 'new'
        return False
    if regle.params.att_entree.liste:
        compare = all([obj.attributs[i] == ref.attributs[i]
                       for i in regle.params.att_entree.liste])
    else:
        atts = {i for i in obj.attributs if i[0] != "#"}
        kref = {i for i in ref.attributs if i[0] != "#"}
    #    id_att = atts == kref
        compare = atts == kref and all([obj.attributs[i] == ref.attributs[i]
                                        for i in atts]) and obj.geom == ref.geom
    if compare:
        return True
    obj.redirect = "diff:"
    obj.attributs[regle.params.att_sortie.val] = 'diff'
    ref.attributs[regle.params.att_sortie.val] = 'orig'
    ref.setident(obj.ident) # on force l'identite de l'original
    regle.stock_param.moteur.traite_objet(ref, regle.branchements.brch["orig:"])
    # on remet l'original dans le circuit
    return False

def f_run(regle, obj):
    '''#aide||execute un programme exterieur
  #aide_spec||attribut qui recupere le resultat, parametres , run , nom, parametres
    #pattern||?A;?C;?A;run;C;?C
     #schema||ajout_attribut
    '''
    chaine = ' '.join((regle.params.cmp1.val, regle.params.cmp2.val,
                       obj.attributs.get(regle.params.att_entree.val, regle.params.val_entree.val)))
    fini = subprocess.run(chaine, stderr=subprocess.STDOUT)
    if regle.params.att_sortie.val:
        obj.attributs[regle.params.att_sortie.val] = str(fini)


"""The filtersets tests module."""
import pytest
from django.db.models.query import QuerySet
from django.http import HttpRequest

from communication.serializer_fields import (ParentMessageForeignKey,
                                             UserReviewForeignKey)
from conftest import OBJECTS_TO_CREATE

pytestmark = pytest.mark.django_db

def test_user_review_foreign_key(reviews: QuerySet):
    """Should return the filtered list of reviews by a user."""
    obj = UserReviewForeignKey()
    request = HttpRequest()
    user = reviews[0].professional.user
    request.user = user
    obj._context = {"request": request}  # pylint: disable=protected-access
    result = obj.get_queryset()

    assert reviews.count() == 2
    assert result.count() == 1
    assert result.first().professional.user == user

def test_parent_message_foreign_key(messages: QuerySet):
    """Should return the filtered list of messages by a user."""
    obj = ParentMessageForeignKey()
    request = HttpRequest()
    user = messages[0].sender
    request.user = user
    obj._context = {"request": request}  # pylint: disable=protected-access
    result = obj.get_queryset()

    assert messages.count() == OBJECTS_TO_CREATE * 4
    assert result.count() == OBJECTS_TO_CREATE
    assert result.first().recipient == user


import time, json, os, logging, requests

from cumulocityAPI import C8Y_BASEURL, C8Y_TENANT, C8Y_HEADERS, CumulocityAPI
from arguments_handler import get_profile_generator_mode
from oeeAPI import OeeAPI

def try_int(value):
    try:
        return int(value)
    except:
        return None

PROFILES_PER_DEVICE = try_int(os.environ.get('PROFILES_PER_DEVICE')) or 1
SLEEP_TIME_FOR_PROFILE_CREATION_LOOP = try_int(os.environ.get('SLEEP_TIME_FOR_PROFILE_CREATION_LOOP')) or 60 * 12

MODE = get_profile_generator_mode()

# JSON-PYTHON mapping, to get json.load() working
null = None
false = False
true = True
######################

logging.basicConfig(format='%(asctime)s %(name)s:%(message)s', level=logging.INFO)
log = logging.getLogger("profile-generator")
log.info("using C8Y backend:" + C8Y_BASEURL)
log.info("using C8Y tenant:" + C8Y_TENANT)

c8y_api = CumulocityAPI()
oee_api = OeeAPI()

def delete_profiles():
    simulator_ids = oee_api.get_simulator_ids()
    deleted_profiles = 0
    for simulator_id in simulator_ids:
        log.info(f'deleting profiles for {simulator_id}')
        response = requests.get(f'{C8Y_BASEURL}/inventory/managedObjects/{simulator_id}', headers=C8Y_HEADERS)
        if response.ok:
            child_devices = response.json()['childDevices']['references']
            for child_device in child_devices:
                child_device_id = child_device['managedObject']['id']
                child_device_json = c8y_api.get_managed_object(child_device_id)
                if child_device_json['type'] == c8y_api.OEE_CALCULATION_PROFILE_TYPE:
                    log.info(f'deleting managed object {child_device_id}')
                    deleted_profiles = deleted_profiles + c8y_api.delete_managed_object(child_device_id)
        else:
            log.warning(f'Couldn\'t find the managed object. response: {response}, content: {response.text}')
    log.info(f'profiles deleted: {deleted_profiles}')

if MODE == 'createProfiles':
    log.info('===============================')
    log.info('starting to create profiles ...')
    log.info(f'existing profiles: {c8y_api.count_all_profiles()}')

    counter = 0
    for _ in range(PROFILES_PER_DEVICE):
        for external_id in oee_api.get_simulator_external_ids():
            profile = oee_api.create_and_activate_profile(external_id)
            counter = counter + 1
            if counter % 200 == 0:
                log.info(f'profiles: {c8y_api.count_all_profiles()}. Wait for {SLEEP_TIME_FOR_PROFILE_CREATION_LOOP} minutes')
                # sleep for some time to be able to verify if calculation is still working with the given number of profiles
                time.sleep(SLEEP_TIME_FOR_PROFILE_CREATION_LOOP)
    log.info(f'profiles after execution: {c8y_api.count_all_profiles()}')

if MODE == 'removeSimulatorProfilesViaOee':
    log.info('===============================================')
    log.info('starting to remove all simulator profiles via OEE API ...')
    log.info(f'existing profiles: {c8y_api.count_all_profiles()}')
    oee_api.delete_all_simulators_profiles()
    log.info(f'profiles after execution: {c8y_api.count_all_profiles()}')

if MODE == 'deleteSimulatorProfiles':
    log.info('===================================')
    log.info('starting to delete all simulator profiles ...')
    log.info(f'existing profiles: {c8y_api.count_all_profiles()}')
    delete_profiles()
    log.info(f'profiles after execution: {c8y_api.count_all_profiles()}')

if MODE == 'deleteCalculationCategories':
    log.info('===================================')
    log.info('starting to delete all calculation categories ...')
    log.info(
        f'existing category managed objects: {c8y_api.count_all_categories()}')
    deleted_categories = 0
    for category in c8y_api.get_calculation_categories():
        deleted_categories += c8y_api.delete_managed_object(category['id'])
    log.info(f'Managed_objects deleted: {deleted_categories}')

if MODE == 'createCalculationCategories':
    log.info('===================================')
    log.info('starting to create calculation categories ...')
    with open('./categories.json', 'r') as f:
        categories = f.read()
    if (c8y_api.count_all_categories()) == 0:
        log.info('Create category managed object')
        c8y_api.create_managed_object(categories)
    elif (c8y_api.count_all_categories()) == 1:
        log.info('Update category managed object')
        categories_by_id = {}
        for c in json.loads(categories)['categories'] + c8y_api.get_calculation_categories()[0]['categories']:
            categories_by_id[c['id']] = c
        mo_id = c8y_api.get_calculation_categories()[0]['id']
        fragment = {
            'categories': list(categories_by_id.values())
        }
        c8y_api.update_managed_object(mo_id, json.dumps(fragment))
    else:
        log.warning('More than 1 category managed object! Unable to update managed object')
    log.info('==========Categories created==========')


import os
import tensorflow as tf
from tensorflow.keras.callbacks import EarlyStopping

from utils.callbacks import ModelCheckpoint, TimeHistory
from engine.metrics import (jaccard_index, jaccard_index_softmax, IoU_instances,
                            instance_segmentation_loss, weighted_bce_dice_loss)

def prepare_optimizer(cfg, model):
    """Select the optimizer, loss and metrics for the given model.

       Parameters
       ----------
       cfg : YACS CN object
           Configuration.

       model : Keras model
           Model to be compiled with the selected options.
    """

    assert cfg.TRAIN.OPTIMIZER in ['SGD', 'ADAM']
    assert cfg.LOSS.TYPE in ['CE', 'W_CE_DICE']

    # Select the optimizer
    if cfg.TRAIN.OPTIMIZER == "SGD":
        opt = tf.keras.optimizers.SGD(lr=cfg.TRAIN.LR, momentum=0.99, decay=0.0, nesterov=False)
    elif cfg.TRAIN.OPTIMIZER == "ADAM":
        opt = tf.keras.optimizers.Adam(lr=cfg.TRAIN.LR, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)

    # Compile the model
    if cfg.PROBLEM.TYPE == "CLASSIFICATION":
        model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=["accuracy"])
    elif cfg.LOSS.TYPE == "CE" and cfg.PROBLEM.TYPE == "SEMANTIC_SEG":
        if cfg.MODEL.N_CLASSES > 1:
            model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=[jaccard_index_softmax])
        else:
            model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[jaccard_index])
    elif cfg.LOSS.TYPE == "CE" and cfg.PROBLEM.TYPE == "INSTANCE_SEG":
        if cfg.MODEL.N_CLASSES > 1:
            raise ValueError("Not implemented pipeline option: N_CLASSES > 1 and INSTANCE_SEG")
        else:
            if cfg.DATA.CHANNELS in ["BC", "BCM"]:
                model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[jaccard_index])
            else:
                if cfg.DATA.CHANNELS == "Dv2":
                    model.compile(optimizer=opt, loss=instance_segmentation_loss(cfg.DATA.CHANNEL_WEIGHTS, cfg.DATA.CHANNELS),
                                  metrics=["mse"])
                else:
                    bin_channels = 2 if cfg.DATA.CHANNELS in ["BCD", "BCDv2"] else 1
                    model.compile(optimizer=opt, loss=instance_segmentation_loss(cfg.DATA.CHANNEL_WEIGHTS, cfg.DATA.CHANNELS),
                                  metrics=[IoU_instances(binary_channels=bin_channels)])
    elif cfg.LOSS.TYPE == "W_CE_DICE" and cfg.PROBLEM.TYPE == "SEMANTIC_SEG":
        model.compile(optimizer=opt, loss=weighted_bce_dice_loss(w_dice=0.66, w_bce=0.33), metrics=[jaccard_index])
    elif cfg.LOSS.TYPE == "W_CE_DICE" and cfg.PROBLEM.TYPE == "INSTANCE_SEG":
        raise ValueError("Not implemented pipeline option: LOSS.TYPE == W_CE_DICE and INSTANCE_SEG")

def build_callbacks(cfg):
    """Create training and validation generators.

       Parameters
       ----------
       cfg : YACS CN object
           Configuration.

       Returns
       -------
       callbacks : List of callbacks
           All callbacks to be applied to a model.
    """

    callbacks = []

    # To measure the time
    time_callback = TimeHistory()
    callbacks.append(time_callback)

    # Stop early and restore the best model weights when finished the training
    earlystopper = EarlyStopping(patience=cfg.TRAIN.PATIENCE, verbose=1, restore_best_weights=True)
    callbacks.append(earlystopper)

    # Save the best model into a h5 file in case one need again the weights learned
    os.makedirs(cfg.PATHS.CHECKPOINT, exist_ok=True)
    checkpointer = ModelCheckpoint(cfg.PATHS.CHECKPOINT_FILE, verbose=1, save_best_only=True)
    callbacks.append(checkpointer)

    return callbacks


from albert import *
import os
import pathlib
import shlex
import subprocess

md_iid = '1.0'
md_version = "1.8"
md_name = "Locate"
md_description = "Find and open files using locate"
md_license = "BSD-3"
md_url = "https://github.com/albertlauncher/python/tree/master/locate"
md_bin_dependencies = "locate"

class Plugin(TriggerQueryHandler):

    def id(self):
        return md_id

    def name(self):
        return md_name

    def description(self):
        return md_description

    def defaultTrigger(self):
        return "'"

    def synopsis(self):
        return "<locate params>"

    def initialize(self):
        self.icons = [
            "xdg:preferences-system-search",
            "xdg:system-search",
            "xdg:search",
            "xdg:text-x-generic",
            str(pathlib.Path(__file__).parent / "locate.svg")
        ]

    def handleTriggerQuery(self, query):
        if len(query.string) > 2:

            try:
                args = shlex.split(query.string)
            except ValueError:
                return

            result = subprocess.run(['locate', *args], stdout=subprocess.PIPE, text=True)
            if not query.isValid:
                return
            lines = sorted(result.stdout.splitlines(), reverse=True)
            if not query.isValid:
                return

            for path in lines:
                basename = os.path.basename(path)
                query.add(
                    Item(
                        id=path,
                        text=basename,
                        subtext=path,
                        icon=self.icons,
                        actions=[
                            Action("open", "Open", lambda p=path: openUrl("file://%s" % p))
                        ]
                    )
                )
        else:
            query.add(
                Item(
                    id="updatedb",
                    text="Update locate database",
                    subtext="Type at least three chars for a search",
                    icon=self.icons,
                    actions=[
                        Action("update", "Update", lambda: runTerminal("sudo updatedb"))
                    ]
                )
            )


import cv2
import numpy as np
from matplotlib import pyplot as plt
import os
os.chdir('C:\\Users\\sachi\\.vscode\\GitHubRepos\\OSCV_Exercises')
exetasknum = 1

# Contours can be explained simply as a curve joining all the continuous points (along the boundary), having same color or intensity. The contours are a useful tool for shape analysis and object detection and recognition.

# For better accuracy, use binary images. So before finding contours, apply threshold or canny edge detection.
# findContours function modifies the source image. So if you want source image even after finding contours, already store it to some other variables.
# In OpenCV, finding contours is like finding white object from black background. So remember, object to be found should be white and background should be black.

if exetasknum==1:
    im = cv2.imread('test.jpg')
    imgray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
    ret,thresh = cv2.threshold(imgray,127,255,0)
    contours,hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
    major = cv2.__version__.split('.')[0]
    if major == '3':
        ret, contours, hierarchy = cv2.findContours(im.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) #im.copy()->thresh
    else:
        contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)

    img = cv2.drawContours(im, contours, -1, (0,255,0), 3)

    while True:
        cv2.imshow('img', img)
        k = cv2.waitKey(1) & 0xFF
        if k == 27:
            break

    # img = cv2.drawContours(img, contours, -1, (0,255,0), 3)
    # while True:
    #     cv2.imshow('All Contours', img)
    #     k = cv2.waitKey(1) & 0xFF
    #     if k == 27:
    #         break

    # img = cv2.drawContours(img, contours, 0, (0,255,0), 3)
    # while True:
    #     cv2.imshow('1st Contour', img)
    #     k = cv2.waitKey(1) & 0xFF
    #     if k == 27:
    #         break

OpenCV_Python_Tutorials/017 Contours Getting Started.py


from copy import deepcopy
import numpy as np

def mark_points_in_diagram(coordinates: list[tuple],
    diagram: np.ndarray,
    consider_diagonal: bool = False):

    all_points = deepcopy(coordinates)

    x1, y1 = coordinates[0]
    x2, y2 = coordinates[1]
    # conform to artificial rule of x1 >= x2 for simpler computation later
    if x1 < x2:
        temp_x, temp_y = x1, y1
        x1, y1 = x2, y2
        x2, y2 = temp_x, temp_y

    if x1 == x2:
        for y in range(min(y1, y2) + 1, max(y1, y2)):
            all_points.append((x1, y))
    elif y1 == y2:
        for x in range(x2 + 1, x1):
            all_points.append((x, y1))
    elif consider_diagonal is False:
        return
    else:
        if x1 - x2 == y1 - y2:
            for x in range(x2 + 1, x1):
                all_points.append((x, y2 + (x - x2)))
        else:
            for x in range(x2 + 1, x1):
                all_points.append((x, y2 - (x - x2)))

    for p in set(all_points):
        x, y = p[0], p[1]
        diagram[y][x] += 1

def calculate_points(diagram):
    total = 0
    for i in np.nditer(diagram):
        if i >= 2:
            total += 1

    return total

if __name__ == "__main__":
    inputs = []
    max_x, max_y = 0, 0

    with open("input.txt") as f:
        for l in f.readlines():
            coordinates = []
            l = l.strip()
            for i in l.split(" -> "):
                pts = i.split(",")

                x, y = int(pts[0]), int(pts[1])
                if x > max_x: max_x = x
                if y > max_y: max_y = y

                coordinates.append((x, y))

            inputs.append(coordinates)

    diagram = np.zeros((max_y + 1, max_x + 1), dtype=int)

    for coordinates in inputs:
        mark_points_in_diagram(coordinates, diagram)

    num_points = calculate_points(diagram)

    print("Answer of AoC 2021 Day 5 Part 1:", num_points)

    diagram_2 = np.zeros((max_y + 1, max_x + 1), dtype=int)

    for coordinates in inputs:
        mark_points_in_diagram(coordinates, diagram_2, consider_diagonal=True)

    num_points_2 = calculate_points(diagram_2)

    print("Answer of AoC 2021 Day 5 Part 2:", num_points_2)


import requests

class ApiBaseActions:
    def __init__(self, base_url):
        if base_url.endswith("/"):
            self.base_url = base_url
        else:
            self.base_url = base_url + "/"
        self.session = requests.Session()

    def make_request(self, method: str, route_url: str = None, **kwargs):
        request_methods = {"GET": self.session.get,
                           "POST": self.session.post
                           }
        if route_url is not None:
            if route_url.startswith("/"):
                route_url = route_url[1:]
            reuest_url = self.base_url + route_url

        if request_methods.get(method) is not None:
            print(f" making {method} request to {reuest_url}")
            response = request_methods[method](reuest_url, **kwargs)
        else:
            raise NameError("Invalid API method provided")
        return response


from seqeval.metrics import classification_report
from seqeval.metrics import f1_score
from typing import List
import os

# the order for evaluating the script is in the main function

def create_fake_conll_form_without_iob_to_emNER_input():
    """tokenized sentences for emNER
    input: iob format"""

    with open("/NerKor/test.iob", "r", encoding="utf-8") as f:
        corpus = f.readlines()

    current = "form\n" # to the emNER form

    for i in range(len(corpus)):
        if corpus[i] != "\n":
            current += corpus[i].split("\t")[0] + "\n"
        else:
            current += "\n"

    with open("input_for_emNER_nerkor_test.conllup", "w", encoding="utf-8") as f:
        f.write(current)

def conll_conv2_iobes():
    """conll convert to iobes form"""

    if not os.path.exists("nekor_test.out"):
        return

    with open("nerkor_test.out", "r", encoding="utf-8") as f:
        corpus = f.readlines()

    current = ""

    for i in range(len(corpus)):
        if i == 0:
            continue
        if corpus[i] != "\n":
            current += corpus[i].split("\t")[0] + "\t" + corpus[i].split("\t")[-1]
        else:
            current += "\n"

    with open("emNER_nerkor_test.iobes", "w", encoding="utf-8") as f:
        f.write(current)

def iobes_convert2_iob():
    """emNER has an iobes output format, so we convert it to simple iob"""

    if not os.path.exists("emNER_nerkor_test.iobes"):
        return

    with open("emNER_nerkor_test.iobes", "r", encoding="utf-8") as f:
        corpus = f.readlines()

    with open("emNER_nerkor_test.iob", "w", encoding="utf-8") as f:
        for i in range(len(corpus)):
            if corpus[i] != "\n":
                line = corpus[i].split("\t")[0] + "\t" + corpus[i].split("\t")[1]
                if line.split("\t")[1].startswith("1"):
                    temp = line.split("\t")[1][1:]
                    line = corpus[i].split("\t")[0] + "\t" + "B" + temp
                if line.split("\t")[1].startswith("E"):
                    temp = line.split("\t")[1][1:]
                    line = corpus[i].split("\t")[0] + "\t" + "I" + temp
                f.write(line)
            else:
                f.write("\n")

def pred():
    if not os.path.exists("emNER_nerkor_test.iob"):
        return

    with open("emNER_nerkor_test.iob", "r", encoding="utf-8") as f:
        pred_iob = f.readlines()

    pred_list = list()
    current_list = list()

    for i in range(len(pred_iob)):
        if len(pred_iob[i].strip()) != 0:
            current_list.append(pred_iob[i].split("\t")[1][:-1])
        else:
            pred_list.append(current_list)
            current_list = list()

    print(len(pred_list))
    return pred_list

def gold():
    with open("/NerKor/test.iob", "r", encoding="utf-8") as f:
        gold_iob = f.readlines()

    gold_list = list()
    current_list = list()

    for i in range(len(gold_iob)):
        if len(gold_iob[i].strip()) != 0:
            current_list.append(gold_iob[i].split("\t")[1][:-1])
        else:
            gold_list.append(current_list)
            current_list = list()

    print(len(gold_list))
    return gold_list

def fscore(gold_iob: List[List[str]], pred_iob: List[List[str]]):
    print(f1_score(gold_iob, pred_iob))
    print(classification_report(gold_iob, pred_iob))

if __name__ == "__main__":
    # create_fake_conll_form_without_iob_to_emNER_input()
    # bash: cat input_for_emNER_nerkor_test.conllup | docker run -i mtaril/emtsv:latest emMorph,emTag,emNER > nerkor_test.out
    # conll_conv2_iob()
    # iobes_convert2_iob()
    fscore(gold(), pred())


"""Training a Transformer Builder model to be used for adversarial attacks."""
import argparse
import random
from functools import partial
from typing import Dict

import numpy as np
import torch
from sklearn.metrics import precision_recall_fscore_support, confusion_matrix
from torch.utils.data.sampler import BatchSampler
from tqdm import tqdm
from transformers import AdamW, get_constant_schedule_with_warmup
from transformers import BertForSequenceClassification
from transformers import BertTokenizer, BertConfig
from transformers import RobertaForSequenceClassification, RobertaConfig, \
    RobertaTokenizer

from builders.data_loader import collate_fever, FeverDataset, \
    BucketBatchSampler, sort_key

def train_model(model: torch.nn.Module,
                train_dl: BatchSampler, dev_dl: BatchSampler,
                optimizer: torch.optim.Optimizer,
                scheduler: torch.optim.lr_scheduler.LambdaLR,
                n_epochs: int) -> (Dict, Dict):
    best_val, best_model_weights = {'val_f1': 0}, None

    for ep in range(n_epochs):
        for i, batch in enumerate(tqdm(train_dl, desc='Training')):
            model.train()
            optimizer.zero_grad()
            loss, _ = model(batch[0],
                                 attention_mask=batch[
                                                    0] !=
                                                tokenizer.pad_token_id,
                                 labels=batch[1])

            loss.backward()

            optimizer.step()
            scheduler.step()

            if i in [600, 700, 100]:
                print(eval_model(model, dev_dl), flush=True)

        val_p, val_r, val_f1, val_loss = eval_model(model, dev_dl)
        current_val = {
            'val_f1': val_f1,
            'val_p': val_p,
            'val_r': val_r,
            'val_loss': val_loss,
            'ep': ep
        }
        print(current_val, flush=True)

        if current_val['val_f1'] > best_val['val_f1']:
            best_val = current_val
            best_model_weights = model.state_dict()

    return best_model_weights, best_val

def eval_model(model: torch.nn.Module, test_dl: BatchSampler):
    model.eval()
    loss_f = torch.nn.CrossEntropyLoss()
    with torch.no_grad():
        labels_all = []
        logits_all = []
        losses = []
        for batch in tqdm(test_dl, desc="Evaluation"):
            loss, logits_val = model(batch[0],
                                     attention_mask=batch[0] > 1,
                                     labels=batch[1])
            loss = loss_f(logits_val, batch[1].long())
            losses.append(loss.item())

            labels_all += batch[1].detach().cpu().numpy().tolist()
            logits_all += logits_val.detach().cpu().numpy().tolist()

        prediction = np.argmax(np.asarray(logits_all).reshape(-1, args.labels),
                               axis=-1)
        p, r, f1, _ = precision_recall_fscore_support(labels_all, prediction,
                                                      average='macro')
        print(confusion_matrix(labels_all, prediction), flush=True)

    return p, r, f1, np.mean(losses)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--gpu", help="Flag for training on gpu",
                        action='store_true', default=False)
    parser.add_argument("--seed", help="Random seed", type=int, default=73)
    parser.add_argument("--labels",
                        help="2 labels if NOT ENOUGH INFO excluded, "
                             "3 otherwise",
                        type=int, default=3)

    parser.add_argument("--train_dataset", help="Path to the train datasets",
                        default='data/train_nli.jsonl', type=str)
    parser.add_argument("--dev_dataset", help="Path to the dev datasets",
                        default='data/dev_nli.jsonl', type=str)
    parser.add_argument("--test_dataset", help="Path to the test datasets",
                        default='data/test_nli.jsonl', type=str)
    parser.add_argument("--type", help="Type of transformer model",
                        choices=['bert', 'roberta'], default='bert')

    parser.add_argument("--model_path",
                        help="Path where the model will be serialized",
                        default='ferver_bert', type=str)

    parser.add_argument("--batch_size", help="Batch size", type=int, default=8)
    parser.add_argument("--lr", help="Learning Rate", type=float, default=5e-5)
    parser.add_argument("--epochs", help="Epochs number", type=int, default=4)
    parser.add_argument("--mode", help="Mode for the script", type=str,
                        default='train', choices=['train', 'test'])

    args = parser.parse_args()

    random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)
    torch.backends.cudnn.deterministic = True
    np.random.seed(args.seed)

    device = torch.device("cuda") if args.gpu else torch.device("cpu")
    if args.type == 'bert':
        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
        transformer_config = BertConfig.from_pretrained('bert-base-uncased',
                                                        num_labels=args.labels)
        model = BertForSequenceClassification(transformer_config).to(device)
    else:
        tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
        transformer_config = RobertaConfig.from_pretrained('roberta-base',
                                                           num_labels=args.labels)  # , use_bfloat16=True
        model = RobertaForSequenceClassification.from_pretrained('roberta-base',
                                                                 config=transformer_config).to(
            device)

    collate_fn = partial(collate_fever, tokenizer=tokenizer, device=device)
    print(args, flush=True)

    param_optimizer = list(model.named_parameters())
    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {
            'params': [p for n, p in param_optimizer if
                       not any(nd in n for nd in no_decay)],
            'weight_decay': 0.01
        },
        {
            'params': [p for n, p in param_optimizer if
                       any(nd in n for nd in no_decay)], 'weight_decay': 0.0
        }]

    optimizer = AdamW(optimizer_grouped_parameters, lr=args.lr)

    if args.mode == 'test':
        test = FeverDataset(args.test_dataset)
        # print(Counter([_x['label'] for _x in test]).most_common(3))
        test_dl = BucketBatchSampler(batch_size=args.batch_size,
                                     sort_key=sort_key, dataset=test,
                                     collate_fn=collate_fn)

        checkpoint = torch.load(args.model_path)

        model.load_state_dict(checkpoint['model'])
        print(eval_model(model, test_dl))

    else:
        print("Loading datasets...")
        train = FeverDataset(args.train_dataset)
        dev = FeverDataset(args.dev_dataset)

        # print(Counter([_x['label'] for _x in train]).most_common(3))
        # print(Counter([_x['label'] for _x in dev]).most_common(3))

        train_dl = BucketBatchSampler(batch_size=args.batch_size,
                                      sort_key=sort_key,
                                      dataset=train,
                                      collate_fn=collate_fn)
        dev_dl = BucketBatchSampler(batch_size=args.batch_size,
                                    sort_key=sort_key, dataset=dev,
                                    collate_fn=collate_fn)

        num_train_optimization_steps = int(
            args.epochs * len(train) / args.batch_size)

        scheduler = get_constant_schedule_with_warmup(optimizer,
                                                      num_warmup_steps=0.05)

        best_model_w, best_perf = train_model(model, train_dl, dev_dl,
                                              optimizer, scheduler, args.epochs)

        checkpoint = {
            'performance': best_perf,
            'args': vars(args),
            'model': best_model_w,
        }
        print(best_perf)
        print(args)

        torch.save(checkpoint, args.model_path)


from rest_framework import serializers
from .models import Product, Ingredient

class IngredientSerializer(serializers.ModelSerializer):
    class Meta:
        model = Ingredient
        fields = ('title', 'price')

class ProductSerializer(serializers.ModelSerializer):
    ingredients = IngredientSerializer(read_only=True, many=True)

    class Meta:
        model = Product
        fields = ('id', 'product_type', 'title',
                  'image', 'description', 'price', 'ingredients')


"""
following file contains example commands used to use the library
it is focused on airfoil design, which uses parameteric definitions of airfoils and runs optimization procedures
"""
# 1. will build the airfoil using the cst parametric definition
# 1.1. define some arbitrary parameters for cst, as vectors P and Q:
P = [.03, .08, .12, .16, .06]
Q = [ .2, .25, .25,  .2, .1]

# here P is camber, Q is thickness distribution. more parameters can be added to allow more geometry flexibility
# 1.2. build airfoil of class CST, using default values of exponents: N1, N2, M1, M2 and 150 points:
from foils.airfoilCST import CSTairfoil, CSTcurve
airfoil = CSTairfoil(P, Q, N1 = .5, N2 = .5,  M1 = 1, M2 = 1, points = 150)
airfoil.plotAirfoil()

# 1.3. the resulting airfoil is nice and smooth, but it most likely requires some additional processing to be useful
# here i mean e.g. finite thickness trailing edge
# so lets build Airfoil class object basing on our airfoil, name it foil as airfoil is taken by CST object:
foil = airfoil.genAirfoil()
foil.plotAirfoil()

# 1.4. it is basically the same airfoil, but we now have plenty of options available for this generic Airfoil class object
# lets modify trailing edge to make it more realistic
foil.cutTE_XFOIL(t = .01, r = 0.25)
foil.plotAirfoil()

# several options are available for geometry modifications including: leading edge radius modification (e.g. to match required radius)
# thickenss and camber scaling (e.g. to match required maximum thickness), etc.

# 1.5. lets take a look at last part of Airfoil class- analysis
# there are options for xfoil and fluent analyses
# fluent requires software, which we might not have, so lets stick to xfoil and run our foil at re = 5e5 at angle of attack = 5 degrees:
foil.runXFOIL(alfa = 5, re=5e5)

# outputs -> (5.0 1.0132 0.01035 -0.1013), which means 1.0132 lift coefficient, 0.01035 drag coefficient and -0.1013 pitching moment coefficient
# a several options for running xfoil are defined as well, which allow to quickly run airfoil polar or run for desired lift coefficient

#==================================================================
# 2. similar procedure can be used to build b-spline based airfoil
from foils.airfoilBSpline import BSplineAirfoil
import numpy as np
# 2.1. lets define vectors of knots which define splines
knotsU = np.array([[0,0], [0., .04], [0.2, .1], [.4, .12],[.6,.1], [.8, .05] ,[1,0] ])
knotsL = np.array([[0,0], [0., -.025], [0.2, -.05], [.4, -0.06],[.6, -.01],[.8, .01], [1,0] ])

# and build BSplineAirfoil class foil:
airfoilb = BSplineAirfoil(knotsU, knotsL)
airfoilb.plotAirfoil()

# plot presents control polygon and resulting airfoil
# 2.2. the same procedure as for cst foil can be used to bspline foil:
foilb = airfoilb.genAirfoil()
foilb.plotAirfoil(name = 'bspline foil as Airfoil object')
foilb.cutTE_XFOIL(t = .01, r = 0.25)
foilb.plotAirfoil(name = 'thickened te of bspline foil')

# needless to mention, this library allows processing of airfoil to match required geoemtry parameters of any form:
# chord, maximum thickness, leading edge radius, trailing edge thickness, camber and thickness scaling, rotating, etc.
# hence it can be used to define airfoil stacking in 3d wing-type design cases.

#==================================================================
# 3. run airfoil design using evolutionary process
from foils.airfoilDesigner import airfoilDesigner
xs = [.02, .3, .9]
weights = [1, 1, 1]
maxiter = 80

design = airfoilDesigner([1],'CST', 10, ts = [.04, .11, .015], xs = xs, weights = weights)
design.runOptimization(maxiter = maxiter)