# coding:utf8
__author__ = 'Marcelo Ferreira da Costa Gomes'

# Apply filters of interest in a dataframe containing SINAN-SRAG data
import pandas as pd
import numpy as np
import argparse
import logging
import re
from argparse import RawDescriptionHelpFormatter
from .insert_epiweek import insert_epiweek
from .delay_table import extract_quantile, delayimputation, createtable

module_logger = logging.getLogger('update_system.sinan_filter_of_interest')
tabela_siglauf = {'RO': 11,
                 'AC': 12,
                 'AM': 13,
                 'RR': 14,
                 'PA': 15,
                 'AP': 16,
                 'TO': 17,
                 'MA': 21,
                 'PI': 22,
                 'CE': 23,
                 'RN': 24,
                 'PB': 25,
                 'PE': 26,
                 'AL': 27,
                 'SE': 28,
                 'BA': 29,
                 'MG': 31,
                 'ES': 32,
                 'RJ': 33,
                 'SP': 35,
                 'PR': 41,
                 'SC': 42,
                 'RS': 43,
                 'MS': 50,
                 'MT': 51,
                 'GO': 52,
                 'DF': 53,
                 'NI': 99}

filtro_dict = {1: '_hospdeath',
               2: '_sragnofever',
               3: ''}

dict_dtypes = {
    'NU_NOTIFIC': 'str',
    'DT_NOTIFIC': 'str',
    'SEM_NOT': 'Int64',
    'ANO': 'Int16',
    'DT_SIN_PRI': 'str',
    'SEM_PRI': 'Int64',
    'SG_UF_NOT': 'str',
    'ID_REGIONA': 'str',
    'CO_REGIONA': 'Int32',
    'ID_MUNICIP': 'str',
    'CO_MUN_NOT': 'Int32',
    'ID_UNIDADE': 'str',
    'CO_UNI_NOT': 'Int32',
    'NM_UN_INTE': 'str',
    'CO_UN_INTE': 'Int32',
    'LAB_AN': 'str',
    'CO_LAB_AN': 'Int32',
    'RES_CAPITAL': 'str',
    'NOT_CAPITAL': 'str',
    'NU_CPF': 'Int64',
    'NU_CNS': 'str',
    'NM_PACIENT': 'str',
    'CS_SEXO': 'str',
    'DT_NASC': 'str',
    'NU_IDADE_N': 'str',
    'TP_IDADE': 'Int8',
    'COD_IDADE': 'str',
    'CS_GESTANT': 'Int8',
    'CS_RACA': 'Int8',
    'CS_ETINIA': 'str',
    'TP_POV_CT': 'str',
    'CS_ESCOL_N': 'Int8',
    'NM_MAE_PAC': 'str',
    'NU_CEP': 'str',
    'ID_PAIS': 'str',
    'CO_PAIS': 'Int16',
    'SG_UF': 'str',
    'ID_RG_RESI': 'str',
    'CO_RG_RESI': 'Int16',
    'ID_MN_RESI': 'str',
    'CO_MUN_RES': 'Int32',
    'NM_BAIRRO': 'str',
    'NM_LOGRADO': 'str',
    'NU_NUMERO': 'str',
    'NM_COMPLEM': 'str',
    'NU_DDD_TEL': 'Int8',
    'NU_TELEFON': 'str',
    'CS_ZONA': 'Int8',
    'SURTO_SG': 'Int8',
    'NOSOCOMIAL': 'Int8',
    'AVE_SUINO': 'Int8',
    'FEBRE': 'Int8',
    'TOSSE': 'Int8',
    'GARGANTA': 'Int8',
    'DISPNEIA': 'Int8',
    'DESC_RESP': 'Int8',
    'SATURACAO': 'Int8',
    'DIARREIA': 'Int8',
    'VOMITO': 'Int8',
    'OUTRO_SIN': 'Int8',
    'OUTRO_DES': 'str',
    'PUERPERA': 'Int8',
    'FATOR_RISC': 'str',
    'CARDIOPATI': 'Int8',
    'HEMATOLOGI': 'Int8',
    'SIND_DOWN': 'Int8',
    'HEPATICA': 'Int8',
    'ASMA': 'Int8',
    'DIABETES': 'Int8',
    'NEUROLOGIC': 'Int8',
    'PNEUMOPATI': 'Int8',
    'IMUNODEPRE': 'Int8',
    'RENAL': 'Int8',
    'OBESIDADE': 'Int8',
    'OBES_IMC': 'str',
    'OUT_MORBI': 'Int8',
    'MORB_DESC': 'str',
    'VACINA': 'Int8',
    'DT_UT_DOSE': 'str',
    'MAE_VAC': 'Int8',
    'DT_VAC_MAE': 'str',
    'M_AMAMENTA': 'Int8',
    'DT_DOSEUNI': 'str',
    'DT_1_DOSE': 'str',
    'DT_2_DOSE': 'str',
    'ANTIVIRAL': 'Int8',
    'TP_ANTIVIR': 'Int8',
    'OUT_ANTIV': 'str',
    'DT_ANTIVIR': 'str',
    'HOSPITAL': 'Int8',
    'DT_INTERNA': 'str',
    'SG_UF_INTE': 'str',
    'ID_RG_INTE': 'str',
    'CO_RG_INTE': 'Int16',
    'ID_MN_INTE': 'str',
    'CO_MU_INTE': 'Int32',
    'UTI': 'Int8',
    'DT_ENTUTI': 'str',
    'DT_SAIDUTI': 'str',
    'SUPORT_VEN': 'Int8',
    'RAIOX_RES': 'Int8',
    'RAIOX_OUT': 'str',
    'DT_RAIOX': 'str',
    'AMOSTRA': 'Int8',
    'DT_COLETA': 'str',
    'TP_AMOSTRA': 'Int8',
    'OUT_AMOST': 'str',
    'REQUI_GAL': 'Int64',
    'IF_RESUL': 'Int8',
    'DT_IF': 'str',
    'POS_IF_FLU': 'Int8',
    'TP_FLU_IF': 'Int8',
    'POS_IF_OUT': 'Int8',
    'IF_VSR': 'Int8',
    'IF_PARA1': 'Int8',
    'IF_PARA2': 'Int8',
    'IF_PARA3': 'Int8',
    'IF_ADENO': 'Int8',
    'IF_OUTRO': 'Int8',
    'DS_IF_OUT': 'object',
    'LAB_IF': 'str',
    'CO_LAB_IF': 'Int64',
    'PCR_RESUL': 'Int8',
    'DT_PCR': 'str',
    'POS_PCRFLU': 'Int8',
    'TP_FLU_PCR': 'Int8',
    'PCR_FLUASU': 'Int8',
    'FLUASU_OUT': 'str',
    'PCR_FLUBLI': 'Int8',
    'FLUBLI_OUT': 'str',
    'POS_PCROUT': 'Int8',
    'PCR_VSR': 'Int8',
    'PCR_PARA1': 'Int8',
    'PCR_PARA2': 'Int8',
    'PCR_PARA3': 'Int8',
    'PCR_PARA4': 'Int8',
    'PCR_ADENO': 'Int8',
    'PCR_METAP': 'Int8',
    'PCR_BOCA': 'Int8',
    'PCR_RINO': 'Int8',
    'PCR_OUTRO': 'Int8',
    'DS_PCR_OUT': 'str',
    'LAB_PCR': 'str',
    'CO_LAB_PCR': 'Int64',
    'CLASSI_FIN': 'Int8',
    'CLASSI_OUT': 'str',
    'CRITERIO': 'Int8',
    'EVOLUCAO': 'Int8',
    'DT_EVOLUCA': 'str',
    'NU_DO': 'Int64',
    'DT_ENCERRA': 'str',
    'OBSERVA': 'str',
    'NOME_PROF': 'str',
    'REG_PROF': 'str',
    'DT_DIGITA': 'str',
    'HISTO_VGM': 'Int8',
    'PAIS_VGM': 'str',
    'CO_PS_VGM': 'Int16',
    'LO_PS_VGM': 'str',
    'DT_VGM': 'str',
    'DT_RT_VGM': 'str',
    'PCR_SARS2': 'Int8',
    'PAC_COCBO': 'str',
    'PAC_DSCBO': 'str',
    'OUT_ANIM': 'str',
    'DOR_ABD': 'Int8',
    'FADIGA': 'Int8',
    'PERD_OLFT': 'Int8',
    'PERD_PALA': 'Int8',
    'TOMO_RES': 'Int8',
    'TOMO_OUT': 'str',
    'DT_TOMO': 'str',
    'TP_TES_AN': 'Int8',
    'DT_RES_AN': 'str',
    'RES_AN': 'Int8',
    'POS_AN_FLU': 'Int8',
    'TP_FLU_AN': 'Int8',
    'POS_AN_OUT': 'Int8',
    'AN_SARS2': 'Int8',
    'AN_VSR': 'Int8',
    'AN_PARA1': 'Int8',
    'AN_PARA2': 'Int8',
    'AN_PARA3': 'Int8',
    'AN_ADENO': 'Int8',
    'AN_OUTRO': 'Int8',
    'DS_AN_OUT': 'str',
    'TP_AM_SOR': 'Int8',
    'SOR_OUT': 'str',
    'DT_CO_SOR': 'str',
    'TP_SOR': 'Int8',
    'OUT_SOR': 'str',
    'DT_RES': 'str',
    'RES_IGG': 'Int8',
    'RES_IGM': 'Int8',
    'RES_IGA': 'Int8',
    'DT_ATUALIZACAO': 'str',
    'DOSE_1_COV': 'str',
    'DOSE_2_COV': 'str',
    'LOTE_1_COV': 'str',
    'LOTE_2_COV': 'str',
    'LAB_PR_COV': 'str',
    'VACINA_COV': 'Int8',
    '@VERSION': 'Int64',
    '@TIMESTAMP': 'str'
}

def eq_notna(s: pd.Series, x):
    return pd.notna(s) & s.eq(x)

def ne_orna(s: pd.Series, x):
    return pd.isna(s) | s.ne(x)

def clean_open_field(s: pd.Series):

    specialchar = '\@|\$|\%|\&|\*'
    s = s.where(~s.str.contains(specialchar, regex=True, na=False), 'CENSURADO')

    return s

def check_covid19_regex(df: pd.DataFrame, column: str):

    if sum(~pd.isnull(df[column])) > 0:
        sars_cov_2_regex = 'novo corona|covid|coovid|cov-2|covd-19|' \
                           'cov2|cov 2|cov- 2|cov -2|cov - 2|c0v-2|cav-2|cav 2|arvs-cov|ars- cov|ars cov|ars - ' \
                           'cov|ars-cov|ars2cov|covi-19|civid 19|cobid-19|covis|cov-2|civid-19'
        sars_cov_2_regex_discard = 'nao detectado para covid19'
        row_filter = (df[column].str.contains(sars_cov_2_regex.upper(), regex=True, na=False)) & \
                     (~df[column].str.contains(sars_cov_2_regex_discard.upper(), regex=True, na=False))
    else:
        row_filter = ~pd.isnull(df[column])

    return row_filter

def readtable(fname, sep=','):
    try:
        df = pd.read_csv(fname, sep=sep, dtype=dict_dtypes, low_memory=False, encoding='utf-8')
    except:
        df = pd.read_csv(fname, sep=sep, dtype=dict_dtypes, low_memory=False, encoding='utf-16')

    return df

def date_cleanup(df, dt_cols):
    '''
    Standardize column data and discard those without notification and/or first symptoms date.
    :param df: notifications data frame
    :param dt_cols: list of data columns
    :return: df: standardized data frame
    '''

    # Filter by notification date
    df = df.where(df != -1, None)
    df[dt_cols] = df[dt_cols].where(df[dt_cols] != 10101, None)
    if df.CRITERIO.dtype == 'O':
        df.CRITERIO = df.CRITERIO.where(df.CRITERIO != 'NÃ', None)
    df.dropna(subset=["DT_SIN_PRI", "DT_NOTIFIC"], inplace=True)

    # Convert all date related columns to datetime format
    for col in dt_cols:
        # Convert all date columns to datetime format. Output will have the format YYYY-MM-DD
        dtformat = '%Y%m%d'
        if sum(~pd.isnull(df[col])) > 0:
            sample = df.loc[~pd.isnull(df[col]), col].values[0]
            if isinstance(sample, str):
                if 'T' in sample:
                    df[col] = pd.to_datetime(df[col].str[:10], errors='coerce', format='%Y-%m-%d')
                else:
                    dtsep = '-'
                    if '/' in sample:
                        dtsep = '/'
                    dttest = pd.DataFrame(list(
                        df.loc[~pd.isnull(df[col]), col].str.split(dtsep)
                    ))
                    maxvals = [int(dttest[i].max()) for i in range(3)]
                    del dttest
                    yearpos = maxvals.index(max(maxvals))
                    if yearpos == 2:
                        if maxvals[1] > 12:
                            dtformat = '%m' + dtsep + '%d' + dtsep + '%Y'
                        else:
                            dtformat = '%d' + dtsep + '%m' + dtsep + '%Y'
                    else:
                        dtformat = '%Y' + dtsep + '%m' + dtsep + '%d'
                    df[col] = pd.to_datetime(df[col], errors='coerce', format=dtformat)
            else:
                df[col] = pd.to_datetime(df[col], errors='coerce', format=dtformat)
        else:
            df[col] = pd.to_datetime(df[col])
    # Discard those neither hospitalized nor deceased. For cases from 2009, keep all:
    df = df[(df.DT_SIN_PRI.apply(lambda x: x.year) == 2009) | (eq_notna(df.HOSPITAL, 1)) | (eq_notna(df.EVOLUCAO, 2))]

    return df

def table_compatibility(df):
    '''
    Create column conversion to make old and new data compatible
    :param df: input data frame
    :return:
    '''

    # Create equivalence between 2020-07-27 new variables and old ones:
    df['DT_IF'] = df['DT_RES_AN'].copy()
    if 'RES_AN' in df.columns:
        df['IF_RESUL'] = df['RES_AN'].copy()

    l_new = ['CO_LAB_AN',
             'POS_AN_FLU',
             'TP_FLU_AN',
             'POS_AN_OUT',
             'AN_VSR',
             'AN_PARA1',
             'AN_PARA2',
             'AN_PARA3',
             'AN_ADENO',
             'AN_OUTRO',
             'DS_AN_OUT']
    cols = df.columns
    col_new = []
    col_old = []
    l_intersec = set(l_new).intersection(cols)
    for li in l_intersec:
        col_new.append(li.replace('AN', 'IF'))
        col_old.append(li)

    if len(l_intersec) > 0:
        df[col_new] = df[col_old].copy()

    return

def symptoms_filter(df):

    # Filter by symptoms:
    # All cases, regardless of year, must either attend symptoms definition or have evolved to
    # death. Besides that, cases from 2009 do not need hospitalization to be considered as true case, per Ministry
    # of Health request. For all other years, they must have been hospitalized or have evolved to death.
    # The filter regarding hospitalization and case evolution is done after date columns consolidation to attend
    # 2009's particularity.

    # all
    df['filtro'] = 0
    # hospdeath:
    df.loc[(eq_notna(df.EVOLUCAO, 2)) | (eq_notna(df.HOSPITAL, 1)), 'filtro'] = 1
    # sragnofever:
    df.loc[(df.filtro == 1) & (
            ((eq_notna(df.TOSSE, 1)) | (eq_notna(df.GARGANTA, 1))) &
            ((eq_notna(df.DISPNEIA, 1)) | (eq_notna(df.SATURACAO, 1)) | (eq_notna(df.DESC_RESP, 1)))
    ), 'filtro'] = 2
    # srag:
    df.loc[(df.filtro == 2) & (eq_notna(df.FEBRE, 1)), 'filtro'] = 3

    return df

def filter_db2019(df, tag=None, filtertype='srag'):
    tgtcols = ['AMOSTRA', 'ANTIVIRAL', 'AVE_SUINO', 'CLASSI_FIN', 'CLASSI_OUT', 'CO_LAB_IF', 'CO_LAB_PCR', 'CO_MU_INTE',
               'CO_MUN_NOT', 'CO_MUN_RES', 'CO_PAIS', 'CO_REGIONA', 'CO_RG_RESI', 'CO_UNI_NOT', 'CRITERIO', 'CS_ETINIA',
               'CO_UN_INTE',
               'CS_RACA', 'CS_SEXO', 'DS_IF_OUT', 'DS_PCR_OUT', 'DT_ANTIVIR',
               'DT_COLETA', 'DT_DIGITA', 'DT_ENCERRA', 'DT_EVOLUCA', 'DT_IF', 'DT_INTERNA', 'DT_NOTIFIC', 'DT_PCR',
               'DT_IFI',
               'DT_SIN_PRI', 'DT_UT_DOSE', 'EVOLUCAO', 'FLUASU_OUT', 'FLUBLI_OUT', 'HOSPITAL',
               'ID_MN_RESI', 'ID_MUNICIP', 'ID_REGIONA', 'ID_RG_RESI', 'IF_ADENO', 'IF_OUTRO', 'IF_PARA1',
               'IF_PARA2', 'IF_PARA3', 'IF_RESUL', 'IF_VSR', 'NU_CEP', 'NU_IDADE_N', 'NU_NOTIFIC', 'NM_BAIRRO',
               'NM_LOGRADO', 'OUT_ANTIVIR', 'PCR_ADENO', 'PCR_BOCA', 'PCR_FLUASU', 'PCR_FLUBLI', 'PCR_METAP',
               'PCR_OUTRO', 'PCR_PARA1',
               'PCR_PARA2', 'PCR_PARA3', 'PCR_PARA4', 'PCR_RESUL', 'PCR_RINO', 'PCR_SARS2', 'PCR_VSR', 'POS_IF_FLU',
               'POS_IF_OUT', 'POS_PCRFLU', 'POS_PCROUT', 'SEM_NOT', 'SEM_PRI', 'SG_UF', 'SG_UF_NOT',
               'TP_ANTIVIR', 'TP_FLU_IF', 'TP_FLU_PCR', 'TP_IDADE', 'VACINA']

    tgt_cols_sintomas = ['DESC_RESP',
                         'DISPNEIA',
                         'DIARREIA',
                         'FEBRE',
                         'GARGANTA',
                         'TOSSE',
                         'SATURACAO',
                         'DIARREIA',
                         'VOMITO',
                         'DOR_ABD',
                         'FADIGA',
                         'PERD_OLFT',
                         'PERD_PALA',
                         'OUTRO_SIN',
                         'OUTRO_DES']

    tgtcols_uti = ['UTI',
                   'DT_ENTUTI',
                   'DT_SAIDUTI',
                   'SUPORT_VEN']

    tgtcols_comorb = ['CS_GESTANT',
                      'FATOR_RISC',
                      'PUERPERA',
                      'CARDIOPATI',
                      'HEMATOLOGI',
                      'HEPATICA',
                      'SIND_DOWN',
                      'HEPATICA',
                      'ASMA',
                      'DIABETES',
                      'NEUROLOGIC',
                      'PNEUMOPATI',
                      'IMUNODEPRE',
                      'RENAL',
                      'OBESIDADE',
                      'OBES_IMC',
                      'OUT_MORBI',
                      'MORB_DESC']

    tgtcols_2020_07_27 = ['TP_TES_AN',
                          'DT_RES_AN',
                          'RES_IGA',
                          'RES_IGG',
                          'RES_IGM',
                          'LAB_AN',
                          'CO_LAB_AN',
                          'POS_AN_FLU',
                          'TP_FLU_AN',
                          'POS_AN_OUT',
                          'AN_SARS2',
                          'AN_VSR',
                          'AN_PARA1',
                          'AN_PARA2',
                          'AN_PARA3',
                          'AN_ADENO',
                          'AN_OUTRO',
                          'DS_AN_OUT',
                          'SUPORT_VEN',
                          'RAIOX_RES',
                          'RAIOX_OUT',
                          'DT_RAIOX',
                          'TOMO_RES',
                          'TOMO_OUT',
                          'DT_TOMO']
    tgt_cols_vac_covid = ['VACINA_COV', 'DOSE_1_COV', 'DOSE_2_COV', 'LAB_PR_COV', 'FNT_IN_COV']

    if 'DT_RES_AN' in df.columns:
        table_compatibility(df)

    tgtcols = list(set(tgtcols).union(tgt_cols_sintomas).union(tgtcols_uti).union(tgtcols_comorb).union(
        tgtcols_2020_07_27).union(tgt_cols_vac_covid))
    if 'DT_IF' in df.columns:
        df['DT_IFI'] = df.DT_IF

    cols = df.columns
    for col in set(tgtcols).difference(cols):
        df[col] = None

    df = df[tgtcols].copy()
    df = symptoms_filter(df)

    regexp = re.compile('^DT')
    dt_cols = list(filter(regexp.match, tgtcols))
    df = date_cleanup(df, dt_cols)

    # Registro da idade, em anos.
    # Campo TP_IDADE indica a escala da variável NU_IDADE_N, sendo:
    # 1-Dias, 2-Meses, 3-Anos
    df['idade_em_anos'] = df.NU_IDADE_N.where(df.TP_IDADE == 3, 0)

    # If sample collection field is empty, convert to unknown:
    df.AMOSTRA = df.AMOSTRA.where(pd.notnull(df.AMOSTRA), 9).astype(int)

    # Convert UF abbreviation to numeric code:
    df.loc[pd.notnull(df.SG_UF_NOT), 'SG_UF_NOT'] = df.loc[pd.notnull(df.SG_UF_NOT), 'SG_UF_NOT'].map(tabela_siglauf)
    df.loc[pd.isnull(df.SG_UF_NOT), 'SG_UF_NOT'] = 99
    df.loc[pd.notnull(df.SG_UF), 'SG_UF'] = df.loc[pd.notnull(df.SG_UF), 'SG_UF'].map(tabela_siglauf)
    df.loc[pd.isnull(df.SG_UF), 'SG_UF'] = 99

    # Clean up PCR_RESUL and IF_RESUL fields:
    def labresultcleanup(dfin, x):
        # If sample was collected, IF/PCR result cannot be unknown or empty:
        dfin.loc[(eq_notna(dfin.AMOSTRA, 1)) & (~dfin[x].isin([1, 2, 3, 4])), x] = 5
        # IF PCR/IF result field is marked unknown but sample was not collected, convert to not tested:
        dfin.loc[(eq_notna(dfin[x], 9)) & (eq_notna(dfin.AMOSTRA, 2)), x] = 4
        # If PCR/IF result field is empty and sample was NOT collected, convert to not tested
        dfin[x] = dfin[x].where(pd.notnull(dfin[x]) | (ne_orna(dfin.AMOSTRA, 2)), 4)
        # If PCR/IF result field is empty and sample field is empty or unknown, convert to unknown
        dfin[x] = dfin[x].where(pd.notnull(dfin[x]) | (dfin.AMOSTRA.isin([1, 2])), 9)

        return

    labresultcleanup(df, 'PCR_RESUL')
    labresultcleanup(df, 'IF_RESUL')

    df['FLU_A'] = pd.Series([], dtype='Int8')
    df['FLU_B'] = pd.Series([], dtype='Int8')
    df['FLU_LAB'] = pd.Series([], dtype='Int8')
    df['VSR'] = pd.Series([], dtype='Int8')
    df['PARA1'] = pd.Series([], dtype='Int8')
    df['PARA2'] = pd.Series([], dtype='Int8')
    df['PARA3'] = pd.Series([], dtype='Int8')
    df['PARA4'] = pd.Series([], dtype='Int8')
    df['ADNO'] = pd.Series([], dtype='Int8')
    df['METAP'] = pd.Series([], dtype='Int8')
    df['BOCA'] = pd.Series([], dtype='Int8')
    df['RINO'] = pd.Series([], dtype='Int8')
    df['SARS2'] = pd.Series([], dtype='Int8')
    df['OTHERS'] = pd.Series([], dtype='Int8')

    df['NEGATIVE'] = pd.Series([], dtype='Int8')
    df['POSITIVE'] = pd.Series([], dtype='Int8')
    df['INCONCLUSIVE'] = pd.Series([], dtype='Int8')
    df['DELAYED'] = pd.Series([], dtype='Int8')
    df['TESTED'] = pd.Series([], dtype='Int8')
    df['NOTTESTED'] = pd.Series([], dtype='Int8')
    df['TESTING_IGNORED'] = pd.Series([], dtype='Int8')

    def labresult(x, y=None, pos=1):
        if all(pd.isnull([x, y])):
            return None
        elif pos in [pd.notna(x), pd.notna(y)]:
            return 1
        else:
            return 0

    mask = (df[['TP_FLU_PCR', 'TP_FLU_IF']].notna().sum(axis=1) >= 1)
    df.loc[mask, 'FLU_A'] = df.loc[mask, 'FLU_A'] = (
            eq_notna(df.TP_FLU_PCR[mask], 1) | eq_notna(df.TP_FLU_IF[mask], 1)
    ).astype('Int8')
    df.loc[mask, 'FLU_B'] = df.loc[mask, 'FLU_A'] = (
            eq_notna(df.TP_FLU_PCR[mask], 2) | eq_notna(df.TP_FLU_IF[mask], 2)
    ).astype('Int8')

    df.loc[(eq_notna(df.POS_IF_FLU, 1)) |
           (eq_notna(df.POS_PCRFLU, 1)) |
           (eq_notna(df.FLU_A, 1)) |
           (eq_notna(df.FLU_B, 1)) |
           ((eq_notna(df.CLASSI_FIN, 1)) & (eq_notna(df.CRITERIO, 1))), 'FLU_LAB'] = 1
    df.loc[(ne_orna(df.FLU_LAB, 1)) & ((eq_notna(df.POS_IF_FLU, 2)) | (eq_notna(df.POS_PCRFLU, 2))), 'FLU_LAB'] = 0
    df.loc[(eq_notna(df.IF_VSR, 1)) | (eq_notna(df.PCR_VSR, 1)), 'VSR'] = 1
    df.loc[(eq_notna(df.IF_PARA1, 1)) | (eq_notna(df.PCR_PARA1, 1)), 'PARA1'] = 1
    df.loc[(eq_notna(df.IF_PARA2, 1)) | (eq_notna(df.PCR_PARA2, 1)), 'PARA2'] = 1
    df.loc[(eq_notna(df.IF_PARA3, 1)) | (eq_notna(df.PCR_PARA3, 1)), 'PARA3'] = 1
    df.loc[eq_notna(df.PCR_PARA4, 1), 'PARA4'] = 1
    df.loc[(eq_notna(df.IF_ADENO, 1)) | (eq_notna(df.PCR_ADENO, 1)), 'ADNO'] = 1
    df.loc[eq_notna(df.PCR_BOCA, 1), 'BOCA'] = 1
    df.loc[eq_notna(df.PCR_RINO, 1), 'RINO'] = 1
    df.loc[eq_notna(df.PCR_METAP, 1), 'METAP'] = 1
    df.loc[(eq_notna(df.IF_OUTRO, 1)) | (eq_notna(df.PCR_OUTRO, 1)), 'OTHERS'] = 1

    mask_covid19_ds_pcr = check_covid19_regex(df, 'DS_PCR_OUT')
    mask_covid19_ds_if = check_covid19_regex(df, 'DS_IF_OUT')

    df.loc[(eq_notna(df.PCR_SARS2, 1)) |
           (eq_notna(df.AN_SARS2, 1)) |
           mask_covid19_ds_pcr |
           mask_covid19_ds_if |
           ((eq_notna(df.CLASSI_FIN, 5)) & (eq_notna(df.CRITERIO, 1))),
           'SARS2'] = 1
    df.loc[(eq_notna(df.PCR_SARS2, 1)) |
           mask_covid19_ds_pcr |
           mask_covid19_ds_if,
           'OTHERS'] = pd.NA

    # Positive cases:
    df.loc[(eq_notna(df.POS_PCRFLU, 1)) | (eq_notna(df.POS_PCROUT, 1)), 'PCR_RESUL'] = 1
    df.loc[(eq_notna(df.POS_IF_FLU, 1)) | (eq_notna(df.POS_IF_OUT, 1)), 'IF_RESUL'] = 1
    df.loc[(eq_notna(df.PCR_RESUL, 1)) |
           (eq_notna(df.IF_RESUL, 1)) |
           (eq_notna(df.FLU_LAB, 1)) |
           (eq_notna(df.SARS2, 1)) |
           (eq_notna(df.VSR, 1)) |
           (eq_notna(df.PARA1, 1)) |
           (eq_notna(df.PARA2, 1)) |
           (eq_notna(df.PARA3, 1)) |
           (eq_notna(df.PARA4, 1)) |
           (eq_notna(df.ADNO, 1)) |
           (eq_notna(df.BOCA, 1)) |
           (eq_notna(df.RINO, 1)) |
           (eq_notna(df.METAP, 1)) |
           (eq_notna(df.OTHERS, 1)),
           'POSITIVE'] = 1

    # Negative cases:
    df.loc[(eq_notna(df.POS_PCRFLU, 2)) & (eq_notna(df.POS_PCROUT, 2)), 'PCR_RESUL'] = 2
    df.loc[(eq_notna(df.POS_IF_FLU, 2)) & (eq_notna(df.POS_IF_OUT, 2)), 'IF_RESUL'] = 2
    mask = (
            (ne_orna(df.POSITIVE, 1)) &
            ((eq_notna(df.FLU_LAB, 0)) | (eq_notna(df.POS_PCROUT, 2)) | (eq_notna(df.POS_IF_OUT, 2)))
    )
    df.loc[mask, 'NEGATIVE'] = 1

    df.loc[(((eq_notna(df.IF_RESUL, 2)) & (df.PCR_RESUL.isin([2, 3, 4, 5, 9]))) |
           ((eq_notna(df.PCR_RESUL, 2)) & (df.IF_RESUL.isin([2, 3, 4, 5, 9])))) &
           (ne_orna(df.POSITIVE, 1)), 'NEGATIVE'] = 1
    df.loc[:, 'INCONCLUSIVE'] = (
                                    (
                                            ((eq_notna(df.IF_RESUL, 3)) & (~df.PCR_RESUL.isin([1, 2, 5]))) |
                                            ((eq_notna(df.PCR_RESUL, 3)) & (~df.IF_RESUL.isin([1, 2, 5])))
                                    ) &
                                    (ne_orna(df.POSITIVE, 1)) &
                                    (ne_orna(df.NEGATIVE, 1))
    ).astype(int)
    df.loc[(eq_notna(df.POSITIVE, 1)) | (eq_notna(df.NEGATIVE, 1)) | (eq_notna(df.INCONCLUSIVE, 1)), 'TESTED'] = 1

    df.loc[:, 'DELAYED'] = (
                               (
                                       ((eq_notna(df.IF_RESUL, 5)) & (pd.isnull(df.TESTED))) |
                                       ((eq_notna(df.PCR_RESUL, 5)) & (pd.isnull(df.TESTED)))
                               ) &
                               (ne_orna(df.POSITIVE, 1)) &
                               (ne_orna(df.NEGATIVE, 1)) &
                               (ne_orna(df.INCONCLUSIVE, 1))
    ).astype(int)

    # Clinical and clinical-epidemiological diagnose:
    df['FLU_CLINIC'] = ((ne_orna(df.POS_IF_FLU, 1)) & (ne_orna(df.POS_PCRFLU, 1)) & (eq_notna(df.CLASSI_FIN, 1)) &
                        (df.CRITERIO.isin([2, 3]))).astype(int)

    notknownrows = (
                           ((eq_notna(df.PCR_RESUL, 9)) | pd.isnull(df.PCR_RESUL)) &
                           ((eq_notna(df.IF_RESUL, 9)) | pd.isnull(df.IF_RESUL)) &
                           (ne_orna(df.POSITIVE, 1)) &
                           (ne_orna(df.NEGATIVE, 1)) &
                           (ne_orna(df.DELAYED, 1)) &
                           (ne_orna(df.INCONCLUSIVE, 1))
    )
    nottestedrows = (
            ~(notknownrows) &
            (df.PCR_RESUL.isin([4, 9])) &
            (df.IF_RESUL.isin([4, 9])) &
            (ne_orna(df.TESTED, 1))
    )
    df['NOTTESTED'] = nottestedrows.astype(int)
    df['TESTING_IGNORED'] = notknownrows.astype(int)

    if tag:
        df['tag'] = tag

    return df

def applysinanfilter(df, tag=None, filtertype='srag'):
    # Filter columns of interest
    # Na solicitação, além das variáveis abaixo, necessitamos também do ID do caso
    tgtcols = ['SEM_NOT', 'DT_NOTIFIC', 'SG_UF_NOT', 'DT_INTERNA', 'DT_SIN_PRI', 'DT_DIGITA', 'HOSPITAL',
               'FEBRE', 'CLASSI_FIN', 'CRITERIO', 'SG_UF', 'ID_MN_RESI', 'ID_RG_RESI', 'SEM_PRI',
               'TOSSE', 'GARGANTA', 'DISPNEIA', 'SATURACAO', 'DESC_RESP', 'EVOLUCAO', 'DT_COLETA', 'IFI', 'DT_IFI',
               'PCR', 'OUT_METODO', 'DS_OUTMET', 'DT_OUTMET', 'RES_FLUA', 'RES_FLUASU', 'RES_FLUB',
               'RES_VSR', 'RES_PARA1', 'RES_PARA2', 'RES_PARA3', 'RES_ADNO', 'RES_OUTRO', 'DT_PCR', 'PCR_RES',
               'PCR_ETIOL', 'PCR_TIPO_H', 'PCR_TIPO_N', 'DT_CULTURA', 'CULT_RES', 'DT_HEMAGLU', 'HEMA_RES',
               'HEMA_ETIOL', 'HEM_TIPO_H', 'HEM_TIPO_N', 'VACINA', 'DT_UT_DOSE', 'ANT_PNEUMO', 'DT_PNEUM',
               'CO_UF_INTE', 'CO_MU_INTE', 'CO_UN_INTE', 'DT_ENCERRA', 'NU_NOTIFIC', 'ID_AGRAVO', 'ID_MUNICIP',
               'ID_REGIONA', 'ID_UNIDADE', 'NU_IDADE_N', 'NM_BAIRRO', 'NM_LOGRADO', 'CS_SEXO', 'CS_RACA',
               'DT_ANTIVIR', 'DT_EVOLUCA']

    tgtcols_uti = ['UTI',
                   'DT_ENTUTI',
                   'DT_SAIDUTI']

    tgtcols_comorb = ['CS_GESTANT',
                      'PUERPERA',
                      'CARDIOPATI',
                      'HEMATOLOGI',
                      'SIND_DOWN',
                      'HEPATICA',
                      'ASMA',
                      'DIABETES',
                      'NEUROLOGIC',
                      'PNEUMOPATI',
                      'IMUNODEPRE',
                      'RENAL',
                      'OBESIDADE',
                      'OBES_IMC',
                      'OUT_MORBI',
                      'MORB_DESC']

    tgtcols = list(set(tgtcols).union(tgtcols_uti).union(tgtcols_comorb))

    cols = df.columns
    if 'RES_VRS' in cols:
        df.rename(columns={'RES_VRS': 'RES_VSR'}, inplace=True)
    if 'DT_PCR_1' in cols:
        df.DT_PCR.update(df.DT_PCR_1)
    if 'DT_OBITO' in cols:
        df.rename(columns={'DT_OBITO': 'DT_EVOLUCA'}, inplace=True)
        if 'DT_EVOL' in cols:
            df.DT_EVOLUCA.update(df.DT_EVOL)
    elif 'DT_EVOL' in cols:
        df.rename(columns={'DT_EVOL': 'DT_EVOLUCA'}, inplace=True)
    if 'METABOLICA' in cols:
        df.rename(columns={'METABOLICA': 'DIABETES'}, inplace=True)

    cols = df.columns
    for col in set(tgtcols).difference(cols):
        df[col] = None

    df = df[tgtcols].copy()

    df = symptoms_filter(df)

    regexp = re.compile('^DT')
    dt_cols = list(filter(regexp.match, tgtcols))
    df = date_cleanup(df, dt_cols)

    # Create columns related to lab result
    # Rows with lab test:
    labrows = ((df.PCR_RES.isin([1, 2, 3])) |
               (df.CULT_RES.isin([1, 2])) |
               (df.HEMA_RES.isin([1, 2, 3])) |
               (eq_notna(df.IFI, 1)) |
               (eq_notna(df.PCR, 1)) |
               (eq_notna(df.OUT_METODO, 1)))

    notknownrows = (
        pd.isnull(df.PCR_RES) &
        pd.isnull(df.CULT_RES) &
        pd.isnull(df.HEMA_RES) &
        pd.isnull(df.IFI) &
        pd.isnull(df.PCR) &
        pd.isnull(df.OUT_METODO)
    )

    nottestedrows = (
        ~(notknownrows) &
        (pd.isnull(df.PCR_RES) | (df.PCR_RES.isin([4]))) &
        (pd.isnull(df.CULT_RES) | (df.CULT_RES.isin([3]))) &
        (pd.isnull(df.HEMA_RES) | (df.HEMA_RES.isin([4]))) &
        (pd.isnull(df.IFI) | (eq_notna(df.IFI, 2))) &
        (pd.isnull(df.PCR) | (eq_notna(df.PCR, 2))) &
        (pd.isnull(df.OUT_METODO) | (eq_notna(df.OUT_METODO, 2)))
    )

    df['FLU_A'] = pd.Series([], dtype='Int8')
    df['FLU_B'] = pd.Series([], dtype='Int8')
    df['FLU_LAB'] = pd.Series([], dtype='Int8')
    df['VSR'] = pd.Series([], dtype='Int8')
    df['PARA1'] = pd.Series([], dtype='Int8')
    df['PARA2'] = pd.Series([], dtype='Int8')
    df['PARA3'] = pd.Series([], dtype='Int8')
    df['PARA4'] = pd.Series([], dtype='Int8')
    df['ADNO'] = pd.Series([], dtype='Int8')
    df['METAP'] = pd.Series([], dtype='Int8')
    df['BOCA'] = pd.Series([], dtype='Int8')
    df['RINO'] = pd.Series([], dtype='Int8')
    df['SARS2'] = pd.Series([], dtype='Int8')
    df['OTHERS'] = pd.Series([], dtype='Int8')

    df['NEGATIVE'] = pd.Series([], dtype='Int8')
    df['POSITIVE'] = pd.Series([], dtype='Int8')
    df['INCONCLUSIVE'] = pd.Series([], dtype='Int8')
    df['DELAYED'] = pd.Series([], dtype='Int8')
    df['TESTED'] = pd.Series([], dtype='Int8')
    df['NOTTESTED'] = pd.Series([], dtype='Int8')
    df['TESTING_IGNORED'] = pd.Series([], dtype='Int8')

    df['TESTED'] = labrows.astype(int)
    df['NOTTESTED'] = nottestedrows.astype(int)
    df['TESTING_IGNORED'] = notknownrows.astype(int)

    df.loc[labrows, 'FLU_A'] = ((df.PCR_ETIOL[labrows].isin([1, 2, 4])) | (df.HEMA_ETIOL[labrows].isin([1, 2, 4])) |
                                (eq_notna(df.RES_FLUA[labrows], 1))).astype(int)
    df.loc[labrows, 'FLU_B'] = ((eq_notna(df.PCR_ETIOL[labrows], 3)) | (eq_notna(df.HEMA_ETIOL[labrows], 3)) |
                                (eq_notna(df.RES_FLUB[labrows], 1))).astype(int)
    df.loc[labrows, 'VSR'] = (eq_notna(df.RES_VSR[labrows], 1)).astype(int)
    df.loc[labrows, 'PARA1'] = (eq_notna(df.RES_PARA1[labrows], 1)).astype(int)
    df.loc[labrows, 'PARA2'] = (eq_notna(df.RES_PARA2[labrows], 1)).astype(int)
    df.loc[labrows, 'PARA3'] = (eq_notna(df.RES_PARA3[labrows], 1)).astype(int)
    df.loc[labrows, 'ADNO'] = (eq_notna(df.RES_ADNO[labrows], 1)).astype(int)
    df.loc[labrows, 'SARS2'] = 0

    df.loc[labrows, 'OTHERS'] = (
        (eq_notna(df.PCR_ETIOL[labrows], 5)) |
        (eq_notna(df.HEMA_ETIOL[labrows], 5)) |
        (eq_notna(df.RES_OUTRO[labrows], 1))).astype(int)
    df.loc[labrows, 'DELAYED'] = ((pd.isnull(df.PCR_RES[labrows]) | eq_notna(df.PCR_RES[labrows], 4)) &
                                  (pd.isnull(df.HEMA_RES[labrows]) | eq_notna(df.HEMA_RES[labrows], 4)) &
                                  (pd.isnull(df.RES_FLUA[labrows]) | eq_notna(df.RES_FLUA[labrows], 4)) &
                                  (pd.isnull(df.RES_FLUB[labrows]) | eq_notna(df.RES_FLUB[labrows], 4)) &
                                  (pd.isnull(df.RES_VSR[labrows]) | eq_notna(df.RES_VSR[labrows], 4)) &
                                  (pd.isnull(df.RES_PARA1[labrows]) | eq_notna(df.RES_PARA1[labrows], 4)) &
                                  (pd.isnull(df.RES_PARA2[labrows]) | eq_notna(df.RES_PARA2[labrows], 4)) &
                                  (pd.isnull(df.RES_PARA3[labrows]) | eq_notna(df.RES_PARA3[labrows], 4)) &
                                  (pd.isnull(df.RES_ADNO[labrows]) | eq_notna(df.RES_ADNO[labrows], 4)) &
                                  (pd.isnull(df.RES_OUTRO[labrows]) | eq_notna(df.RES_OUTRO[labrows], 4))).astype(int)
    df.loc[labrows, 'INCONCLUSIVE'] = ((eq_notna(df.DELAYED[labrows], 0)) &
                                       (pd.isnull(df.PCR_RES[labrows]) | df.PCR_RES[labrows].isin([3, 4])) &
                                       (pd.isnull(df.HEMA_RES[labrows]) | df.HEMA_RES[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_FLUA[labrows]) | df.RES_FLUA[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_FLUB[labrows]) | df.RES_FLUB[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_VSR[labrows]) | df.RES_VSR[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_PARA1[labrows]) | df.RES_PARA1[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_PARA2[labrows]) | df.RES_PARA2[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_PARA3[labrows]) | df.RES_PARA3[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_ADNO[labrows]) | df.RES_ADNO[labrows].isin([3, 4])) &
                                       (pd.isnull(df.RES_OUTRO[labrows]) | df.RES_OUTRO[labrows].isin([3, 4]))).astype(
        int)
    df.loc[labrows, 'NEGATIVE'] = ((eq_notna(df.FLU_A[labrows], 0)) &
                                   (eq_notna(df.FLU_B[labrows], 0)) &
                                   (eq_notna(df.VSR[labrows], 0)) &
                                   (eq_notna(df.PARA1[labrows], 0)) &
                                   (eq_notna(df.PARA2[labrows], 0)) &
                                   (eq_notna(df.PARA3[labrows], 0)) &
                                   (eq_notna(df.ADNO[labrows], 0)) &
                                   (eq_notna(df.OTHERS[labrows], 0)) &
                                   (eq_notna(df.DELAYED[labrows], 0)) &
                                   (eq_notna(df.INCONCLUSIVE[labrows], 0))).astype(int)
    df.loc[labrows &
           (ne_orna(df.INCONCLUSIVE, 1)) &
           (ne_orna(df.NEGATIVE, 1)) &
           (ne_orna(df.DELAYED, 1)),
           'POSITIVE'] = 1

    # Clinical and clinical-epidemiological diagnose:
    df['FLU_CLINIC'] = ((ne_orna(df.FLU_A, 1)) &
                        (ne_orna(df.FLU_B, 1)) &
                        (eq_notna(df.CLASSI_FIN, 1)) &
                        (df.CRITERIO.isin([2, 3]))).astype(int)

    df.NU_IDADE_N = df.NU_IDADE_N.astype(np.float)

    def f_idade(x):

        # System registers age with the following format:
        # TAAA
        # T: type (1-hours, 2-days, 3-months, 4-years)
        #   Hours used only if age < 24h, days used only if 24h <= age < 30d, months only if 30d <= a < 12 months
        # AAA: count on the respective scale
        # Ex.:
        # 2010 : 10 days
        # 3010: 10 months
        # 4010: 10 years
        if np.isnan(x):
            return np.nan
        if x < 4000:
            return 0
        else:
            return x - 4000

    if (df.NU_IDADE_N > 4000).any(axis = 0):
        df['idade_em_anos'] = df['NU_IDADE_N'].apply(f_idade)
    else:
        df['idade_em_anos'] = df['NU_IDADE_N']
    if tag:
        df['tag'] = tag

    return df

def delays_dist(df_in=pd.DataFrame(), filtertype='srag', append_file=None, dfappend=None):
    module_logger.info('Entered function: delay_dist')

    if filtertype not in ['srag', 'sragnofever', 'hospdeath']:
        exit('Invalid filter type: %s' % filtertype)

    suff = ''
    if filtertype != 'srag':
        suff = '_%s' % filtertype

    df_reg = pd.read_csv('../data/regioesclimaticas.csv', low_memory=False)[['Código', 'Região', 'Região oficial']]
    df_reg.rename(columns={'Código': 'UF', 'Região': 'Regional', 'Região oficial': 'Regiao'}, inplace=True)

    def _cleanup(df_input):
        tgt_cols = ['SG_UF_NOT',
                    'DT_NOTIFIC_epiyearweek',
                    'DT_NOTIFIC_epiyear',
                    'DT_NOTIFIC_epiweek',
                    'DT_SIN_PRI_epiyearweek',
                    'DT_SIN_PRI_epiyear',
                    'DT_SIN_PRI_epiweek',
                    'DT_DIGITA_epiyearweek',
                    'DT_DIGITA_epiyear',
                    'DT_DIGITA_epiweek',
                    'Notific2Digita_DelayWeeks',
                    'SinPri2Digita_DelayWeeks',
                    'SinPri2Antivir_DelayWeeks',
                    'SinPri2Notific_DelayWeeks',
                    'SinPri2Coleta_DelayWeeks',
                    'SinPri2Interna_DelayWeeks',
                    'Notific2Encerra_DelayWeeks',
                    'Coleta2IFI_DelayWeeks',
                    'Coleta2PCR_DelayWeeks',
                    'Notific2Coleta_DelayWeeks',
                    'Notific2Antivir_DelayWeeks',
                    'Digita2Antivir_DelayWeeks',
                    'Interna2Evoluca_DelayWeeks',

                    'Notific2Digita_DelayDays',
                    'SinPri2Digita_DelayDays',
                    'SinPri2Antivir_DelayDays',
                    'SinPri2Notific_DelayDays',
                    'SinPri2Coleta_DelayDays',
                    'SinPri2Interna_DelayDays',
                    'Notific2Encerra_DelayDays',
                    'Coleta2IFI_DelayDays',
                    'Coleta2PCR_DelayDays',
                    'Notific2Coleta_DelayDays',
                    'Notific2Antivir_DelayDays',
                    'Digita2Antivir_DelayDays',
                    'Interna2Evoluca_DelayDays',

                    'sragflu',
                    'obitoflu',
                    'sragcovid',
                    'obitocovid',
                    'obito',
                    'filtro']
        df = df_input[tgt_cols].rename(columns={'DT_NOTIFIC_epiyear': 'epiyear',
                                                'DT_NOTIFIC_epiweek': 'epiweek',
                                                'SG_UF_NOT': 'UF'})

        df.loc[pd.isnull(df.UF), 'UF'] = 99
        df.UF = df.UF.astype(float).astype(int)
        # Insert region info:
        df = df.merge(df_reg, on='UF')
        df['Pais'] = 'BR'

        cols = ['DT_DIGITA_epiyear', 'DT_DIGITA_epiweek', 'SinPri2Digita_DelayWeeks']
        try:
            df[cols] = df[cols].astype(int)
        except:
            df[cols] = df[cols].astype(float)
        cols = ['epiyear', 'epiweek']
        try:
            df[cols] = df[cols].astype(int)
        except:
            df[cols] = df[cols].astype(float)
        cols = ['DT_SIN_PRI_epiyear', 'DT_SIN_PRI_epiweek']
        try:
            df[cols] = df[cols].astype(int)
        except:
            df[cols] = df[cols].astype(float)
        return df

    df = _cleanup(df_in)

    def wide2long(dfwide2long):

        dfwide2long['id'] = df.index
        dfwide2long['srag'] = 1
        varcols = ['srag',
                   'sragflu',
                   'obitoflu',
                   'sragcovid',
                   'obitocovid',
                   'obito']
        dfwide2long_out = dfwide2long[['id'] + varcols].melt(id_vars='id',
                                                             var_name='dado',
                                                             value_name='valor')[lambda x: (x.valor == 1)]
        dfwide2long_out = dfwide2long_out.merge(dfwide2long, how='left', on='id').drop(columns=['valor', 'id']+varcols)
        dfwide2long.drop(columns=['id', 'srag'], inplace=True)

        return dfwide2long_out

    df_out = wide2long(df)

    out_cols = ['Coleta2IFI_DelayDays', 'Coleta2IFI_DelayWeeks', 'Coleta2PCR_DelayDays',
                'Coleta2PCR_DelayWeeks', 'DT_DIGITA_epiweek', 'DT_DIGITA_epiyear',
                'DT_DIGITA_epiyearweek', 'DT_NOTIFIC_epiyearweek', 'DT_SIN_PRI_epiweek',
                'DT_SIN_PRI_epiyear', 'DT_SIN_PRI_epiyearweek',
                'Digita2Antivir_DelayDays', 'Digita2Antivir_DelayWeeks',
                'Notific2Antivir_DelayDays', 'Notific2Antivir_DelayWeeks',
                'Notific2Coleta_DelayDays', 'Notific2Coleta_DelayWeeks',
                'Notific2Digita_DelayDays', 'Notific2Digita_DelayWeeks',
                'Notific2Encerra_DelayDays', 'Notific2Encerra_DelayWeeks',
                'SinPri2Antivir_DelayDays', 'SinPri2Antivir_DelayWeeks',
                'SinPri2Coleta_DelayDays', 'SinPri2Coleta_DelayWeeks',
                'SinPri2Digita_DelayDays', 'SinPri2Digita_DelayWeeks',
                'SinPri2Notific_DelayDays', 'SinPri2Notific_DelayWeeks',
                'SinPri2Interna_DelayDays', 'SinPri2Interna_DelayWeeks',
                'Interna2Evoluca_DelayDays', 'Interna2Evoluca_DelayWeeks',
                'UF', 'dado', 'filtro',
                'epiweek', 'epiyear', 'Regional', 'Regiao', 'Pais']

    df_out = df_out.sort_values(by=['dado', 'UF', 'DT_SIN_PRI_epiyearweek', 'DT_NOTIFIC_epiyearweek',
                                    'DT_DIGITA_epiyearweek'])
    df_out = df_out.reset_index()[out_cols]
    if append_file:
        tmp = pd.read_csv(append_file)
        df_out = tmp.append(df_out[out_cols], ignore_index=True, sort=False)

    for k, suffix in filtro_dict.items():
        df_out.loc[(df_out.filtro >= k), out_cols].to_csv('../../data/data/delay_table%s.csv' % suffix, index=False)

    #### Opportunities estimation
    tgt_cols = ['UF', 'Regional', 'Regiao', 'Pais', 'dado', 'filtro', 'DT_SIN_PRI_epiyearweek', 'DT_SIN_PRI_epiyear',
                'DT_SIN_PRI_epiweek', 'SinPri2Digita_DelayWeeks', 'DT_DIGITA_epiyearweek', 'DT_DIGITA_epiyear',
                'DT_DIGITA_epiweek']
    # Generate quantiles' file:
    df_out = df_out[tgt_cols].rename(columns={'DT_SIN_PRI_epiyearweek': 'epiyearweek', 'DT_SIN_PRI_epiyear': 'epiyear',
                                     'DT_SIN_PRI_epiweek': 'epiweek', 'SinPri2Digita_DelayWeeks': 'delayweeks'})
    df_out.UF = df_out.UF.astype(int).astype(str)

    for i, ft in enumerate(['hospdeath', 'sragnofever', 'srag']):
        extract_quantile(df_out[df_out.filtro >= (i+1)], ft)

    del df_out

    # Impute digitalization date if needed:
    if dfappend is not None:
        dfappend = _cleanup(dfappend)
        df = dfappend.append(df, ignore_index=True, sort=False)

    opp_cols = list(set(tgt_cols).difference(['dado']).union(['sragflu', 'obitoflu', 'sragcovid', 'obitocovid',
                                                              'obito']))
    df = df[opp_cols].rename(columns={'DT_SIN_PRI_epiyearweek': 'epiyearweek', 'DT_SIN_PRI_epiyear': 'epiyear',
                                      'DT_SIN_PRI_epiweek': 'epiweek', 'SinPri2Digita_DelayWeeks': 'delayweeks'})
    df.UF = df.UF.astype(int).astype(str)
    df = delayimputation(df)

    df_out = wide2long(df)

    # Create tables and save to file:
    for k, suffix in filtro_dict.items():
        df_outnew = createtable(df_out[df_out.filtro >= k])
        for dataset in df_out.dado.unique():
            fout = '../clean_data/%s%s_sinpri2digita_table_weekly.csv' % (dataset, suffix)
            module_logger.info('Write table %s' % fout)
            df_outnew[df_outnew.dado == dataset].to_csv(fout, index=False)

    return

def main(flist, sep=',', yearmax=None, filtertype='srag', append_cases=None, append_delay=None):

    if filtertype not in ['srag', 'sragnofever', 'hospdeath']:
        module_logger.error('Invalid filter type: %s', filtertype)
        exit(1)

    suff = ''
    if filtertype != 'srag':
        suff = '_%s' % filtertype

    df = pd.DataFrame()
    for fname in flist:
        module_logger.info('Processing database file: %s', fname)
        dftmp = readtable(fname, sep)
        # Check if data file has 2019's database or not:
        if int(re.findall(r'\d+', fname)[0]) < 2019:
            module_logger.info('DB pre-2019')
            df = df.append(applysinanfilter(dftmp, tag=fname, filtertype=filtertype), ignore_index=True, sort=True)
        else:
            module_logger.info('DB 2019 onwards')
            df = df.append(filter_db2019(dftmp, tag=fname, filtertype=filtertype), ignore_index=True, sort=True)

    del dftmp

    if yearmax:
        df = df[(df.DT_SIN_PRI.apply(lambda x: x.year) <= yearmax)]

    df.NM_LOGRADO = clean_open_field(df.NM_LOGRADO)
    df.DS_PCR_OUT = clean_open_field(df.DS_PCR_OUT)
    df.DS_IF_OUT = clean_open_field(df.DS_IF_OUT)
    df.DS_AN_OUT = clean_open_field(df.DS_AN_OUT)
    df.OUTRO_DES = clean_open_field(df.OUTRO_DES)
    df.MORB_DESC = clean_open_field(df.MORB_DESC)
    df.RAIOX_OUT = clean_open_field(df.RAIOX_OUT)
    df.TOMO_OUT = clean_open_field(df.TOMO_OUT)

    # Clean obvious duplicates
    df.drop_duplicates(inplace=True, ignore_index=True)

    # Convert date fields to text and leave NaT as empty cell
    regexp = re.compile('^DT')
    target_cols = list(filter(regexp.match, df.columns))
    df[target_cols] = df[target_cols].applymap(lambda x: str(x.date())).where(lambda x: x != 'NaT', np.nan)
    df = insert_epiweek(df)

    if 'ID_UNIDADE' in df.columns:
        df.CO_UNI_NOT.update(df.ID_UNIDADE)

    mask_sanity_check = (
            (eq_notna(df.SARS2, 1)) &
            (
                    (df.DT_SIN_PRI_epiyear.astype(int) < 2020) |
                    ((df.DT_SIN_PRI_epiyear.astype(int) == 2020) &
                     (df.DT_SIN_PRI_epiweek.astype(int) < 8))
            )
    )
    if sum((df.SARS2[mask_sanity_check])) > 1:
        print('ATENÇÃO: Caso de COVID-19 anterior a 2020 08. Entrar em contato com GT-Influenza')
        df.loc[mask_sanity_check, 'SARS2'] = 0

    if append_cases:
        tmp = readtable(append_cases)
        dfout = tmp.append(df, ignore_index=True)
    else:
        dfout = df.copy()

    mask_flu = ((eq_notna(dfout.FLU_A, 1)) |
                (eq_notna(dfout.FLU_B, 1)) |
                (eq_notna(dfout.FLU_CLINIC, 1)))
    mask_covid = (eq_notna(dfout.SARS2, 1))
    mask_obito = (eq_notna(dfout.EVOLUCAO, 2))
    mask_obitoflu = mask_flu & mask_obito
    mask_obitocovid = mask_covid & mask_obito

    for k, suffix in filtro_dict.items():
        dfout[dfout.filtro >= k].to_csv('../clean_data/clean_data_srag%s_epiweek.csv' % suffix, index=False)
        dfout[(dfout.filtro >= k) & mask_flu].to_csv('../clean_data/clean_data_sragflu%s_epiweek.csv' % suffix, index=False)
        dfout[(dfout.filtro >= k) & mask_obitoflu].to_csv('../clean_data/clean_data_obitoflu%s_epiweek.csv' % suffix,
                                                    index=False)
        dfout[(dfout.filtro >= k) & mask_covid].to_csv('../clean_data/clean_data_sragcovid%s_epiweek.csv' % suffix,
                                                 index=False)
        dfout[(dfout.filtro >= k) & mask_obitocovid].to_csv('../clean_data/clean_data_obitocovid%s_epiweek.csv' % suffix,
                                                      index=False)
        dfout[(dfout.filtro >= k) & mask_obito].to_csv('../clean_data/clean_data_obito%s_epiweek.csv' % suffix, index=False)
    del dfout

    def masks(dfin: pd.DataFrame):
        mask_flu = ((eq_notna(dfin.FLU_A, 1)) |
                    (eq_notna(dfin.FLU_B, 1)) |
                    (eq_notna(dfin.FLU_CLINIC, 1)))
        mask_covid = (eq_notna(dfin.SARS2, 1))
        mask_obito = (eq_notna(dfin.EVOLUCAO, 2))
        mask_obitoflu = mask_flu & mask_obito
        mask_obitocovid = mask_covid & mask_obito
        dfin['obito'] = mask_obito.astype(int)
        dfin['sragflu'] = mask_flu.astype(int)
        dfin['obitoflu'] = mask_obitoflu.astype(int)
        dfin['sragcovid'] = mask_covid.astype(int)
        dfin['obitocovid'] = mask_obitocovid.astype(int)

        return

    masks(df)
    if append_cases:
        masks(tmp)
        delays_dist(df, filtertype, append_delay, dfappend=tmp)
    else:
        delays_dist(df, filtertype)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Clean SINAN SRAG table.\n" +
                                                 "python3 sinan_clean.py --path ../data/influ*.csv --sep ,\n",
                                     formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('--path', nargs='*', action='append', help='Path to data file')
    parser.add_argument('--sep', help='Column separator', default=',')
    parser.add_argument('--year', help='Maximum year', default=None)
    parser.add_argument('--filtertype', help='Default=srag. Which filter should be used? [srag, sragnofever, '
                                             'hospdeath]', default='srag')
    args = parser.parse_args()
    print(args)
    main(args.path[0], args.sep, args.year, args.filtertype)


import base64
import json
import re
from datetime import datetime, timezone, timedelta
from typing import Optional, Any

from .log import Log

def lowercase(string): return str(string).lower()

def uppercase(string): return str(string).upper()

def snakecase(string):
    string = re.sub(r"[\-\.\s]", '_', str(string))
    if not string:
        return string
    return lowercase(string[0]) + re.sub(r"[A-Z]", lambda matched: '_' + lowercase(matched.group(0)), string[1:])

def camelcase(string):
    string = re.sub(r"\w[\s\W]+\w", '', str(string))
    if not string:
        return string
    return lowercase(string[0]) + re.sub(r"[\-_\.\s]([a-z])", lambda matched: uppercase(matched.group(1)), string[1:])

def capitalcase(string: str):
    string = str(string)
    if not string:
        return string
    return uppercase(string[0]) + string[1:]

def pascalcase(string: str): return capitalcase(camelcase(string))

def titlecase(string): return ' '.join([capitalcase(word) for word in snakecase(string).split("_")])

def clean_camelcase(key: str):
    use_key = camelcase(key)
    if use_key[-1] == '_':
        use_key = use_key[0:-1]
    return use_key

def ex_message(e: Exception):
    if hasattr(e, 'message'):
        return e.message
    return str(e)

def log(o: Any):
    print(o)
    return o

def index_of(target: str, needle: str):
    try:
        return target.index(needle)
    except ValueError:
        return -1

def last_index_of(target: str, needle: str):
    try:
        return target.rindex(needle)
    except ValueError:
        return -1

def left_part(str_val: Optional[str], needle: str):
    if str_val is None:
        return None
    pos = index_of(str_val, needle)
    return str_val if pos == -1 else str_val[:pos]

def right_part(str_val: Optional[str], needle: str):
    if str_val is None:
        return None
    pos = index_of(str_val, needle)
    return str_val if pos == -1 else str_val[pos + len(needle):]

def last_left_part(str_val: Optional[str], needle: str):
    if str_val is None:
        return None
    pos = last_index_of(str_val, needle)
    return str_val if pos == -1 else str_val[:pos]

def last_right_part(str_val: Optional[str], needle: str):
    if str_val is None:
        return None
    pos = last_index_of(str_val, needle)
    return str_val if pos == -1 else str_val[pos + len(needle):]

def split_on_first(s: Optional[str], c: str):
    if str is None or str == "":
        return [s]
    pos = index_of(s, c)
    if pos >= 0:
        return [s[:pos], s[pos + 1:]]
    return [s]

def split_on_last(s: Optional[str], c: str):
    if str is None or str == "":
        return [s]
    pos = last_index_of(s, c)
    if pos >= 0:
        return [s[:pos], s[pos + 1:]]
    return [s]

def to_timespan(duration: timedelta):
    total_seconds = duration.total_seconds()
    whole_seconds = total_seconds // 1
    seconds = whole_seconds
    sec = int(seconds % 60 if seconds >= 60 else seconds)
    seconds = seconds // 60
    min = int(seconds % 60)
    seconds = seconds // 60
    hours = int(seconds % 60)
    days = seconds // 24
    remaining_secs = float(sec + (total_seconds - whole_seconds))

    sb = ["P"]
    if days > 0:
        sb.append(f"{days}D")

    if days == 0 or hours + min + sec + remaining_secs > 0:
        sb.append("T")
        if hours > 0:
            sb.append(f"{hours}H")
        if min > 0:
            sb.append(f"{min}M")

        if remaining_secs > 0:
            sec_fmt = "{:.7f}".format(remaining_secs)
            sec_fmt = sec_fmt.rstrip('0')
            sec_fmt = sec_fmt.rstrip('.')
            sb.append(sec_fmt)
            sb.append("S")
        elif len(sb) == 2:  # PT
            sb.append("0S")

    xsd = ''.join(sb)
    # print(f"XSD: {xsd}, {days}:{hours}:{min}:{remaining_secs}")
    return xsd

def from_timespan(s: Optional[str]):
    if s is None:
        return None
    days = 0
    hours = 0
    minutes = 0
    seconds = 0
    ms = 0.0

    if s[0] != "P":
        raise ValueError(f"{s} is not a valid XSD Duration")

    s = s[1:]  # strip P
    t = split_on_first(s, 'T')
    has_time = len(t) == 2

    d = split_on_first(t[0], 'D')
    if len(d) == 2:
        days = int(d[0])

    if has_time:
        h = split_on_first(t[1], 'H')
        if len(h) == 2:
            hours = int(h[0])

        m = split_on_first(h[len(h) - 1], 'M')
        if len(m) == 2:
            minutes = int(m[0])

        s = split_on_first(m[len(m) - 1], 'S')
        if len(s) == 2:
            ms = float(s[0])

        seconds = int(ms)
        ms -= seconds

    # print(f"\n\ntimedelta({str})[{has_time}] = {hours}:{minutes}:{seconds}\n\n")
    return timedelta(days=days, hours=hours, minutes=minutes, seconds=seconds, milliseconds=int(ms * 1000))

_MIN_UTC_DATE = datetime.min.replace(tzinfo=timezone.utc)
_MIN_EPOCH = _MIN_UTC_DATE.timestamp()
_MAX_UTC_DATE = datetime.max.replace(tzinfo=timezone.utc)

def to_datetime(date: datetime):
    try:
        return f"/Date({int(date.timestamp() * 1000)})/"
    except Exception as e:
        Log.debug(f"to_datetime({date}): e")
        return None

def from_datetime(json_date: str):
    if json_date.startswith("/Date("):
        epoch_and_zone = left_part(right_part(json_date, "("), ")")
        epoch_str = epoch_and_zone
        if index_of(epoch_and_zone[1:], '-') >= 0:
            epoch_str = last_left_part(epoch_and_zone, '-')
        if index_of(epoch_and_zone[1:], '+') >= 0:
            epoch_str = last_left_part(epoch_and_zone, '+')
        epoch = int(epoch_str)
        try:
            return datetime.fromtimestamp(epoch / 1000, timezone.utc)
        except Exception as e:
            if epoch < _MIN_EPOCH:
                return _MIN_UTC_DATE
            else:
                return _MAX_UTC_DATE

    # need to reduce to 6f precision and remove trailing Z
    has_sec_fraction = index_of(json_date, '.') >= 0
    is_utc = json_date.endswith('Z')
    if is_utc:
        json_date = json_date[0:-1]
    if has_sec_fraction:
        sec_fraction = last_right_part(json_date, '.')
        tz = ''
        if '+' in sec_fraction:
            tz = '+' + right_part(sec_fraction, '+')
            sec_fraction = left_part(sec_fraction, '+')
        elif '-' in sec_fraction:
            sec_fraction = left_part(sec_fraction, '-')
        if len(sec_fraction) > 6:
            json_date = last_left_part(json_date, '.') + '.' + sec_fraction[0:6] + tz

    if is_utc:
        return datetime.fromisoformat(json_date).replace(tzinfo=timezone.utc)
    else:
        return datetime.fromisoformat(json_date)

def to_bytearray(value: Optional[bytes]):
    if value is None:
        return None
    return base64.b64encode(value).decode('ascii')

def from_bytearray(base64str: Optional[str]):
    return base64.b64decode(base64str)

def from_base64url_safe(input_str: str):
    output = input_str
    output = output.replace('-', '+')
    output = output.replace('_', '/')
    pad = len(output) % 4
    if pad == 2:
        output += "=="
    elif pad == 3:
        output += "="
    elif pad != 0:
        raise ValueError("Illegal base46url string!")
    return base64.b64decode(output)

def _decode_base64url_payload(payload: str):
    payload_bytes = from_base64url_safe(payload)
    payload_json = payload_bytes.decode('utf-8')
    return json.loads(payload_json)

def inspect_jwt(jwt: str):
    head = _decode_base64url_payload(left_part(jwt, '.'))
    body = _decode_base64url_payload(left_part(right_part(jwt, '.'), '.'))
    exp = int(body['exp'])
    return head, body, datetime.fromtimestamp(exp, timezone.utc)


import os
import json
import requests

class app_info_mode():
    def __init__(self):
        self.path = os.path.abspath(os.path.dirname(__file__))
        self.mode = "translate"
        self.print_format = "t_mode"
        self.version = "v2.0"
        self.fromlang = 'en'
        self.tolang = 'zh'
        #get the appid&seckey;if none set before,then input
        #and store in info.json,if stored in info.json,then
        #read it
    def translate_init(self):
        fd = open(self.path+"/info.json","r")
        info_dict = json.load(fd)
        info_private_appid = info_dict["private"]["appid"]
        info_private_seckey = info_dict["private"]["seckey"]
        if info_private_appid == info_private_seckey:
            self.appid = input("[translate mode]set the appid:")
            self.seckey = input("[translate mode]set the seckey:")
            fd.close()

            info_dict["private"]["appid"]=self.appid
            info_dict["private"]["seckey"]=self.seckey

            #make the str that will write into the info.json
            #file a easy format to read
            info_str = str()
            for i in json.dumps(info_dict):
                info_str = info_str + i
                if i == ',':
                    info_str = info_str+'\n'

            #wirte the str into the info.json
            fd = open(self.path+"/info.json","w")
            fd.write(info_str)
            fd.close()
        else:
            self.appid = info_private_appid
            self.seckey = info_private_seckey
            fd.close()

    def robot_token_update(self):
        api_id = input("[robot mode]set api_id:")
        seckey = input("[robot mode]set seckey:")
        url = "https://aip.baidubce.com/oauth/2.0/token"
        req_url = url+f"?grant_type=client_credentials&client_id={api_id}&client_secret={seckey}"
        payload = ""
        headers = {'Content-Type': 'application/json','Accept': 'application/json'}
        response = requests.request("POST", req_url, headers=headers, data=payload)
        response = json.loads(response.text)

        if response["access_token"]:
           self.token = response["access_token"]
           fd = open(self.path+"/info.json",'r')
           info_dict = json.load(fd)
           fd.close()
           info_dict["token"]=response["access_token"]
           info_dict = json.dumps(info_dict)
           info_str = str()

           for i in info_dict:
               info_str = info_str+i
               if i == ',':
                   info_str=info_str+'\n'

           fd = open(self.path+"/info.json",'w')
           fd.write(info_str)
           fd.close()
    def token_get(self):
        fd = open(self.path+"/info.json",'r')
        info_dict = json.load(fd)
        #print(info_dict["token"])
        if info_dict["token"]=="none":
            #print(info_dict["token"])
            self.robot_token_update()
        else:
            #print('111')
            self.token = info_dict["token"]

if __name__ == "__main__":
    app_info = app_info_mode()
    print(app_info.appid,app_info.seckey)


# Polinomial Linear Regression

# 1) Importing Libraries

import numpy as np                 # Mathematics fucntopns
import matplotlib.pyplot as plt    # Plot and Charts
import pandas as pd                #Data Set management and import them
import seaborn as sns              # Covariance Matrix

# 2) importing Dataset
poli_dataset = pd.read_csv('Position_Salaries.csv')
X = poli_dataset.iloc[:,1:2].values   # dependend variable matrix means that will helo in prediction [rows , columns]
Y = poli_dataset.iloc[:,2].values      # independent variable vector means that need to be predicted [rows , columns]
#poli_dataset.iloc[:,1:2].values  we specify 1:2 in it to make it matrix ,upper bound will not be included so relax

# 3)Taking Care of Missing Data (We take mean of that column and put in missing area thats simple)
from sklearn.preprocessing import Imputer  # importing class for data missing values
imputer = Imputer(missing_values = 'NaN' , strategy = 'mean' , axis = 0)   # making object of that class
imp_1 = imputer.fit(X[:, 0:1])  # fit above imputer object to our matrix X (we can do this way as well to select column by index no by  imp_1 = imputer.fit(X[:, [1,2]])   )
X[:,  0:1] = imp_1.transform(X[:, 0:1])   # LHS X dataset and RHs imp_object to implement missing values  (   X[:, 1:3] = imp_1.transform(X[:, 1:3])   )

"""
# 4) Categorical Data and Encoding it - in our case STATE is categorical variable
from sklearn.preprocessing import LabelEncoder,OneHotEncoder
labelencoder_X = LabelEncoder()
startup_X[:, 3]  = labelencoder_X.fit_transform(startup_X[:, 3])
#X = np.array(X,dtype= 'int64')
#Here we are splitting categorial variable states in column as no of catogories - no of columns
onehotencoder = OneHotEncoder(categorical_features = [3])
startup_X = onehotencoder.fit_transform(startup_X).toarray() # Dummy variable  is created against state

#labelencoder_Y = LabelEncoder()
#Y  = labelencoder_Y.fit_transform(Y)

# 5) Avoiding Dummy Variable Trap
    # We always remove 1 column from dummy variable like in our case we generated three columns and dummy_var = dn-1
startup_X = startup_X[:, 1:]
"""

#NOTE We have very small data , to predict very accurate result we are not splitting our dataset instead we are going with full -Skipping this step
# 6) Split Data into Train and Test
from sklearn.cross_validation import train_test_split
X_train , X_test , Y_train, Y_test = train_test_split(X,Y , test_size = 0.2 , random_state=0) #test_size can be 0.3 or 1/3

# Plotting data into chart to see curve
plt.scatter(X , Y, color = 'red') #regressor.predict(X_train) tp predict the value of salary
plt.title('Polinomial Regression')
plt.xlabel('Position')
plt.ylabel('Salary')
plt.show()

# 6) Fitting Linear Regression to Dataset    (It is just made to compare)
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg .fit(X,Y)

# 7) Fitting Polynomial Regression to Dataset
from sklearn.preprocessing import PolynomialFeatures
poly_reg = PolynomialFeatures(degree = 4)
X_poly = poly_reg.fit_transform(X)
lin_reg2 = LinearRegression()
lin_reg2 .fit(X_poly,Y)

# 8 ) Visualization

# Linear Regression
plt.scatter(X, Y, color = 'red')
plt.plot(X,lin_reg.predict(X) ,color = 'blue') #regressor.predict(X) tp predict the value of salary
plt.title('Linear Regression')
plt.xlabel('Exp position')
plt.ylabel('Salary')
plt.show() # If we see result it is not goog model to show

# Polynomial Linear Regression
plt.scatter(X, Y, color = 'red')
plt.plot(X,lin_reg2.predict(X_poly) ,color = 'blue') #Ploynomial regressor.predict(X) tp predict the value of salary
plt.title('Polynimial Regression')
plt.xlabel('Exp position')
plt.ylabel('Salary')
plt.show()

# Polynomial Linear Regression (better visulaization)
x_grid = np.arange(min(X),max(X),0.1)
x_grid = x_grid.reshape(len(x_grid),1)
plt.scatter(X, Y, color = 'red')
plt.plot(x_grid,lin_reg2.predict(poly_reg.fit_transform(x_grid)) ,color = 'blue') #Ploynomial regressor.predict(X) tp predict the value of salary
plt.title('Polynimial Regression')
plt.xlabel('Exp position')
plt.ylabel('Salary')
plt.show()

# 9) Predictions Result
# Linear Regression Prediction
pos_pred = lin_reg.predict(X)
pos_pred1 = lin_reg.predict(6.5)

# Polunomial Regression Prediction
pos_pred2  = lin_reg2.predict(X_poly)
pos_pred3  = lin_reg2.predict(poly_reg.fit_transform(6.5))

Machine Learning A-Z/Part 2 - Regression/Section 6 - Polynomial Regression/POLINOMIAL_LINEAR_REGRESSION.py


from __future__ import annotations

import contextlib
import difflib
import hashlib
import os
import pathlib
import random
import string
from typing import Callable, Protocol
from xml.dom import minidom
from xml.etree import ElementTree

import numpy as np
import pytest
from click.testing import CliRunner
from PIL import Image

import vpype as vp

def random_line(length: int) -> np.ndarray:
    return np.random.rand(length) + 1j * np.random.rand(length)

@pytest.fixture
def runner():
    return CliRunner(mix_stderr=False)

@pytest.fixture
def config_manager():
    return vp.ConfigManager()

@contextlib.contextmanager
def set_current_directory(path: pathlib.Path):
    origin = path.absolute()
    try:
        os.chdir(path)
        yield
    finally:
        os.chdir(origin)

@pytest.fixture(scope="session")
def config_file_factory(tmpdir_factory):
    def _make_config_file(text: str) -> str:
        path = os.path.join(tmpdir_factory.mktemp("config_file"), "config.toml")
        with open(path, "w") as fp:
            fp.write(text)
        return path

    return _make_config_file

class LineCollectionMaker(Protocol):
    def __call__(
        self, line_count: int | None = ..., with_metadata: bool = ...
    ) -> vp.LineCollection:
        ...

@pytest.fixture
def make_line_collection() -> LineCollectionMaker:
    def _make_line_collection(
        line_count: int | None = None, with_metadata: bool = True
    ) -> vp.LineCollection:
        if line_count is None:
            line_count = random.randint(1, 10)
        if with_metadata:
            metadata = {
                vp.METADATA_FIELD_COLOR: vp.Color(
                    random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)
                ),
                vp.METADATA_FIELD_NAME: "".join(
                    random.choice(string.ascii_letters) for _ in range(10)
                ),
                vp.METADATA_FIELD_PEN_WIDTH: vp.convert_length(f"{random.random()}mm"),
            }
        else:
            metadata = {}

        lc = vp.LineCollection(metadata=metadata)
        for _ in range(line_count):
            lc.append(
                [
                    random.random() * 100 + random.random() * 100j
                    for _ in range(random.randint(2, 10))
                ]
            )
        return lc

    return _make_line_collection

class DocumentMaker(Protocol):
    def __call__(self, layer_count: int = ...) -> vp.Document:
        ...

@pytest.fixture
def make_document(make_line_collection) -> DocumentMaker:
    def _make_document(layer_count: int = 1) -> vp.Document:
        doc = vp.Document()
        doc.page_size = 1000.0 * random.random(), 1000.0 * random.random()
        for _ in range(layer_count):
            doc.add(make_line_collection(), with_metadata=True)
        return doc

    return _make_document

# IMAGE COMPARE SUPPORT
# ideally, this would be cleanly factored into a pytest plug-in akin to pytest-mpl

REFERENCE_IMAGES_DIR = (
    os.path.dirname(__file__) + os.path.sep + "data" + os.path.sep + "baseline"
)

def pytest_addoption(parser):
    parser.addoption(
        "--store-ref-images",
        action="store_true",
        help="Write reference image for assert_image_similarity().",
    )

    parser.addoption(
        "--skip-image-similarity",
        action="store_true",
        help="Skip tests using assert_image_similarity().",
    )

    parser.addoption(
        "--store-ref-svg",
        action="store_true",
        help="Write reference SVGs for reference_svg().",
    )

def write_image_similarity_fail_report(
    image: Image.Image,
    reference_image: Image.Image,
    image_array: np.ndarray,
    reference_image_array: np.ndarray,
    test_id: str,
    diff: float,
) -> None:
    report_dir = pathlib.Path.cwd() / "test_report_img_sim" / test_id
    report_dir.mkdir(parents=True, exist_ok=True)
    diff_img = Image.fromarray(np.abs(reference_image_array - image_array).astype(np.uint8))

    image.save(str(report_dir / "test_image.png"))
    reference_image.save(str(report_dir / "reference_image.png"))
    diff_img.save(str(report_dir / "difference_image.png"))
    np.save(str(report_dir / "test_image_array.npy"), image_array)
    np.save(str(report_dir / "reference_image_array.npy"), reference_image_array)
    with open(str(report_dir / "report.txt"), "w") as fp:
        fp.write(f"Test ID: {test_id}\nComputed different: {diff}")

@pytest.fixture
def assert_image_similarity(request) -> Callable:
    if request.config.getoption("--skip-image-similarity"):
        pytest.skip("image similarity test skipped (--skip-image-similarity)")

    store_ref_image = request.config.getoption("--store-ref-images")
    test_id = request.node.name
    test_id = test_id.replace("[", "-").replace("]", "-").replace("/", "_").rstrip("-")
    path = REFERENCE_IMAGES_DIR + os.path.sep + test_id + ".png"

    def _assert_image_similarity(img: Image.Image) -> None:
        nonlocal store_ref_image, test_id, path

        if store_ref_image:  # pragma: no cover
            img.save(path)
            pytest.skip("storing reference images")
        else:
            try:
                ref_img = Image.open(path)
            except FileNotFoundError:
                pytest.fail(f"reference image {path} not found")
                return

            img = img.convert(ref_img.mode)
            img = img.resize(ref_img.size)
            ref_img_arr = np.asarray(ref_img).astype("float")
            img_arr = np.asarray(img).astype("float")
            sum_sq_diff = np.mean((ref_img_arr - img_arr) ** 2)

            if sum_sq_diff != 0:
                normalized_sum_sq_diff = sum_sq_diff / np.sqrt(sum_sq_diff)
                if normalized_sum_sq_diff > 6.5:  # pragma: no cover
                    write_image_similarity_fail_report(
                        img, ref_img, img_arr, ref_img_arr, test_id, normalized_sum_sq_diff
                    )
                    pytest.fail(
                        f"image similarity test failed (rms: {normalized_sum_sq_diff})"
                    )

    return _assert_image_similarity

def _read_svg_lines(path: pathlib.Path) -> list[str]:
    tree = ElementTree.parse(path)
    xml_str = ElementTree.tostring(tree.getroot())
    # ET.canonicalize doesn't exist on Python 3.7
    canon = ElementTree.canonicalize(xml_str, strip_text=True)  # type: ignore
    lines = minidom.parseString(canon).toprettyxml().splitlines()
    return [line for line in lines if "<dc:source" not in line and "<dc:date" not in line]

def _write_reference_svg_fail_report(
    ref_lines: list[str],
    test_lines: list[str],
    test_id: str,
) -> None:
    report_dir = pathlib.Path(__file__).parent.parent / "test_report_reference_svg" / test_id
    report_dir.mkdir(parents=True, exist_ok=True)
    (report_dir / "reference.svg").write_text("\n".join(ref_lines))
    (report_dir / "test.svg").write_text("\n".join(test_lines))

@pytest.fixture
def reference_svg(request, tmp_path) -> Callable:
    """Compare an SVG output to a saved reference.

    Use `--store-ref-svg` to save reference SVGs.

    Example::

        def test_ref_svg(reference_svg):
            with reference_svg() as path:
                export_svg_to(path)
    """

    store_ref_svg = request.config.getoption("--store-ref-svg")
    test_id = "refsvg_" + hashlib.md5(request.node.name.encode()).hexdigest() + ".svg"
    ref_path = pathlib.Path(REFERENCE_IMAGES_DIR) / test_id
    temp_file = tmp_path / test_id

    @contextlib.contextmanager
    def _reference_svg():
        nonlocal ref_path, temp_file, store_ref_svg

        yield temp_file

        if store_ref_svg:  # pragma: no cover
            ref_path.write_bytes(temp_file.read_bytes())
        else:
            if not ref_path.exists():  # pragma: no cover
                pytest.fail("reference SVG does not exist")

            temp_lines = _read_svg_lines(temp_file)
            ref_lines = _read_svg_lines(ref_path)

            if len(temp_lines) != len(ref_lines) or not all(
                a == b for a, b in zip(temp_lines, ref_lines)
            ):
                delta = difflib.unified_diff(
                    temp_lines,
                    ref_lines,
                    fromfile="<test result>",
                    tofile=str(ref_path.relative_to(pathlib.Path(REFERENCE_IMAGES_DIR))),
                    lineterm="",
                )

                _write_reference_svg_fail_report(ref_lines, temp_lines, request.node.name)

                pytest.fail("generated SVG does not match reference:\n" + "\n".join(delta))

    return _reference_svg


from flask import Blueprint, request, jsonify, json
from db import db, app, ma
from flask import Flask,  redirect, request, jsonify, json, session, render_template
from model.turnos  import turnos, turnosSchema

routes_turnos = Blueprint("routes_turnos", __name__)

turno_schema = turnosSchema()
turnos_schema = turnosSchema(many=True)

@routes_turnos.route('/turnos', methods=['GET'])
def obtenerturnos():
    returnall = turnos.query.all()

    resultado_turnos = turnos_schema.dump(returnall)
    return jsonify(resultado_turnos)

@routes_turnos.route('/saveturnos', methods=['POST'])
def guardar_turnos():
    newturnos = request.json['id_vehiculo','id_rol','Hora_inicio','Hora_Fin']
    new_tur = turnos(newturnos)
    db.session.add(new_tur)
    db.session.commit()
    return redirect('/turnos')

@routes_turnos.route('/eliminarturnos/<id>', methods=['GET'] )
def eliminartur(id):
    tur = turnos.query.get(id)
    db.session.delete(tur)
    db.session.commit()
    return jsonify(turnos_schema.dump(tur))

@routes_turnos.route('/actualizarturnos', methods=['POST'] )
def actualizartur():
    id = request.json['id']
    tur = request.json['id_vehiculo','id_rol','Hora_inicio','Hora_Fin']
    pusuario = turnos.query.get(id)
    pusuario.Nombre_turnos = tur
    db.session.commit()
    return redirect('/turnos')


"""Tests for pydent.base.py."""
import copy

import pytest

from pydent import AqSession
from pydent import ModelBase
from pydent import ModelRegistry
from pydent.exceptions import NoSessionError
from pydent.marshaller import add_schema
from pydent.marshaller import exceptions
from pydent.marshaller import fields
from pydent.marshaller import SchemaRegistry

# def test_model_base():
#     """
#     Upon instantiation, .session should be None
#     """
#     m = ModelBase()
#     assert m.session is None

@pytest.fixture(scope="function")
def base():
    old_schemas = dict(SchemaRegistry.schemas)
    old_models = dict(ModelRegistry.models)

    yield ModelBase

    new_schemas = set(SchemaRegistry.schemas.keys()).difference(set(old_schemas))
    new_models = set(ModelRegistry.models.keys()).difference(set(old_models))

    for s in new_schemas:
        SchemaRegistry.schemas.pop(s)

    for m in new_models:
        ModelRegistry.models.pop(m)

@pytest.fixture(scope="function")
def mymodel(base):
    @add_schema
    class MyModel(base):
        pass

    yield MyModel

def test_record_id():
    """Creating a ModelBase should create a new record id 'rid.' For each
    instance, a new 'rid' should be created."""

    @add_schema
    class MyModel(ModelBase):
        pass

    @add_schema
    class MyOtherModel(ModelBase):
        pass

    m = MyModel()
    m2 = MyOtherModel()
    m3 = MyModel()
    assert m.rid != m2.rid
    assert m2.rid != m3.rid
    assert m.rid != m3.rid

def test_deepcopy():
    """Deepcopy should retain attributes exactly."""

    @add_schema
    class MyModel(ModelBase):
        pass

    m = MyModel()
    copied = copy.deepcopy(m)
    assert m.rid == copied.rid

@pytest.mark.parametrize(
    "copy_method",
    [pytest.param(lambda x: x.copy()), pytest.param(lambda x: copy.copy(x))],
)
def test_copy(copy_method):
    """Copy should anonymize models."""

    @add_schema
    class MyModel(ModelBase):
        def __init__(self, id):
            super().__init__(id=id)

    m = MyModel(5)
    copied = copy_method(m)
    assert m.rid != copied.rid
    assert copied.id is None

def test_copy_anonymizes_nested_relationships():
    """Copy should recursively anonymize all models."""

    @add_schema
    class MyModel(ModelBase):
        def __init__(self, id):
            super().__init__(id=id)

    @add_schema
    class MyOtherModel(ModelBase):
        def __init__(self, id):
            super().__init__(id=id)

    m = MyModel(1)
    m2 = MyOtherModel(2)
    m3 = MyModel(3)
    m3.other = m2
    m2.other = m

    rid1 = m.rid
    rid2 = m2.rid
    rid3 = m3.rid

    copied = m3.copy()
    assert copied.id is None
    assert copied.other.id is None
    assert copied.other.other.id is None

    assert copied.rid != rid1
    assert copied.other.rid != rid2
    assert copied.other.other.rid != rid3

    assert m3.rid == rid3
    assert m3.other.rid == rid2
    assert m3.other.other.rid == rid1

def test_basic_constructor(mymodel):
    """Model should absorb the kwargs."""
    m = mymodel(name="SomeName", id=2)
    assert m.name == "SomeName"
    assert m.id == 2
    data = m.dump()
    data.pop("rid")
    assert data == {"name": "SomeName", "id": 2}

def test_base_constructor_with_marshaller(mymodel):
    """MyModel initializes should absorb kwargs into attributes.

    With a schema, those attributes are also tracked and available for
    dumping.
    """
    m = mymodel(name="model", id=5)
    assert m.name == "model"
    assert m.id == 5
    mdump = m.dump()
    del mdump["rid"]
    assert mdump == {"name": "model", "id": 5}

def test_connect_to_session(mymodel, fake_session):
    """Connecting to other sessions afterward should not be allowed."""
    m = mymodel()
    assert m.session is None

    # connect to session
    m.connect_to_session(fake_session)
    assert m.session == fake_session

    # attempt to connect to another session
    fake_session2 = copy.copy(fake_session)

    with pytest.raises(Exception):
        m.connect_to_session(fake_session2)

def test_empty_relationships(mymodel):
    m = mymodel()
    assert m.get_relationships() == {}

def test_check_for_session(mymodel, fake_session):
    """If session is none, _check_for_session should raise an
    AttributeError."""
    m = mymodel()
    with pytest.raises(NoSessionError):
        m._check_for_session()

    m.connect_to_session(fake_session)
    m._check_for_session()

def test_model_registry(mymodel):
    """We expect get_model to return the value in the models dictionary."""

    assert "MyModel" in ModelRegistry.models
    assert ModelRegistry.models["MyModel"] == mymodel
    assert ModelRegistry.get_model("MyModel") == mymodel
    del ModelRegistry.models["MyModel"]

def test_no_model_in_registry():
    """ModelRegistry should raise error if model doesn't exist."""
    with pytest.raises(exceptions.ModelRegistryError):
        ModelRegistry.get_model("SomeModelThatDoesntExist")

def test_find_no_session(mymodel):
    """ModelBase should raise AttributeError if no session is attacheded."""
    m = mymodel()
    with pytest.raises(NoSessionError):
        m.find_callback(None, None)

def test_where_no_session(mymodel):
    """ModelBase should raise AttributeError if no session is attacheded."""
    m = mymodel()
    with pytest.raises(NoSessionError):
        m.where_callback(None, None)

def test_where_and_find(mymodel, monkeypatch, fake_session):
    """Calling the 'where' wrapper on a ModelBase should attempt to get a model
    interface and call 'where' or 'find' on the interface.

    In this case, a fake model interface is returned in which the
    methods return the parameter passed in
    """

    def fake_model_interface(self, model_name):
        """A fake model interface to test where."""

        class FakeInterface:
            def find(id):
                return id

            def where(params):
                return params

        return FakeInterface

    monkeypatch.setattr(
        AqSession, AqSession.model_interface.__name__, fake_model_interface
    )

    m = mymodel()
    ModelRegistry.models["FakeModel"] = ModelBase
    m.connect_to_session(fake_session)
    assert m.where_callback("FakeModel", 5) == 5
    assert m.find_callback("FakeModel", 6) == 6

def test_print(mymodel):
    m = mymodel()
    print(m)
    m.print()

def test_load_many(base, fake_session):
    @add_schema
    class Child(base):
        pass

    @add_schema
    class Parent(base):
        fields = dict(children=fields.Relationship("Child", "get_children", many=True))

        def get_children(self, model_name):
            return None

    parent = Parent.load_from(
        {"id": 10, "children": [{"id": 1, "name": "Child1"}, {"id": 2}]},
        fake_session.utils,
    )
    print(parent.children)
    assert len(parent.children) == 2
    assert isinstance(parent.children[0], Child)
    assert parent.children[0].name == "Child1"

def test_uri(base):
    """Expect with with the `include_uri` key includes the default URI."""

    @add_schema
    class MyModel(base):
        pass

    model = MyModel()
    model.id = 10
    assert model.uri == "http://aquarium.org/my_models/10"

def test_dump_uri(base):
    """Expect with with the `include_uri` key includes the default URI."""

    @add_schema
    class MyModel(base):
        pass

    model = MyModel()
    model.id = 10
    no_uri = model.dump()
    with_uri = model.dump(include_uri=True)
    assert "__uri__" not in no_uri
    assert "__uri__" in with_uri
    assert with_uri["__uri__"] == "http://aquarium.org/my_models/10"

def test_dump_model(base):
    """Expect with with the `include_uri` key includes the default URI."""

    @add_schema
    class MyModel(base):
        pass

    model = MyModel()
    model.id = 10
    no_mt = model.dump()
    with_mt = model.dump(include_model_type=True)
    assert "__model__" not in no_mt
    assert "__model__" in with_mt
    assert with_mt["__model__"] == "MyModel"


import os
import maya.cmds as cmds
from functools import partial
import Core
reload(Core)

class UIContainer():
    frozen = False

def build(parent,
          imagesPath,
          iconSize=25,
          height=20,
          marginSize=5):
    """
    build widget
    @param parent : parent layout in maya
    @imagesPath : str path
    """
    # - Freeze Viewport---------------------------------------------------------------
    cmds.rowLayout(numberOfColumns=1, parent=parent)
    cmds.iconTextButton(style='iconOnly',
                        image1=os.path.join(imagesPath, 'freeze.png'),
                        hi=os.path.join(imagesPath, 'freeze_hi.png'),
                        width=iconSize, mw=marginSize, height=iconSize, mh=marginSize,
                        label='freeze',
                        annotation='Freeze / unFreeze viewport',
                        c=freeze)
    cmds.setParent("..")

def freeze(*args):
    """
    """
    if UIContainer.frozen == True:
        cmds.refresh(su=False)
        print('Viewport is unfrozen')
        UIContainer.frozen = False
    else:
        cmds.refresh(su=True)
        print('Viewport is frozen')
        UIContainer.frozen = True


from datetime import datetime

import cv2 as cv
import matplotlib.pyplot as plt

def perform_sift_performant(query, train):
    start = datetime.now()
    img1 = cv.imread(query, cv.IMREAD_GRAYSCALE)  # queryImage
    img2 = cv.imread(train, cv.IMREAD_GRAYSCALE)  # trainImage

    # Initiate SIFT detector
    sift = cv.SIFT_create()

    # find the keypoints and descriptors with SIFT
    kp1, des1 = sift.detectAndCompute(img1, None)
    kp2, des2 = sift.detectAndCompute(img2, None)

    # FLANN parameters
    FLANN_INDEX_KDTREE = 1
    index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
    search_params = dict(checks=50)  # or pass empty dictionary
    flann = cv.FlannBasedMatcher(index_params, search_params)
    matches = flann.knnMatch(des1, des2, k=2)

    # Need to draw only good matches, so create a mask
    matchesMask = [[0, 0] for _ in range(len(matches))]

    # ratio test as per Lowe's paper
    for i, (m, n) in enumerate(matches):
        if m.distance < 0.6 * n.distance:
            matchesMask[i] = [1, 0]
    print(datetime.now() - start)
    draw_params = dict(matchColor=(0, 255, 0),
                       singlePointColor=(255, 0, 0),
                       matchesMask=matchesMask,
                       flags=cv.DrawMatchesFlags_DEFAULT)
    img3 = cv.drawMatchesKnn(img1, kp1, img2, kp2, matches, None, **draw_params)
    plt.imshow(img3, ), plt.show()


'''
47. 이진 트리 직렬화 & 역직렬화
이진트리를 배열로 직렬화하고, 반대로 역직렬화하는 기능을 구현하라.
https://leetcode.com/problems/serialize-and-deserialize-binary-tree/
'''
from collections import deque

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Codec:
    def serialize(self, root):
        queue = deque()
        queue.append(root)
        answer = [-1]
        while queue:
            node = queue.popleft()
            if not node:
                answer.append(None)
            else:
                answer.append(node.val)
                queue.append(node.left)
                queue.append(node.right)

        while answer[-1] is None:
            answer.pop()
        return answer

    def deserialize(self, data):
        if len(data) == 1:
            return None

        data = deque(data[1:])
        root = TreeNode(data.popleft())
        queue = deque([root])
        while queue:
            node = queue.popleft()
            if node and data:
                left = data.popleft()
                node.left = TreeNode(left) if left is not None else None
                queue.append(node.left)
            if node and data:
                right = data.popleft()
                node.right = TreeNode(right) if right is not None else None
                queue.append(node.right)
        return root

if __name__ == '__main__':
    root = TreeNode(1)
    root.left = TreeNode(2)
    root.right = TreeNode(3)
    root.right.left = TreeNode(4)
    root.right.right = TreeNode(5)
    root.right.left.left = TreeNode(6)
    root.right.left.right = TreeNode(7)

    ser = Codec()
    deser = Codec()

    print(ser.serialize(root))
    print(deser.deserialize(ser.serialize(root)))


from Utils import *

from webbrowser import open as open_url

def _main():
    from main import main
    GithubPanel().panel(main)

class GithubPanel():

    def __init__(self) -> None:
        from main import VALIDATING_CHAINS
        self.VALIDATING_CHAINS = VALIDATING_CHAINS

    def panel(self, main_panel_func):
        options = {
            "f": ["Workflows", self.workflows],
            "d": ["Adds dependabot if not already", self.dependabot],
            "w": ["Open chain in browser\n", self.website],

            "m": ["Main Panel", main_panel_func],
            "e": ["Exit", exit],
        }
        aliases = {}
        while True:
            selector("&f", "Github", options=options, aliases=aliases)

    def website(self):
        chains = select_chains("Select a chain(s) to open the website for (space to select)")
        location = pick(["parent", "fork"], "Open the 'parent' repo or 'fork' version? (select 1)")[0]
        cprint(f"Opening Website for: {','.join(chains)}...")
        for chain in chains:
            info = self.VALIDATING_CHAINS.get(chain, None)
            if info == None: continue
            url_path = info[location].replace(".git", "").split(":")[1]
            open_url(f"https://github.com/{url_path}")

    def workflows(self):
        # select a chain, and add workflows to it
        chain = pick(get_downloaded_chains(), "Select a chain to add workflows to")[0]
        # os.chdir(os.path.join(current_dir, chain))
        cprint(f"&aAdding workflows to {chain}")
        Workflow(chain).add_workflows()

    def dependabot(self): # combine with workflows?
        chain = pick(get_downloaded_chains(), "Select a chain to add dependabot to")[0]
        cprint(f"&aAdding dependabot to {chain} if it is not already there")
        Workflow(chain).add_dependabot()

class Workflow():
    def __init__(self, folder_name) -> None:
        self.folder_name = folder_name
        self.chain_dir = os.path.join(current_dir, self.folder_name)
        from main import WORKFLOWS, VALIDATING_CHAINS, yml_files
        self.WORKFLOWS = WORKFLOWS
        self.VALIDATING_CHAINS = VALIDATING_CHAINS
        self.yml_files = yml_files

    def available_workflows(self):
        return os.listdir(self.WORKFLOWS)

    def upgrade_actions(self):
        # check if there is a github folder
        if not os.path.isdir(os.path.join(self.chain_dir, ".github")):
            return
        # Turn all actions/checkout@v3 -> actions/checkout@v3.1.0

    def _write_workflow(self, workflow_name):
        if workflow_name not in self.available_workflows():
            print(f"Workflow {workflow_name} not found in {self.WORKFLOWS}")
            return

        main_branch_name = self.VALIDATING_CHAINS[self.folder_name]["branch"]

        os.chdir(os.path.join(current_dir, self.folder_name))
        print(f"Writing {workflow_name} to .github/workflows/{workflow_name} for {self.folder_name}")
        with open(os.path.join(self.yml_files, "workflows", workflow_name), "r") as f2:
            with open(f".github/workflows/{workflow_name}", "w") as f:
                f.write(f2.read().replace("_MAIN_BRANCH_", main_branch_name))
        os.chdir(current_dir)

    def add_workflows(self):
        os.chdir(self.chain_dir)
        if not os.path.exists(".github/workflows"):
            os.makedirs(".github/workflows", exist_ok=True)

        workflow_options = [x for x in os.listdir(self.WORKFLOWS) if x not in os.listdir(".github/workflows")]

        title = "Workflow options to install (space, then enter) [ignores already installed]"
        selected = [s[0] for s in pick(workflow_options, title, multiselect=True, min_selection_count=0, indicator="=> ")]

        for file in selected:
            self._write_workflow(file)
        os.chdir(current_dir)

    def add_dependabot(self, simulate=False):
        # check if self.folder_name/.github exists, if so see if any files have "dependabot" in them
        p = os.path.join(self.chain_dir, ".github")
        if not os.path.isdir(p):
            os.mkdir(p)

        containsDependABot = False
        for root, dirs, files in os.walk(p, topdown=True):
            for name in files:
                if "dependabot" in name:
                    containsDependABot = True
                    break

        if containsDependABot:
            if simulate == False:
                print(f"{self.folder_name} already has dependabot")
            return

        # https://github.com/eve-network/eve, put in .yml_files folder
        # check if folder chains .github/dependabot.yml dir
        main_branch = self.VALIDATING_CHAINS[self.folder_name]["branch"] # main, master, etc

        if simulate:
            print(f"SIMULATE: Would add dependabot to {self.folder_name}")
            return
        else:
            print(f"Adding dependabot to {self.folder_name}...")
            os.chdir(self.chain_dir)
            if not os.path.exists(".github/dependabot.yml"):
                os.makedirs(".github", exist_ok=True)
                # write yml_files/dependabot.yml to .github/dependabot.yml
                print(f"Writing dependabot.yml to .github/dependabot.yml for {self.folder_name} as it did not have it...")
                with open(f".github/dependabot.yml", "w") as f:
                    with open(os.path.join(self.yml_files, "dependabot.yml"), "r") as f2:
                        f.write(f2.read().replace("_MAIN_BRANCH_", main_branch))
            os.chdir(current_dir)

if __name__ == "__main__":
    _main()


import os
from shutil import copyfile
from typing import List, TYPE_CHECKING

import jinja2
from ..checks.issues import Issues
from ..checks.file_size import FileSizeCheck
from .utils import create_parent_directory
from ..checks.base import Check

if TYPE_CHECKING:
    from combine.core import ContentDirectory

class File:
    def __init__(self, path: str, content_directory: "ContentDirectory") -> None:
        self.path = path
        self.references: List[str] = []
        self.content_directory = content_directory
        self.content_relative_path = os.path.relpath(
            self.path, self.content_directory.path
        )

        self.root, self.extension = os.path.splitext(self.content_relative_path)
        self.root_parts = os.path.split(self.root)
        self.name_without_extension = self.root_parts[-1]

        self.output_relative_path = self._get_output_relative_path()

    def __repr__(self) -> str:
        return f"<{self.__class__.__name__} {self.path}>"

    def _get_output_relative_path(self) -> str:
        return self.content_relative_path

    def load(self, jinja_environment: jinja2.Environment) -> None:
        """Load properties that can vary depending on content of the file"""
        self.references = []

    def render(self, output_path: str, jinja_environment: jinja2.Environment) -> None:
        self.output_path = self._render_to_output(output_path, jinja_environment)

    def _render_to_output(
        self, output_path: str, jinja_environment: jinja2.Environment
    ) -> str:
        target_path = os.path.join(output_path, self.output_relative_path)
        create_parent_directory(target_path)

        if os.path.exists(target_path):
            os.remove(target_path)

        copyfile(self.path, target_path)

        return target_path

    def check_output(self) -> Issues:
        issues = Issues()

        for check in self.get_checks():
            for issue in check.run():
                issues.append(issue)

        if issues:
            issues.print(f"Issues in {self.content_relative_path}")

        return issues

    def get_checks(self) -> List[Check]:
        if self.output_path:
            # Not all files have an output
            return [
                FileSizeCheck(path=self.output_path),
            ]

        return []


import os
import sys
import math
import argparse

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

import matplotlib
matplotlib.use('Agg')
import numpy as np

from models import *
import datasets as dsets
from datasets import PairwiseDataset

models = {
    'cyclegan': CycleGAN,
    'unit': UNIT
}

def load_data(data_type):
    if data_type == 'mnist':
        return dsets.mnist.load_data()
    elif data_type == 'svhn':
        return dsets.svhn.load_data()
    else:
        return dsets.load_data(data_type)

def main():
    # Parsing arguments
    parser = argparse.ArgumentParser(description='Training GANs or VAEs')
    parser.add_argument('--model', type=str, required=True)
    parser.add_argument('--first-data', type=str, required=True)
    parser.add_argument('--second-data', type=str, required=True)
    parser.add_argument('--epoch', type=int, default=500)
    parser.add_argument('--batchsize', type=int, default=50)
    parser.add_argument('--output', default='output')
    parser.add_argument('--zdims', type=int, default=128)
    parser.add_argument('--gpu', type=int, default=0)
    parser.add_argument('--resume', type=str, default=None)
    parser.add_argument('--testmode', action='store_true')

    args = parser.parse_args()

    # select gpu
    os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)

    # Make output direcotiry if not exists
    if not os.path.isdir(args.output):
        os.mkdir(args.output)

    # Load datasets
    x_data = load_data(args.first_data)
    x_data = x_data.images * 2.0 - 1.0
    y_data = load_data(args.second_data)
    y_data = y_data.images * 2.0 - 1.0

    datasets = PairwiseDataset(x_data, y_data)
    num_data = len(datasets)

    # Construct model
    if args.model not in models:
        raise Exception('Unknown model:', args.model)

    model = models[args.model](
        input_shape=datasets.shape[1:],
        z_dims=args.zdims,
        output=args.output
    )

    if args.resume is not None:
        model.load_model(args.resume)

    # Make samples
    x_samples = datasets.x_data[num_data:num_data+25]
    y_samples = datasets.y_data[num_data:num_data+25]
    samples = (x_samples, y_samples)

    # Training loop
    model.main_loop(datasets, samples,
        epochs=args.epoch,
        batchsize=args.batchsize,
        reporter=['loss', 'g_loss', 'd_loss', 'g_acc', 'd_acc'])

if __name__ == '__main__':
    main()


from sys import exit

from kojak import cli
from kojak.exceptions import KojakException
from kojak.models import Analyze
from kojak.reports import Report

from pbr.version import VersionInfo

def main():
    args = cli.argparser().parse_args()
    version = VersionInfo("kojak")
    if args.version:
        print("kojak v{version}".format(version=version))
        return 0
    filename = args.file
    if not filename:
        return 0

    try:
        analyze = Analyze(filename)
    except KojakException as err:
        print(str(err))
        return 1
    else:
        report = Report()
        report.rendering("cli.txt", analyze)
    footer = "Generated by kojak v{version}".format(version=version)
    print("-" * len(footer))
    print(footer)
    print("-" * len(footer))
    return 0

if __name__ == "__main__":
    exit(main())


from views import ViewsInterface
from .checkers import check_menu
from .simulations import Simulations

def main_menu(view: ViewsInterface):
    options = [
        (
            'Effectuer une simulation',
            Simulations.run,
            {
                'view': view
            }
        ),
        (
            'Quitter',
            quit,
            {}
        )
    ]
    while True:
        option_index = check_menu(
            view=view,
            items=[option[0] for option in options],
            title='Menu principal'
        )
        options[option_index][1](**options[option_index][2])