import numpy as np
from pycifstar.global_ import Global

from EasyApp.Logic.Logging import console
from Logic.Helpers import IO

try:
    import cryspy
    from cryspy.H_functions_global.function_1_cryspy_objects import \
        str_to_globaln
    from cryspy.A_functions_base.function_2_space_group import \
        get_it_coordinate_system_codes_by_it_number
    console.debug('CrysPy module imported')
except ImportError:
    console.error('No CrysPy module found')

class Parameter(dict):

    def __init__(self,
                value,
                permittedValues = None,
                idx = 0,
                error = 0.0,
                min = -np.inf,
                max = np.inf,
                absDelta = None,
                pctDelta = None,
                units = '',
                category = '',
                prettyCategory = '',
                rowName = '',
                name = '',
                prettyName = '',
                shortPrettyName = '',
                icon = '',
                categoryIcon = '',
                url = '',
                cifDict = '',
                optional = False,
                enabled = True,
                fittable = False,
                fit = False):
        self['value'] = value
        self['permittedValues'] = permittedValues
        self['idx'] = idx
        self['optional'] = optional
        self['enabled'] = enabled
        self['fittable'] = fittable
        self['fit'] = fit
        self['error'] = error
        self['group'] = ""
        self['min'] = min
        self['max'] = max
        self['absDelta'] = absDelta
        self['pctDelta'] = pctDelta
        self['category'] = category
        self['prettyCategory'] = prettyCategory
        self['rowName'] = rowName
        self['name'] = name
        self['prettyName'] = prettyName
        self['shortPrettyName'] = shortPrettyName
        self['icon'] = icon
        self['categoryIcon'] = categoryIcon
        self['url'] = url
        self['cifDict'] = cifDict
        self['parentIndex'] = 0
        self['parentName'] = ''
        self['units'] = units

class CryspyParser:

    @staticmethod
    def cifToDict(cif):
        obj = Global()
        obj.take_from_string(cif)
        data = obj.datas[0]
        out = {'name': data.name, 'items': {}, 'loops': {}}
        for item in data.items.items:
            out['items'][item.name] = item.value
        for loop in data.loops:
            category = loop.prefix
            out['loops'][category] = {}
            for paramIdx, fullParamName in enumerate(loop.names):
                paramName = fullParamName.replace(category, '')
                paramValues = [values[paramIdx] for values in loop.values]
                out['loops'][category][paramName] = paramValues
        return out

    @staticmethod
    def dataBlocksToCif(blocks):
        cif = ''
        for block in blocks:
            cif += CryspyParser.dataBlockToCif(block) + '\n\n'
        return cif

    @staticmethod
    def dataBlockToCif(block, includeBlockName=True):
        cif = ''

        if includeBlockName:
            cif += f"data_{block['name']['value']}"
            cif += '\n\n'

        if 'params' in block:
            for category in block['params'].values():
                for param in category.values():
                    if param["optional"]:
                        continue

                    value = param["value"]
                    if value is None:
                        continue

                    if isinstance(value, (float, int)):  # If parameter is of float type
                        value = np.float32(value)  # Simplifies output
                        if param["fit"]:
                            error = param["error"]
                            #error = np.float32(error)  # Simplifies output
                            if error == 0:
                                paramStr = f'{value}()' # Adds empty brackets for standard uncertainty for free params
                            else:
                                _, _, paramStr = IO.toStdDevSmalestPrecision(value, error) # Adds brackets with standard uncertainty for free params
                        else:
                            paramStr = str(value)  # Keeps 32bit presicion format in contrast to f'{...}'
                    elif isinstance(value, str):  # If parameter is of string type
                        if ' ' in value:  # Adds quotes to text with spaces, e.g. P n m a -> "P n m a"
                            paramStr = f'"{value}"'
                        else:
                            paramStr = f'{value}'
                    else:
                        console.error(f'Unsupported parameter type {type(value)} for {value}')
                        continue

                    cif += f'{param["category"]}.{param["name"]} {paramStr}'
                    cif += '\n'
                cif += '\n'

        if 'loops' in block:
            for categoryName, category in block['loops'].items():
                cif += '\nloop_\n'

                # loop header
                row0 = category[0]
                for param in row0.values():
                    if param["optional"]:
                        continue
                    cif += f'{categoryName}.{param["name"]}\n'

                # loop data
                for row in category:
                    line = ''
                    for param in row.values():
                        if param["optional"]:
                            continue

                        value = param["value"]
                        if value is None:
                            continue

                        if isinstance(value, (float, int)):  # If parameter is number
                            value = np.float32(value)  # Simplifies output
                            if param["fit"]:
                                error = param["error"]
                                #error = np.float32(error)  # Simplifies output
                                if error == 0:
                                    paramStr = f'{value}()' # Adds empty brackets for standard uncertainty for free params
                                else:
                                    _, _, paramStr = IO.toStdDevSmalestPrecision(value, error) # Adds brackets with standard uncertainty for free params
                            else:
                                paramStr = str(value)  # Keeps 32bit presicion format in contrast to f'{...}'
                        elif isinstance(value, str):  # If parameter is of string type
                            if ' ' in value:  # Adds quotes to text with spaces, e.g. P n m a -> "P n m a"
                                paramStr = f'"{value}"'
                            else:
                                paramStr = f'{value}'
                        else:
                            console.error(f'Unsupported parameter type {type(value)} for {value}')
                            continue

                        line += paramStr + ' '
                    line = line.rstrip()
                    cif += f'{line}\n'
        cif = cif.strip()
        cif = cif.replace('\n\n\n', '\n\n')
        return cif

    @staticmethod
    def edCifToCryspyCif(edCif):
        rawToEdNamesCif = {
            '_symmetry_space_group_name_H-M': '_space_group.name_H-M_alt',
            '_atom_site_thermal_displace_type': '_atom_site.ADP_type',
            '_atom_site_adp_type': '_atom_site.ADP_type',
            '_atom_site_U_iso_or_equiv': '_atom_site.U_iso_or_equiv'
        }
        edToCryspyNamesMap = {
            '_atom_site.site_symmetry_multiplicity': '_atom_site_multiplicity',

            '_diffrn_radiation.probe': '_setup_radiation',
            '_diffrn_radiation_wavelength.wavelength': '_setup_wavelength',

            '_pd_calib.2theta_offset': '_setup_offset_2theta',

            '_pd_instr.resolution_u': '_pd_instr_resolution_u',
            '_pd_instr.resolution_v': '_pd_instr_resolution_v',
            '_pd_instr.resolution_w': '_pd_instr_resolution_w',
            '_pd_instr.resolution_x': '_pd_instr_resolution_x',
            '_pd_instr.resolution_y': '_pd_instr_resolution_y',

            '_pd_instr.reflex_asymmetry_p1': '_pd_instr_reflex_asymmetry_p1',
            '_pd_instr.reflex_asymmetry_p2': '_pd_instr_reflex_asymmetry_p2',
            '_pd_instr.reflex_asymmetry_p3': '_pd_instr_reflex_asymmetry_p3',
            '_pd_instr.reflex_asymmetry_p4': '_pd_instr_reflex_asymmetry_p4',

            '_pd_phase_block.id': '_phase_label',
            '_pd_phase_block.scale': '_phase_scale',

            '_pd_meas.2theta_range_min': '_range_2theta_min',
            '_pd_meas.2theta_range_max': '_range_2theta_max',
            '_pd_meas.2theta_scan': '_pd_meas_2theta',

            '_pd_meas.intensity_total_su': '_pd_meas_intensity_sigma',  # before intensity_total!
            '_pd_meas.intensity_total': '_pd_meas_intensity',

            '_pd_background.line_segment_X': '_pd_background_2theta',
            '_pd_background.line_segment_intensity': '_pd_background_intensity',

            '_model.cif_file_name': '_model_cif_file_name',
            '_experiment.cif_file_name': '_experiment_cif_file_name'
        }
        edToCryspyValuesMap = {
            'x-ray': 'X-rays',
            'neutron': 'neutrons',
            'neutronss': 'neutrons',
        }
        cryspyCif = edCif
        for rawName, edName in rawToEdNamesCif.items():
            cryspyCif = cryspyCif.replace(rawName, edName)
        for edName, cryspyName in edToCryspyNamesMap.items():
            cryspyCif = cryspyCif.replace(edName, cryspyName)
        for edValue, cryspyValue in edToCryspyValuesMap.items():
            cryspyCif = cryspyCif.replace(edValue, cryspyValue)
        return cryspyCif

    @staticmethod
    def cryspyCifToModelsObj(cryspyCif):
        cryspyModelsObj = str_to_globaln(cryspyCif)

        # Replace atom types
        for dataBlock in cryspyModelsObj.items:
            if type(dataBlock) is not cryspy.E_data_classes.cl_1_crystal.Crystal:
                continue
            for item in dataBlock.items:
                if type(item) is not cryspy.C_item_loop_classes.cl_1_atom_site.AtomSiteL:
                    continue
                for cryspy_atom in item.items:
                    if cryspy_atom.type_symbol == 'D':
                        cryspy_atom.type_symbol = '2H'
                    elif cryspy_atom.type_symbol == 'H1+':
                        cryspy_atom.type_symbol = 'H'

        return cryspyModelsObj

    @staticmethod
    def starObjToEdProject(starObj):
        edProject = {'name': '', 'params': {}, 'loops': {}}

        # DATABLOCK ID

        edProject['name'] = dict(Parameter(
            starObj.name,
            icon = 'archive',
            url = 'https://docs.easydiffraction.org/app/project/dictionaries/',
        ))

        # DATABLOCK SINGLES

        for param in starObj.items.items:
            if param.name == '_project.description':
                category, name = param.name.split('.')
                if category not in edProject['params']:
                    edProject['params'][category] = {}
                edProject['params'][category][name] = dict(Parameter(
                    param.value,
                    category = category,
                    name = name,
                    prettyName = 'Description',
                    url = 'https://docs.easydiffraction.org/app/project/dictionaries/',
                ))

        # DATABLOCK TABLES

        for loop in starObj.loops:
            category = loop.prefix

            if category == '_model_cif_file':
                edModels = []
                for rowIdx, rowItems in enumerate(loop.values):
                    edModel = {}
                    for columnIdx, columnName in enumerate(loop.names):
                        if columnName.replace('.', '_')  == '_model_cif_file_name':  # NEED FIX
                            edModel['cif_file_name'] = dict(Parameter(
                                rowItems[columnIdx],
                                category = '_model',
                                name = 'cif_file_name',
                                prettyName = 'Model file',
                                url = 'https://easydiffraction.org'
                            ))
                    edModels.append(edModel)
                edProject['loops']['_model'] = edModels

            elif category == '_experiment_cif_file':
                edExperiments = []
                for rowIdx, rowItems in enumerate(loop.values):
                    edExperiment = {}
                    for columnIdx, columnName in enumerate(loop.names):
                        if columnName.replace('.', '_') == '_experiment_cif_file_name':  # NEED FIX
                            edExperiment['cif_file_name'] = dict(Parameter(
                                rowItems[columnIdx],
                                category = '_experiment',
                                name = 'cif_file_name',
                                prettyName = 'Experiment file',
                                url = 'https://easydiffraction.org'
                            ))
                    edExperiments.append(edExperiment)
                edProject['loops']['_experiment'] = edExperiments

        return edProject

    @staticmethod
    def cryspyObjToEdModels(cryspy_obj):
        ed_phases = []

        for data_block in cryspy_obj.items:
            # Skip non-crystal data blocks

            if type(data_block) is not cryspy.E_data_classes.cl_1_crystal.Crystal:
                continue

            # If crystal data block found

            cryspy_phase = data_block.items
            ed_phase = {'name': '', 'params': {}, 'loops': {}}

            # DATABLOCK ID

            ed_phase['name'] = dict(Parameter(
                data_block.data_name.lower(),
                icon='layer-group',
                url='https://docs.easydiffraction.org/app/project/dictionaries/',
            ))

            for item in cryspy_phase:

                # DATABLOCK SINGLES

                # Space group category
                if type(item) is cryspy.C_item_loop_classes.cl_2_space_group.SpaceGroup:
                    if not '_space_group' in ed_phase['params']:
                        ed_phase['params']['_space_group'] = {}
                    ed_phase['params']['_space_group']['name_H-M_alt'] = dict(Parameter(
                        item.name_hm_alt,
                        category='_space_group',
                        name='name_H-M_alt',
                        shortPrettyName='name',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_space_group/',
                        cifDict='core'
                    ))
                    ed_phase['params']['_space_group']['IT_coordinate_system_code'] = dict(Parameter(
                        item.it_coordinate_system_code,
                        category='_space_group',
                        name='IT_coordinate_system_code',
                        shortPrettyName='code',
                        permittedValues=list(get_it_coordinate_system_codes_by_it_number(item.it_number)),
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_space_group/',
                        cifDict='core'
                    ))
                    ed_phase['params']['_space_group']['crystal_system'] = dict(Parameter(
                        item.crystal_system,
                        optional=True,
                        category='_space_group',
                        name='crystal_system',
                        shortPrettyName='crystal system',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_space_group/',
                        cifDict='core'
                    ))
                    ed_phase['params']['_space_group']['IT_number'] = dict(Parameter(
                        item.it_number,
                        optional=True,
                        category='_space_group',
                        name='IT_number',
                        shortPrettyName='number',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_space_group/',
                        cifDict='core'
                    ))

                # Cell category
                elif type(item) is cryspy.C_item_loop_classes.cl_1_cell.Cell:
                    if not '_cell' in ed_phase['params']:
                        ed_phase['params']['_cell'] = {}
                    ed_phase['params']['_cell']['length_a'] = dict(Parameter(
                        item.length_a,
                        error=item.length_a_sigma,  # NEED FIX: rename 'error' to 'su'
                        category='_cell',
                        prettyCategory='cell',
                        name='length_a',
                        prettyName='length a',
                        shortPrettyName='a',  # NEED FIX: rename 'shortPrettyName' to 'shortName' ???
                        icon='ruler',
                        categoryIcon='cube',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_cell/',
                        cifDict='core',
                        enabled=not item.length_a_constraint,
                        absDelta=0.1,
                        units='Å',
                        fittable=True,
                        fit=item.length_a_refinement
                    ))
                    ed_phase['params']['_cell']['length_b'] = dict(Parameter(
                        item.length_b,
                        error=item.length_b_sigma,
                        category='_cell',
                        prettyCategory='cell',
                        name='length_b',
                        prettyName='length b',
                        shortPrettyName='b',
                        icon='ruler',
                        categoryIcon='cube',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_cell/',
                        cifDict='core',
                        enabled=not item.length_b_constraint,
                        absDelta=0.1,
                        units='Å',
                        fittable=True,
                        fit=item.length_b_refinement
                    ))
                    ed_phase['params']['_cell']['length_c'] = dict(Parameter(
                        item.length_c,
                        error=item.length_c_sigma,
                        category='_cell',
                        prettyCategory='cell',
                        name='length_c',
                        prettyName='length c',
                        shortPrettyName='c',
                        icon='ruler',
                        categoryIcon='cube',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_cell/',
                        cifDict='core',
                        enabled=not item.length_c_constraint,
                        absDelta=0.1,
                        units='Å',
                        fittable=True,
                        fit=item.length_c_refinement
                    ))
                    ed_phase['params']['_cell']['angle_alpha'] = dict(Parameter(
                        item.angle_alpha,
                        error=item.angle_alpha_sigma,
                        category='_cell',
                        prettyCategory='cell',
                        name='angle_alpha',
                        prettyName='angle α',
                        shortPrettyName='α',
                        icon='ruler',
                        categoryIcon='less-than',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_cell/',
                        cifDict='core',
                        enabled=not item.angle_alpha_constraint,
                        absDelta=1.0,
                        units='°',
                        fittable=True,
                        fit=item.angle_alpha_refinement
                    ))
                    ed_phase['params']['_cell']['angle_beta'] = dict(Parameter(
                        item.angle_beta,
                        error=item.angle_beta_sigma,
                        category='_cell',
                        prettyCategory='cell',
                        name='angle_beta',
                        prettyName='angle β',
                        shortPrettyName='β',
                        icon='ruler',
                        categoryIcon='less-than',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_cell/',
                        cifDict='core',
                        enabled=not item.angle_beta_constraint,
                        absDelta=1.0,
                        units='°',
                        fittable=True,
                        fit=item.angle_beta_refinement
                    ))
                    ed_phase['params']['_cell']['angle_gamma'] = dict(Parameter(
                        item.angle_gamma,
                        error=item.angle_gamma_sigma,
                        category='_cell',
                        prettyCategory='cell',
                        name='angle_gamma',
                        prettyName='angle γ',
                        shortPrettyName='γ',
                        icon='ruler',
                        categoryIcon='less-than',
                        url='https://docs.easydiffraction.org/app/project/dictionaries/_cell/',
                        cifDict='core',
                        enabled=not item.angle_gamma_constraint,
                        absDelta=1.0,
                        units='°',
                        fittable=True,
                        fit=item.angle_gamma_refinement
                    ))

                # DATABLOCK TABLES

                # Atom site category
                elif type(item) is cryspy.C_item_loop_classes.cl_1_atom_site.AtomSiteL:
                    cryspy_atoms = item.items
                    ed_atoms = []
                    for idx, cryspy_atom in enumerate(cryspy_atoms):
                        ed_atom = {}
                        ed_atom['label'] = dict(Parameter(
                            cryspy_atom.label,
                            idx=idx,
                            category='_atom_site',
                            name='label',
                            shortPrettyName='label',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core'
                        ))
                        ed_atom['type_symbol'] = dict(Parameter(
                            cryspy_atom.type_symbol,
                            idx=idx,
                            category='_atom_site',
                            name='type_symbol',
                            shortPrettyName='type',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core'
                        ))
                        ed_atom['fract_x'] = dict(Parameter(
                            cryspy_atom.fract_x,
                            error=cryspy_atom.fract_x_sigma,
                            idx=idx,
                            category='_atom_site',
                            prettyCategory='atom',
                            rowName=cryspy_atom.label,
                            name='fract_x',
                            prettyName='fract x',
                            shortPrettyName='x',
                            icon='map-marker-alt',
                            categoryIcon='atom',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core',
                            enabled=not cryspy_atom.fract_x_constraint,
                            absDelta=0.05,
                            fittable=True,
                            fit=cryspy_atom.fract_x_refinement
                        ))
                        ed_atom['fract_y'] = dict(Parameter(
                            cryspy_atom.fract_y,
                            error=cryspy_atom.fract_y_sigma,
                            idx=idx,
                            category='_atom_site',
                            prettyCategory='atom',
                            rowName=cryspy_atom.label,
                            name='fract_y',
                            prettyName='fract y',
                            shortPrettyName='y',
                            icon='map-marker-alt',
                            categoryIcon='atom',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core',
                            enabled=not cryspy_atom.fract_y_constraint,
                            absDelta=0.05,
                            fittable=True,
                            fit=cryspy_atom.fract_y_refinement
                        ))
                        ed_atom['fract_z'] = dict(Parameter(
                            cryspy_atom.fract_z,
                            error=cryspy_atom.fract_z_sigma,
                            idx=idx,
                            category='_atom_site',
                            prettyCategory='atom',
                            rowName=cryspy_atom.label,
                            name='fract_z',
                            prettyName='fract z',
                            shortPrettyName='z',
                            icon='map-marker-alt',
                            categoryIcon='atom',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core',
                            enabled=not cryspy_atom.fract_z_constraint,
                            absDelta=0.05,
                            fittable=True,
                            fit=cryspy_atom.fract_z_refinement
                        ))
                        ed_atom['occupancy'] = dict(Parameter(
                            cryspy_atom.occupancy,
                            error=cryspy_atom.occupancy_sigma,
                            idx=idx,
                            category='_atom_site',
                            prettyCategory='atom',
                            rowName=cryspy_atom.label,
                            name='occupancy',
                            prettyName='occ',
                            shortPrettyName='occ',
                            icon='fill',
                            categoryIcon='atom',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core',
                            enabled=not cryspy_atom.occupancy_constraint,
                            absDelta=0.05,
                            fittable=True,
                            fit=cryspy_atom.occupancy_refinement
                        ))
                        ed_atom['ADP_type'] = dict(Parameter(
                            'Biso',  # cryspy_atom.adp_type,
                            idx=idx,
                            category='_atom_site',
                            name='ADP_type',
                            shortPrettyName='type',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core'
                        ))
                        b_iso_or_equiv = 0.0
                        b_iso_or_equiv_sigma = 0.0
                        if hasattr(cryspy_atom, 'b_iso_or_equiv'):
                            b_iso_or_equiv = cryspy_atom.b_iso_or_equiv
                            b_iso_or_equiv_sigma = cryspy_atom.b_iso_or_equiv_sigma
                        if hasattr(cryspy_atom, 'u_iso_or_equiv'):
                            b_iso_or_equiv = round(cryspy_atom.u_iso_or_equiv * 8 * np.pi ** 2, 4)
                            b_iso_or_equiv_sigma = round(cryspy_atom.u_iso_or_equiv_sigma * 8 * np.pi ** 2, 4)
                        ed_atom['B_iso_or_equiv'] = dict(Parameter(
                            b_iso_or_equiv,
                            error=b_iso_or_equiv_sigma,
                            idx=idx,
                            category='_atom_site',
                            prettyCategory='atom',
                            rowName=cryspy_atom.label,
                            name='B_iso_or_equiv',
                            prettyName='Biso',
                            shortPrettyName='iso',
                            icon='arrows-alt',
                            categoryIcon='atom',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core',
                            enabled=not cryspy_atom.b_iso_or_equiv_constraint,
                            absDelta=0.1,
                            units='Å²',
                            fittable=True,
                            fit=cryspy_atom.b_iso_or_equiv_refinement
                        ))
                        ed_atom['site_symmetry_multiplicity'] = dict(Parameter(
                            cryspy_atom.multiplicity,
                            optional=True,
                            idx=idx,
                            category='_atom_site',
                            name='site_symmetry_multiplicity',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core'
                        ))
                        ed_atom['Wyckoff_symbol'] = dict(Parameter(
                            cryspy_atom.wyckoff_symbol,
                            optional=True,
                            category='_atom_site',
                            name='Wyckoff_symbol',
                            url='https://docs.easydiffraction.org/app/project/dictionaries/_atom_site/',
                            cifDict='core'
                        ))
                        ed_atoms.append(ed_atom)
                    ed_phase['loops']['_atom_site'] = ed_atoms

            ed_phases.append(ed_phase)

        return ed_phases

    @staticmethod
    def cryspyObjAndDictToEdExperiments(cryspy_obj, cryspy_dict):  # NEED to be modified similar to cryspyObjAndDictToEdModels -> cryspyObjToEdModels

        experiment_names = []
        exp_substrings = ['pd_', 'data_'] # possible experiment prefixes
        # get experiment names from cryspy_dict
        for key in cryspy_dict.keys():
            for substring in exp_substrings:
                if key.startswith(substring):
                    key = key.replace(substring, "").replace("_", "")
                    experiment_names.append(key)

        ed_experiments_meas_only = []
        ed_experiments_no_meas = []

        for data_block in cryspy_obj.items:
            data_block_name = data_block.data_name

            if data_block_name in experiment_names:
                cryspy_experiment = data_block.items
                ed_experiment_no_meas = {'name': '', 'params': {}, 'loops': {}}
                ed_experiment_meas_only = {'name': '', 'loops': {}}

                # DATABLOCK ID

                ed_experiment_no_meas['name'] = dict(Parameter(
                    data_block_name,
                    icon = 'microscope',
                    url = 'https://docs.easydiffraction.org/app/project/dictionaries/',
                ))
                ed_experiment_meas_only['name'] = dict(Parameter(
                    data_block_name,
                    icon = 'microscope',
                    url = 'https://docs.easydiffraction.org/app/project/dictionaries/',
                ))

                for item in cryspy_experiment:

                    # DATABLOCK SINGLES

                    # Ranges category
                    if type(item) == cryspy.C_item_loop_classes.cl_1_range.Range:
                        ed_experiment_no_meas['params']['_pd_meas'] = {}
                        ed_experiment_no_meas['params']['_pd_meas']['2theta_range_min'] = dict(Parameter(
                            item.ttheta_min,
                            optional = True,
                            category = '_pd_meas',
                            name = '2theta_range_min',
                            prettyName = 'range min',
                            shortPrettyName = 'min',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/',
                            cifDict = 'pd'
                        ))
                        ed_experiment_no_meas['params']['_pd_meas']['2theta_range_max'] = dict(Parameter(
                            item.ttheta_max,
                            optional = True,
                            category = '_pd_meas',
                            name = '2theta_range_max',
                            prettyName = 'range max',
                            shortPrettyName = 'max',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/',
                            cifDict = 'pd'
                        ))
                        ed_experiment_no_meas['params']['_pd_meas']['2theta_range_inc'] = dict(Parameter(
                            0.1,  # default value to be updated later
                            optional = True,
                            category = '_pd_meas',
                            name = '2theta_range_inc',
                            prettyName = 'range inc',
                            shortPrettyName = 'inc',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/',
                            cifDict = 'pd'
                        ))

                # Start from the beginnig after reading ranges
                for item in cryspy_experiment:

                    # DATABLOCK SINGLES

                    # Setup section (cryspy)
                    if type(item) == cryspy.C_item_loop_classes.cl_1_setup.Setup:
                        if not '_diffrn_radiation' in ed_experiment_no_meas['params']:
                            ed_experiment_no_meas['params']['_diffrn_radiation'] = {}
                        ed_experiment_no_meas['params']['_diffrn_radiation']['probe'] = dict(Parameter(
                            item.radiation.replace('neutrons', 'neutron').replace('X-rays', 'x-ray'),  # NEED FIX
                            permittedValues = ['neutron', 'x-ray'],
                            category = '_diffrn_radiation',
                            name = 'probe',
                            shortPrettyName = 'probe',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_diffrn_radiation/',
                            cifDict = 'core'
                        ))
                        if not '_diffrn_radiation_wavelength' in ed_experiment_no_meas['params']:
                            ed_experiment_no_meas['params']['_diffrn_radiation_wavelength'] = {}
                        ed_experiment_no_meas['params']['_diffrn_radiation_wavelength']['wavelength'] = dict(Parameter(
                            item.wavelength,
                            error = item.wavelength_sigma,
                            category = '_diffrn_radiation_wavelength',
                            prettyCategory = 'radiation',
                            name = 'wavelength',
                            prettyName = 'wavelength',
                            shortPrettyName = 'wavelength',
                            icon = 'radiation',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_diffrn_radiation/',
                            cifDict = 'core',
                            absDelta = 0.01,
                            units = 'Å',
                            fittable = True,
                            fit = item.wavelength_refinement
                        ))
                        if not '_pd_calib' in ed_experiment_no_meas['params']:
                            ed_experiment_no_meas['params']['_pd_calib'] = {}
                        ed_experiment_no_meas['params']['_pd_calib']['2theta_offset'] = dict(Parameter(
                            item.offset_ttheta,
                            error = item.offset_ttheta_sigma,
                            category = '_pd_calib',
                            prettyCategory = 'calib',
                            name = '2theta_offset',
                            prettyName = '2θ offset',
                            shortPrettyName = 'offset',
                            icon = 'arrows-alt-h',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_calib/',
                            cifDict = 'pd',
                            absDelta = 0.2,
                            units = '°',
                            fittable = True,
                            fit = item.offset_ttheta_refinement
                        ))

                    # Instrument resolution section (cryspy)
                    elif type(item) is cryspy.C_item_loop_classes.cl_1_pd_instr_resolution.PdInstrResolution:
                        if not '_pd_instr' in ed_experiment_no_meas['params']:
                            ed_experiment_no_meas['params']['_pd_instr'] = {}
                        ed_experiment_no_meas['params']['_pd_instr']['resolution_u'] = dict(Parameter(
                            item.u,
                            error = item.u_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'resolution_u',
                            prettyName = 'resolution u',
                            shortPrettyName = 'u',
                            icon = 'grip-lines-vertical',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.1,
                            fittable = True,
                            fit = item.u_refinement
                        ))
                        ed_experiment_no_meas['params']['_pd_instr']['resolution_v'] = dict(Parameter(
                            item.v,
                            error = item.v_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'resolution_v',
                            prettyName = 'resolution v',
                            shortPrettyName = 'v',
                            icon = 'grip-lines-vertical',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.1,
                            fittable = True,
                            fit = item.v_refinement
                        ))
                        ed_experiment_no_meas['params']['_pd_instr']['resolution_w'] = dict(Parameter(
                            item.w,
                            error = item.w_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'resolution_w',
                            prettyName = 'resolution w',
                            shortPrettyName = 'w',
                            icon = 'grip-lines-vertical',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.1,
                            fittable = True,
                            fit = item.w_refinement
                        ))
                        ed_experiment_no_meas['params']['_pd_instr']['resolution_x'] = dict(Parameter(
                            item.x,
                            error = item.x_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'resolution_x',
                            prettyName = 'resolution x',
                            shortPrettyName = 'x',
                            icon = 'grip-lines-vertical',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.1,
                            fittable = True,
                            fit = item.x_refinement
                        ))
                        ed_experiment_no_meas['params']['_pd_instr']['resolution_y'] = dict(Parameter(
                            item.y,
                            error = item.y_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'resolution_y',
                            prettyName = 'resolution y',
                            shortPrettyName = 'y',
                            icon = 'grip-lines-vertical',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.1,
                            fittable = True,
                            fit = item.y_refinement
                        ))

                    # Peak assymetries section (cryspy)
                    elif type(item) is cryspy.C_item_loop_classes.cl_1_pd_instr_reflex_asymmetry.PdInstrReflexAsymmetry:
                        if not '_pd_instr' in ed_experiment_no_meas['params']:
                            ed_experiment_no_meas['params']['_pd_instr'] = {}
                        ed_experiment_no_meas['params']['_pd_instr']['reflex_asymmetry_p1'] = dict(Parameter(
                            item.p1,
                            error = item.p1_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'reflex_asymmetry_p1',
                            prettyName = 'asymmetry p1',
                            shortPrettyName = 'p1',
                            icon = 'balance-scale-left',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.5,
                            fittable = True,
                            fit = item.p1_refinement
                        ))
                        ed_experiment_no_meas['params']['_pd_instr']['reflex_asymmetry_p2'] = dict(Parameter(
                            item.p2,
                            error = item.p2_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'reflex_asymmetry_p2',
                            prettyName = 'asymmetry p2',
                            shortPrettyName = 'p2',
                            icon = 'balance-scale-left',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.5,
                            fittable = True,
                            fit = item.p2_refinement
                        ))
                        ed_experiment_no_meas['params']['_pd_instr']['reflex_asymmetry_p3'] = dict(Parameter(
                            item.p3,
                            error = item.p3_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'reflex_asymmetry_p3',
                            prettyName = 'asymmetry p3',
                            shortPrettyName = 'p3',
                            icon = 'balance-scale-left',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.5,
                            fittable = True,
                            fit = item.p3_refinement
                        ))
                        ed_experiment_no_meas['params']['_pd_instr']['reflex_asymmetry_p4'] = dict(Parameter(
                            item.p4,
                            error = item.p4_sigma,
                            category = '_pd_instr',
                            prettyCategory = 'inst',
                            name = 'reflex_asymmetry_p4',
                            prettyName = 'asymmetry p4',
                            shortPrettyName = 'p4',
                            icon = 'balance-scale-left',
                            url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_instr/',
                            absDelta = 0.5,
                            fittable = True,
                            fit = item.p4_refinement))

                    # Phases section (cryspy)
                    elif type(item) is cryspy.C_item_loop_classes.cl_1_phase.PhaseL:
                        cryspy_phases = item.items
                        ed_phases = []
                        for idx, cryspy_phase in enumerate(cryspy_phases):
                            ed_phase = {}
                            ed_phase['id'] = dict(Parameter(
                                cryspy_phase.label,
                                idx = idx,
                                category = '_pd_phase_block',
                                name = 'id',
                                shortPrettyName = 'label',
                                url = 'https://docs.easydiffraction.org/app/project/dictionaries/_phase/',
                                cifDict = 'pd'
                            ))
                            ed_phase['scale'] = dict(Parameter(
                                cryspy_phase.scale,
                                error = cryspy_phase.scale_sigma,
                                idx = idx,
                                category = '_pd_phase_block',
                                prettyCategory = 'phase',
                                rowName = cryspy_phase.label,
                                name = 'scale',
                                prettyName = 'scale',
                                shortPrettyName = 'scale',
                                icon = 'weight',
                                categoryIcon = 'layer-group',
                                url = 'https://docs.easydiffraction.org/app/project/dictionaries/_phase/',
                                cifDict = 'pd',
                                pctDelta = 25,
                                fittable = True,
                                fit = cryspy_phase.scale_refinement
                            ))
                            ed_phases.append(ed_phase)
                        ed_experiment_no_meas['loops']['_pd_phase_block'] = ed_phases

                    # Background section (cryspy)
                    elif type(item) is cryspy.C_item_loop_classes.cl_1_pd_background.PdBackgroundL:
                        cryspy_bkg_points = item.items
                        ed_bkg_points = []
                        for idx, cryspy_bkg_point in enumerate(cryspy_bkg_points):
                            ed_bkg_point = {}
                            ed_bkg_point['line_segment_X'] = dict(Parameter(
                                cryspy_bkg_point.ttheta,
                                idx = idx,
                                category = '_pd_background',
                                name = 'line_segment_X',
                                prettyName = '2θ',
                                shortPrettyName = '2θ',
                                url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_background/',
                                cifDict = 'pd'
                            ))
                            ed_bkg_point['line_segment_intensity'] = dict(Parameter(
                                cryspy_bkg_point.intensity,
                                error = cryspy_bkg_point.intensity_sigma,
                                idx = idx,
                                category = '_pd_background',
                                prettyCategory = 'bkg',
                                rowName = f'{cryspy_bkg_point.ttheta:g}°',  # formatting float to str without trailing zeros
                                name = 'line_segment_intensity',
                                prettyName = 'intensity',
                                shortPrettyName = 'Ibkg',
                                icon = 'mountain',
                                categoryIcon = 'wave-square',
                                url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_background/',
                                cifDict = 'pd',
                                pctDelta = 25,
                                fittable = True,
                                fit = cryspy_bkg_point.intensity_refinement
                            ))
                            ed_bkg_point['X_coordinate'] = dict(Parameter(
                                '2theta',
                                idx = idx,
                                category = '_pd_background',
                                name = 'X_coordinate',
                                prettyName = 'X coord',
                                shortPrettyName = 'X coord',
                                url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_background/',
                                cifDict = 'pd'
                            ))
                            ed_bkg_points.append(ed_bkg_point)
                        ed_experiment_no_meas['loops']['_pd_background'] = ed_bkg_points

                    # Measured data section (cryspy)
                    elif type(item) is cryspy.C_item_loop_classes.cl_1_pd_meas.PdMeasL:
                        cryspy_meas_points = item.items
                        ed_meas_points = []
                        for idx, cryspy_meas_point in enumerate(cryspy_meas_points):
                            ed_meas_point = {}
                            ed_meas_point['2theta_scan'] = dict(Parameter(
                                cryspy_meas_point.ttheta,
                                idx = idx,
                                category = '_pd_meas',
                                name = '2theta_scan',
                                shortPrettyName = '2θ',
                                url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_meas/',
                                cifDict = 'pd'
                            ))
                            ed_meas_point['intensity_total'] = dict(Parameter(
                                cryspy_meas_point.intensity,
                                idx = idx,
                                category = '_pd_meas',
                                name = 'intensity_total',
                                shortPrettyName = 'I',
                                url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_meas/',
                                cifDict = 'pd'
                            ))
                            ed_meas_point['intensity_total_su'] = dict(Parameter(
                                cryspy_meas_point.intensity_sigma,
                                idx = idx,
                                category = '_pd_meas',
                                name = 'intensity_total_su',
                                shortPrettyName = 'sI',
                                url = 'https://docs.easydiffraction.org/app/project/dictionaries/_pd_meas/',
                                cifDict = 'pd'
                            ))
                            ed_meas_points.append(ed_meas_point)
                        ed_experiment_meas_only['loops']['_pd_meas'] = ed_meas_points

                        # Modify range_inc based on the measured data points in _pd_meas loop
                        pd_meas_2theta_range_min = ed_meas_points[0]['2theta_scan']['value']
                        pd_meas_2theta_range_max = ed_meas_points[-1]['2theta_scan']['value']
                        pd_meas_2theta_range_inc = (pd_meas_2theta_range_max - pd_meas_2theta_range_min) / (len(ed_meas_points) - 1)
                        ed_experiment_no_meas['params']['_pd_meas']['2theta_range_inc']['value'] = pd_meas_2theta_range_inc

            if ed_experiment_meas_only is not None:
                ed_experiments_meas_only.append(ed_experiment_meas_only)
            if ed_experiment_no_meas is not None:
                ed_experiments_no_meas.append(ed_experiment_no_meas)

        return ed_experiments_meas_only, ed_experiments_no_meas


from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.pipeline import Pipeline

from sklearn import svm
from sklearn.linear_model import LogisticRegression

from sklearn.naive_bayes import MultinomialNB, ComplementNB
from sklearn.ensemble import BaggingClassifier

# ------------------------------------- SVM ------------------------------------- #
BOW_TFIDF_UNI_SVM = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1,1), max_df=0.65)),
    ('tfidf', TfidfTransformer(smooth_idf=True, use_idf=True)),
    ('fselect', SelectKBest(chi2, k='all')),
    ('clf', svm.SVC(C=10, gamma=1, kernel='rbf')),
])

# ---------------------------------- LOGISTIC REGRESSION ---------------------------------- #
BOW_TFIDF_UNI_Logistic_Regression = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1,1), max_df=0.65)),
    ('tfidf', TfidfTransformer(smooth_idf=True, use_idf=True)),
    ('fselect', SelectKBest(chi2, k=3500)),
    ('clf', LogisticRegression(C=1, max_iter=1500)),
])

# ----------------------------------- NAIVE-BAYES ----------------------------------- #
BOW_TFIDF_UNI_MultinomialNB = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1, 1), max_df=0.65)),
    ('tfidf', TfidfTransformer(smooth_idf=True, use_idf=True)),
    ('fselect', SelectKBest(chi2, k='all')),
    ('clf', MultinomialNB(alpha=1)),
])

BOW_TFIDF_UNI_ComplementNB = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1, 1), max_df=0.65)),
('tfidf', TfidfTransformer(smooth_idf=True, use_idf=True)),
    ('fselect', SelectKBest(chi2, k=3500)),
    ('clf', ComplementNB(alpha=1)),
])

# ---------------------------------- Bagging ---------------------------------- #
BOW_TFIDF_UNI_Bagging_SVM = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1,1), max_df=1.0)),
    ('tfidf', TfidfTransformer(smooth_idf=True, use_idf=True)),
    ('fselect', SelectKBest(chi2, k='all')),
    ('clf', BaggingClassifier(base_estimator=svm.SVC(), n_estimators=10)),
])

BOW_TFIDF_UNI_Bagging_Logistic_Regression = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1,1), max_df=0.85)),
    ('tfidf', TfidfTransformer(smooth_idf=True, use_idf=True)),
    ('fselect', SelectKBest(chi2, k=3000)),
    ('clf', BaggingClassifier(base_estimator=LogisticRegression(max_iter=1500), n_estimators=100)),
])


from flask import request, current_app
from app.model.categorydb import CategoryManagementDB
from flask_restful import Resource
import json

class CategoryManagement(Resource):
    def post(self):
        try:
            content = request.get_json()
            if content is None:
                current_app.logger.error("No arguments passed to the api call /api/category(post). check parameters")
                return {"Error": "Not all parameters are sent"}, 500
            print ("The content is ", content)
            request_id = content["request_id"] if "request_id" in content else None
            value = content["value"] if "value" in content else None
            description = content["description"] if "description" in content else None
            parent_id = content["parent_id"] if "parent_id" in content else None
            web_category_id = content["web_category_id"] if "web_category_id" in content else None
            merch_category_id = content["merch_category_id"] if "merch_category_id" in content else None
            brand_id = content["brand_id"] if "brand_id" in content else None

            web_subcategory_id = content["web_subcategory_id"] if "web_subcategory_id" in content else None
            web_subcategory_list = content["web_subcategory_list"] if "web_subcategory_list" in content else None
            age_hi = content["age_hi"] if "age_hi" in content else None
            age_lo = content["age_lo"] if "age_lo" in content else None
            gender = content["gender"] if "gender" in content else None
            image_url = content["image_url"] if "image_url" in content else None
            print ("Request id is ", request_id)
            # Insert merch category
            objCategory = CategoryManagementDB()
            output_hash = {}
            if request_id == 1:
                if not objCategory.add_merch_category(value,description,age_lo, age_hi, gender, output_hash):
                    return {"Error": "Unable to add merch category"},400
                print ("Successfully added ", value, output_hash.get("merch_category_id"))
                return {"merch_category_id": output_hash.get("merch_category_id")}, 200
            if request_id == 2:
                if not objCategory.add_web_category(value,description,age_lo, age_hi, gender, output_hash):
                    return {"Error": "Unable to add web category"},400
                print ("Successfully added ", value, output_hash.get("web_category_id"))
                return {"web_category_id": output_hash.get("web_category_id")}, 200
            if request_id == 3:
                if not objCategory.add_web_subcategory(web_subcategory_id, value,description, parent_id, age_lo, age_hi, gender,image_url, output_hash):
                    return {"Error": "Unable to add interests"},400
                print ("Successfully added ", value, output_hash.get("web_subcategory_id"))
                return {"web_subcategory_id": output_hash.get("web_subcategory_id")}, 200
            if request_id == 4:
                if not objCategory.add_brand(value,description, age_lo, age_hi, gender, output_hash):
                    return {"Error": "Unable to addd brand"},400
                print ("Successfully added ", value, output_hash.get("brand_id"))
                return {"brand_id": output_hash.get("brand_id")}, 200
            if request_id == 5: # link subcategory to category
                if not objCategory.link_subcategory(web_category_id, web_subcategory_id,output_hash):
                    return {"Error": "Unable to link subcategory"},400
                print ("Successfully added ", value, output_hash.get("web_subcategory_id"))
                return {"status": "success"}, 200
            if request_id == 6: # link brand to subcategory
                #lsubcat = list(eval(web_subcategory_list))
                print ("Calling brand mapping function")
                if not objCategory.link_brand_to_subcategory(web_subcategory_id,brand_id):
                    return {"status": "Failure"},400
                print ("Successfully added ", value, output_hash.get("web_subcategory_id"))
                return {"status": "success"}, 200
            if request_id == 7: # link merch to web nodes
                if not objCategory.link_merch_to_web_nodes(merch_category_id, web_category_id):
                    return {"status": "Failure"},400
                print ("Successfully added ", merch_category_id, web_category_id)
                return {"status": "success"}, 200
        except Exception as e:
            current_app.logger.error("Error in processing the request" + str(e))
            print ("The error is ", e)
            return {"Error": "Unable to process the request " + str(e)}, 500

    def get(self):
        loutput = []
        #changes are not shown
        objCategory = CategoryManagementDB()
        request_id = request.args.get("request_id", type=int)
        category_id = request.args.get("category_id", type=int)
        subcategory_id = request.args.get("subcategory_id", type=str)
        brand_id = request.args.get("brand_id", type=str)
        subcategory_brand_list = request.args.getlist("subcategory_brand_list")
        age_hi = request.args.get("age_hi", type=int)
        age_lo = request.args.get("age_lo", type=int)
        gender = request.args.get("gender", type=str)
        friend_circle_id = request.args.get("friend_circle_id", type=str)
        output = []
        if request_id == 1 :
            if not objCategory.get_web_category(output):
                return {"Error": "Error in fetching the category"}, 400
            return json.loads(json.dumps(output)), 200

        if request_id == 5:
            if not objCategory.get_distinct_category_subcategory(output):
                return {"Error": "Error in fetching the subcategory to brands combo"}, 400
            return {"data": json.loads(json.dumps(output))}, 200

        if age_lo is None or age_hi is None or gender is None:
            return {"Error": " Age and gender cannot be null"}, 400

        if request_id == 2:
            if not objCategory.get_web_subcategory(category_id,age_lo, age_hi, gender, output):
                return {"Error": "Error in fetching the subcategory"}, 400
            return {json.loads((json.dumps(output)))}, 200

        if request_id == 3:
            if not objCategory.get_brands( age_lo, age_hi, gender,  output):
                return {"Error": "Error in fetching the brands"}, 400
            return {"data": json.loads(json.dumps(output))}, 200

        if request_id == 4:
            if not objCategory.get_web_subcategory_brands(subcategory_brand_list, age_lo, age_hi, gender, output):
                return {"Error": "Error in fetching the subcategory to brands combo"}, 400
            return output, 200


#import necessary Libraries
import bs4
import requests
from bs4 import BeautifulSoup

#Setup Variable
r= requests.get('https://finance.yahoo.com/trending-tickers')
soup = bs4.BeautifulSoup(r.text,'lxml')
table = soup.find_all('table', class_ = 'yfinlist-table W(100%) BdB Bdc($tableBorderGray)')[0]

#Run the program
with open ('Trending_tickers.txt', 'w') as r:
    for row in table.find_all('tr'):
        for cell in row.find_all('td'):
            r.write(cell.text.ljust(37))
        r.write('\n')


from packages import *
from load_data import load_cifar10_1d, load_mnist_1d, load_notmnist, load_yelp

class fcn(nn.Module):
    def __init__(self, dim, hidden_size=100):
        super(fcn, self).__init__()
        self.fc1 = nn.Linear(dim, hidden_size)
        self.activate = nn.ReLU()
        self.fc2 = nn.Linear(hidden_size, 1)
    def forward(self, x):
        return self.fc2(self.activate(self.fc1(x)))

class Neural_epsilon:
    def __init__(self, dim, p = 0.1, lamdba=1, nu=1, hidden=100):
        self.func = fcn(dim, hidden_size=hidden)
        self.context_list = []
        self.reward = []
        self.lamdba = lamdba
        self.total_param = sum(p.numel() for p in self.func.parameters() if p.requires_grad)
        self.U = lamdba * torch.ones((self.total_param,))
        self.nu = nu
        self.p = p  # probability of making exploration

    def select(self, context):
        tensor = torch.from_numpy(context).float()
        mu = self.func(tensor)
        res = []
        for fx in mu:
            res.append(fx.item())
        epsilon = np.random.binomial(1, self.p)
        if epsilon:
            #print("random")
            arm = np.random.choice(len(context), 1)
        else:
            #print("greedy")
            arm = np.argmax(res)
        return arm, res

    def update(self, context, reward):
        self.context_list.append(torch.from_numpy(context.reshape(1, -1)).float())
        self.reward.append(reward)

    def train(self):
        optimizer = optim.SGD(self.func.parameters(), lr=1e-3)
        length = len(self.reward)
        index = np.arange(length)
        np.random.shuffle(index)
        cnt = 0
        tot_loss = 0
        while True:
            batch_loss = 0
            for idx in index:
                c = self.context_list[idx]
                r = self.reward[idx]
                delta = self.func(c) - r
                loss = delta * delta
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
                batch_loss += loss.item()
                tot_loss += loss.item()
                cnt += 1
                if cnt >= 10000:
                    return tot_loss / 1000
            if batch_loss / length <= 1e-3:
                return batch_loss / length

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Neural-epsilon')
    parser.add_argument('--dataset', default='mnist', type=str, help='mnist, cifar10, notmnist, yelp')
    args = parser.parse_args()

    arg_size = 1
    arg_shuffle = 1
    arg_seed = 0
    arg_nu = 1
    arg_lambda = 0.001
    arg_hidden = 100

    if args.dataset == "mnist":
        b = load_mnist_1d()
    elif args.dataset == "cifar10":
        b = load_cifar10_1d()
    elif args.dataset == "yelp":
        b = load_yelp()
    elif args.dataset == "notmnist":
        b = load_notmnist()

    summ = 0
    regrets = []
    l = Neural_epsilon(dim = b.dim)
    for t in range(10000):
        context, rwd = b.step()
        arm_select, res = l.select(context)
        r = rwd[arm_select]
        if not np.isscalar(r):
            r= r[0]
        reg = np.max(rwd) - r
        summ+=reg
        l.update(context[arm_select], r)
        if t<2000:
            if t%10 == 0:
                #print(rwd)
                loss = l.train()
        else:
            if t%100 == 0:
                loss = l.train()
        regrets.append(summ)
        if t % 50 == 0:
            print('{}: {:}, {:.3}, {:.3e}'.format(t, summ, summ/(t+1), loss))

    print("round:", t, summ)


# a module to contain all spotify spotipy API extract and transform functions

import spotipy
from matplotlib import pyplot as plt
from spotipy.oauth2 import SpotifyClientCredentials

from settings import Settings

def generalAuthSpotify(ap_settings):
	"""Initializes general spotify credentials"""
	scc = SpotifyClientCredentials(ap_settings.client_id, ap_settings.client_secret)
	sp = spotipy.Spotify(client_credentials_manager = scc)

	return sp

def getPitchInfo(urn, ap_settings, sp):
	"""Extracts a tracks pitch and rythm info"""
	#Get Track Info
	track = sp.track(urn)
	t_name = track['name']
	t_artist = track['artists'][0]['name']

	#extract audio analysis segments
	audio = sp.audio_analysis(urn)
	bars = audio['bars']
	beats = audio['beats']
	tatums = audio['tatums']
	sections = audio['sections']
	segments = audio['segments']

	#extract segment pitch details
	#create list of pitches
	pitches = []
	#create dictonary of pitches and x values
	pitch_times = {}
	#create list for pitch x and y values
	pitch_times_x = []
	pitch_times_y = []

	#add pitch segment pitch records to that list
	for segment in segments:
		#read in relevant pitch information
		start = segment['start']
		pitch = segment['pitches']
		num_pitches = len(pitch)
		duration = segment['duration']
		#determine segments pitch divisor
		pitch_divisor = duration/num_pitches
		#define x passer
		x_passer = start

		#determine the dominant pitch based on the 12 pitch classes
		#set pitch class determinant parameters
		index = 0
		max = 0
		pitch_int = 0
		num_ones = 0

		for p in pitch:
			#log if there are more than 1 perfect matches
			if p == 1:
				num_ones =+ 1
			#if there is more than 1 perfect match, terminate the loop and log it
			if num_ones > 1:
				too_noisy = True
				p_int = 99
				break
			else:
				too_noisy = False
			#store the highest probabilty of a pitch class
			if p > max:
				max = p
				pitch_int = index
			else:
				pitch_int = pitch_int

			#keep tracking the index
			index += 1
		y_val = pitch_int
		pitch_times_y.append(y_val)

	#get other rhythm information
	#create x coordinate lists. ts = timestamp
	bar_ts = []
	beat_ts = []
	tatum_ts = []
	section_ts = []
	segment_ts = []
	#create y coordinate lists
	bar_y = []
	beat_y = []
	tatum_y = []
	section_y = []
	segment_y = []
	pitch_y = [] #y coordinate of segments as a function of pitch class integer
	#define y coordinates
	y_bar = ap_settings.y_bar
	y_beat = ap_settings.y_beat
	y_tatum = ap_settings.y_tatum
	y_section = ap_settings.y_section
	y_segment = ap_settings.y_segment

	#get segment coordinates for base layers
	for bar in bars:
		x = bar['start']
		bar_ts.append(x)
		bar_y.append(y_bar)
	for beat in beats:
		x = beat['start']
		beat_ts.append(x)
		beat_y.append(y_beat)
	for tatum in tatums:
		x = tatum['start']
		tatum_ts.append(x)
		tatum_y.append(y_tatum)
	for section in sections:
		x = section['start']
		section_ts.append(x)
		section_y.append(y_section)
	for segment in segments:
		#convert it to minutes
		x = segment['start']
		segment_ts.append(x)
		segment_y.append(y_segment)

	#create a dictionary for the return
	pitchinfo = {'track name': t_name, 'track artist': t_artist, 'bar ts': bar_ts,
				'bar y': bar_y, 'beat ts': beat_ts, 'tatum ts': tatum_ts,
				'tatum y': tatum_y, 'section ts': section_ts, 'section y':
				section_y, 'segment ts': segment_ts, 'segment y': segment_y,
				'pitch values': pitch_times_y}
	#print(pitch_times_y)
	#pitch_ys = pitchinfo['pitch values']
	#print(pitch_ys)
	#return all dectionary of extracted values
	return pitchinfo

def pitchClassToFreq(pitch_info, ap_settings):
	"""
	Function that looks up frequencies for pitch class and converts to
	conventional music notes and frequency values
	"""
	#create pitch list
	pitch_notes = []
	#create frequency list
	pitch_freqs = []

	pitch_list = pitch_info['pitch values']

	for pitch in pitch_list:
		#get pitch class note name
		for name, int in ap_settings.pitch_class.items():
			if int == pitch:
				note = name
				pitch_notes.append(note)
				break
		for name, freq in ap_settings.pitch_freq.items():
			if name == note:
				hz = freq
				pitch_freqs.append(hz)
				break
	#create dictionary for return
	pitchtransformed = {'notes': pitch_notes, 'frequencies': pitch_freqs}

	return pitchtransformed

def getTracks(username, sp):
	""" A function to get tracks and info from a group of playlists"""

	#set limits for testing
	testlimit = 1
	#read in the playlists
	plists = sp.user_playlists(username, testlimit)

	#create a track name dictionary of playlist results
	plistTracks = {}

	#fill dictionary with playlist results
	for plist in plists['items']:
		# Get playlist name
		plist_name = plist['name']
		print(plist_name)

		# Read in playlist specific info
		plist = sp.user_playlist(username, plist['id'],
			fields = 'artist, tracks, name, duration_ms, next')

		# Get the tracks from the playlist
		tracks = plist['tracks']

		# Iterate through the tracks and load track info
		for i, item in enumerate(tracks['items']):
			# Get track info
			track = item['track']
			t_name = track['name']
			t_artist = track['artists'][0]['name']
			t_album = track['album']['name']
			t_dur = track['duration_ms']
			t_pop = track['popularity']
			#get tracks audio features if needed
			if return_dets == 'af_y':
				t_uri = track['uri']
				audio = sp.audio_features(t_uri)[0]
			else:
				audio = [0]

			#convert from ms to s and round to the nearest second
			t_dur_sec = t_dur//1000

			#set dictionary key
			entryname = plist_name + '_' + t_name

			#load info into track dictionary
			track = {'playlist': plist_name, 'name': t_name, 'artist': t_artist,
				'album': t_album, 'duration': t_dur_sec, 'popularity': t_pop,
				'audio_features': audio}
			#update main dictionary
			plistTracks[entryname] = track

		# rest to not trip API limit
		time.sleep(0.5)

	return plistTracks

def findDuplicates(tracks, filepath, filename):
	"""Finds duplicates in a dictinary of tracks"""
	# create a duplicate tracking dictionary
	trackDups = {}

	# iterate through the tracks to find duplicates
	for plist_name, track in tracks.items():
		try:
			name = track['name']
			duration = track['duration']
			artist = track['artist']
			popularity = track['popularity']
			# look for existing entries
			if name in trackDups:
				# if a name and duration match, increment the count
				if duration == trackDups[name][0]:
					#print(name + " " + str(trackDups[duration][0]))
					count = trackDups[name][1]
					trackDups[name] = (duration, count+1, artist, popularity)
			else:
				# add dicitonary entry as tuple (duration, count)
				trackDups[name] = (duration, 1, artist, popularity)
		except:
			# ignore
			pass

	total_duplicates = len(tracks) - len(trackDups)
	print("Total Tracks: " + str(len(trackDups)))
	print("Total Duplicates: " + str(total_duplicates))

	# store duplicates as (name, count) tuples
	dups = []
	for k, v in trackDups.items():
		if v[1] > 1:
			dups.append((v[1], k))
	# save duplicates to a file
	#print("# of duplicate tracks to be saved: " + str(len(dups)))
	if len(dups) > 0:
		print("Found %d duplicates. Track names saved to dup.txt" % len(dups))
	else:
		print("No duplicate tracks found!")
	f = open(filepath + filename, "w")
	for val in dups:
		f.write("[%d] %s\n" % (val[0], val[1]))
	f.close()

def displayAudioFeatures(urn):
	track = sp.track(urn)
	t_name = track['name']
	t_artist = track['artists'][0]['name']
	audio = sp.audio_features(urn)[0]
	#acousticness = audio['acousticness']

	print("Artist: " + t_artist)
	print("Song: " + t_name)
	print("\nAudio Analysis: \n")

	for key, value in audio.items():
		try:
			print(key + ' ' + value)
		except:
			print(key + ' ' + str(value))


#Set Values that should not be updated every round (only in the beggining)
import time
import random
lives = 10
compLives = 10
winAmmount = 0
tieAmmount = 0
lossAmmount = 0
optionsArray = ["rock", "paper", "scissors"]
numOptionsArray = [1,2,3]
chosenOption = 0
resultArray = ["Tie","Win","Lose"]
matchCounter = 0

while True:

    #Starting all variables for each round
    isDone = False
    result = 0
    compSetAnswer = ""

    #Player choses option
    option = input("What option will you choose? Rock, Paper, Scissors or Exit\n").lower()

    #Translate the option into a number (I know i could do this more efficient but it works fine like this for now)
    if option == "rock":
        chosenOption = 1
    elif option == "paper":
        chosenOption = 2
    elif option == "scissors":
        chosenOption = 3
    elif option == "exit":
        print("You forfeited " + "\nIt took " + str(matchCounter) + " matches to forfeit\nWins: " + str(winAmmount) + "\nTies: " + str(tieAmmount) + "\nLosses: " + str(lossAmmount))
        txtFile = open("results.txt","w")
        inputTxt = ["Matches: " + str(matchCounter) + "\nWins: " + str(winAmmount) +" \nLosses: " + str(lossAmmount) + "\nTies: " + str(tieAmmount)]
        txtFile.writelines(inputTxt)
        txtFile.close()
        exit()
    else:
        chosenOption = 4

    #Computer choses its option.
    compAnswer = random.choice(numOptionsArray)

    #Computer chosing loading screen
    bar = [
        "<('o'<)",
        "(>‘o’)>",
        "<('o'<)",
        "(>‘o’)>",
        "<('o'<)",
        "(>‘o’)>"
    ]
    i = 0
    while isDone == False:
        print(bar[i % len(bar)], end="\r")
        time.sleep(.4)
        i += 1
        if i == 4:
            isDone = True

    #Changing the computer option to text for use later
    if compAnswer == numOptionsArray[0]:
        compSetAnswer = "rock"
    elif compAnswer == numOptionsArray[1]:
        compSetAnswer = "paper"
    elif compAnswer == numOptionsArray[2]:
        compSetAnswer = "scissors"

    #Checking what the result will be accoring to what you and the computer chose
    if chosenOption == compAnswer:
        result = 1
    elif chosenOption == numOptionsArray[0]:
        if compAnswer == numOptionsArray[1]:
            result = 3
        if compAnswer == numOptionsArray[2]:
            result = 2
    elif chosenOption == numOptionsArray[1]:
        if compAnswer == numOptionsArray[0]:
            result = 2
        if compAnswer == numOptionsArray[2]:
            result = 3
    elif chosenOption == numOptionsArray[2]:
        if compAnswer == numOptionsArray[0]:
            result = 3
        if compAnswer == numOptionsArray[1]:
            result = 2
    else:
        result = 4

    #Changing values depending on what the result is and also chosing what result is what
    if result == 1:
        print("It was a tie.\nYou chose " + str(option) + " and your opponent chose " + str(compSetAnswer) + "\nLives: " + str(lives) + "\n")
        tieAmmount += 1
        matchCounter += 1
    elif result == 2:
        print("You won.\nYou chose " + str(option) + " and your opponent chose " + str(compSetAnswer) + "\nLives: " + str(lives) + "\n")
        winAmmount += 1
        lives += 1
        compLives += -1
        matchCounter += 1
    elif result == 3:
        print("You lost.\nYou chose " + str(option) + " and your opponent chose " + str(compSetAnswer) + "\nLives: " + str(lives) + "\n")
        lossAmmount += 1
        lives += -1
        compLives += 1
        matchCounter += 1
    elif result == 4:
        print("Incorrect answer try again\n")

    #Checking if you succed in beating the computer or lose
    if compLives < 1:
        print("YOU ARE THE CHAMPION CONGRATULAITONS\nIt took " + str(matchCounter) + " matches to win\nWins: " + str(winAmmount) + "\nTies: " + str(tieAmmount) + "\nLosses: " + str(lossAmmount))
        txtFile = open("results.txt","w")
        inputTxt = ["Matches: " + str(matchCounter) + "\nWins: " + str(winAmmount) +" \nLosses: " + str(lossAmmount) + "\nTies: " + str(tieAmmount)]
        txtFile.writelines(inputTxt)
        txtFile.close()
        exit()
    elif lives < 1:
        print("You lost better luck next time!\nIt took " + str(matchCounter) + " matches to lose\nWins: " + str(winAmmount) + "\nTies: " + str(tieAmmount) + "\nLosses: " + str(lossAmmount))
        txtFile = open("results.txt","w")
        inputTxt = ["Matches: " + str(matchCounter) + "\nWins: " + str(winAmmount) +" \nLosses: " + str(lossAmmount) + "\nTies: " + str(tieAmmount)]
        txtFile.writelines(inputTxt)
        txtFile.close()
        exit()

#Copyright © Viktor Landström 2019


import numpy as np

from tensorflow.python.eager import backprop
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import custom_gradient
from tensorflow.python.ops import \
gradient_checker_v2 as gradient_checker
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import sparse_ops
# needs this to register gradient for SoftmaxCrossEntropyWithLogits:
import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
from tensorflow.python.platform import test
from tensorflow.python.platform import tf_logging

def _random_complex(shape, dtype):
  data = np.random.random_sample(shape).astype(dtype.as_numpy_dtype)
  if dtype.is_complex:
    data.imag = np.random.random_sample(shape)
  return data

@test_util.run_all_in_graph_and_eager_modes
class GradientCheckerTest(test.TestCase):

  def testSparseTensorReshape(self):
    x = constant_op.constant(2.0, shape=(2,))

    def sparse_tensor_reshape(values):
      sparse = sparse_tensor.SparseTensor(
          indices=[[0, 0], [1, 2]], values=values, dense_shape=[3, 4])
      sparse = sparse_ops.sparse_reshape(sparse, shape=(12,))
      return sparse.values

    error = gradient_checker.max_error(
        *gradient_checker.compute_gradient(sparse_tensor_reshape, [x]))

    self.assertLess(error, 1e-4)

  def testWithStaticShape(self):
    size = (2, 3)
    constant = constant_op.constant(2.0, shape=size, name="const")

    def add_constant_with_static_shape_check(x):
      self.assertAllEqual(x.shape.as_list(), constant.shape.as_list())
      return x + constant

    x = constant_op.constant(3.0, shape=size, name="x")

    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
        add_constant_with_static_shape_check, [x]))

    self.assertLess(error, 1e-4)

  def testWithArgumentsAsTuple(self):
    size = (2, 3)
    x1 = constant_op.constant(2.0, shape=size, name="x1")
    x2 = constant_op.constant(3.0, shape=size, name="x2")

    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
        lambda x1: math_ops.add(x1, x2), (x1,)))

    tf_logging.info("x1 error = %f", error)
    self.assertLess(error, 1e-4)

  def testAddSimple(self):
    size = (2, 3)
    x1 = constant_op.constant(2.0, shape=size, name="x1")
    x2 = constant_op.constant(3.0, shape=size, name="x2")
    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
        lambda x1: math_ops.add(x1, x2), [x1]))
    tf_logging.info("x1 error = %f", error)
    self.assertLess(error, 1e-4)

  def testBfloat16(self):
    x1 = constant_op.constant(2.0, dtype="bfloat16")
    x2 = constant_op.constant(3.0, dtype="bfloat16")
    # bfloat16 is very imprecise, so we use very large delta and error bar here.
    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
        lambda x1: math_ops.add(x1, x2), [x1], delta=0.1))
    tf_logging.info("x1 error = %f", error)
    self.assertLess(error, 0.07)

  def testAddCustomized(self):
    size = (2, 3)
    x1 = constant_op.constant(2.0, shape=size, dtype=dtypes.float64, name="x1")
    x2 = np.asarray(np.arange(6, dtype=np.float64).reshape(2, 3))
    # checkint gradients for x2 using a special delta
    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
        lambda x2: math_ops.add(x1, x2), [x2], delta=1e-2))
    tf_logging.info("x2 error = %f", error)
    self.assertLess(error, 1e-10)

  def testGather(self):

    def f(params):
      index_values = [1, 3]
      indices = constant_op.constant(index_values, name="i")
      return array_ops.gather(params, indices, name="y")

    p_shape = (4, 2)
    p_size = 8
    params = constant_op.constant(
        np.arange(p_size).astype(np.float64), shape=p_shape, name="p")
    error = gradient_checker.max_error(
        *gradient_checker.compute_gradient(f, [params]))
    tf_logging.info("gather error = %f", error)
    self.assertLess(error, 1e-4)

  def testNestedGather(self):

    def f(params):
      index_values = [1, 3, 5, 6]
      indices = constant_op.constant(index_values, name="i")
      y = array_ops.gather(params, indices, name="y")
      index_values2 = [0, 2]
      indices2 = constant_op.constant(index_values2, name="i2")
      return array_ops.gather(y, indices2, name="y2")

    p_shape = (8, 2)
    p_size = 16
    params = constant_op.constant(
        np.arange(p_size).astype(np.float64), shape=p_shape, name="p")
    error = gradient_checker.max_error(
        *gradient_checker.compute_gradient(f, [params]))
    tf_logging.info("nested gather error = %f", error)
    self.assertLess(error, 1e-4)

  def testComplexMul(self):
    c = constant_op.constant(5 + 7j, dtype=dtypes.complex64)

    def f(x):
      return c * x

    x_shape = c.shape
    x_dtype = c.dtype
    x = constant_op.constant(_random_complex(x_shape, x_dtype))
    analytical, numerical = gradient_checker.compute_gradient(f, [x])
    correct = np.array([[5, -7], [7, 5]])
    self.assertAllEqual(correct, analytical[0])
    self.assertAllClose(correct, numerical[0], rtol=1e-4)
    x = constant_op.constant(_random_complex(x_shape, x_dtype))
    self.assertLess(
        gradient_checker.max_error(*gradient_checker.compute_gradient(f, [x])),
        3e-4)

  def testComplexConj(self):

    def f(x):
      return math_ops.conj(x)

    x_shape = ()
    x_dtype = dtypes.complex64
    x = constant_op.constant(_random_complex(x_shape, x_dtype))
    analytical, numerical = gradient_checker.compute_gradient(f, [x])
    correct = np.array([[1, 0], [0, -1]])
    self.assertAllEqual(correct, analytical[0])
    self.assertAllClose(correct, numerical[0], rtol=2e-5)
    x = constant_op.constant(_random_complex(x_shape, x_dtype))
    self.assertLess(
        gradient_checker.max_error(*gradient_checker.compute_gradient(f, [x])),
        2e-5)

  def testEmptySucceeds(self):

    def f(x):
      return array_ops.identity(x)

    x = constant_op.constant(
        np.random.random_sample((0, 3)), dtype=dtypes.float32)
    for grad in gradient_checker.compute_gradient(f, [x]):
      self.assertEqual(grad[0].shape, (0, 0))
    error = gradient_checker.max_error(
        *gradient_checker.compute_gradient(f, [x]))
    self.assertEqual(error, 0)

  def testEmptyMatMul(self):

    def f(x, y):
      return math_ops.matmul(x, y)

    x = constant_op.constant(
        np.random.random_sample((0, 3)), dtype=dtypes.float32)
    y = constant_op.constant(
        np.random.random_sample((3, 4)), dtype=dtypes.float32)
    for grad in gradient_checker.compute_gradient(f, [x, y]):
      self.assertEqual(grad[0].shape, (0, 0))
      self.assertEqual(grad[1].shape, (0, 12))
    error = gradient_checker.max_error(
        *gradient_checker.compute_gradient(f, [x, y]))
    self.assertEqual(error, 0)

  def testEmptyFails(self):

    @custom_gradient.custom_gradient
    def id_bad_grad(x):
      y = array_ops.identity(x)

      def grad_fn(dy):
        # dx = constant_op.constant(np.zeros((1, 4)), dtype=dtypes.float32)
        dx = array_ops.transpose(dy)
        return dx

      return y, grad_fn

    def f(x):
      return id_bad_grad(x)

    x = constant_op.constant(
        np.random.random_sample((0, 3)), dtype=dtypes.float32)
    bad = r"Empty gradient has wrong shape: expected \(0, 3\), got \(3, 0\)"
    with self.assertRaisesRegex(ValueError, bad):
      gradient_checker.compute_gradient(f, [x])

  def testNaNGradFails(self):

    @custom_gradient.custom_gradient
    def id_nan_grad(x):
      y = array_ops.identity(x)

      def grad_fn(dy):
        dx = np.nan * dy
        # dx = dy
        return dx

      return y, grad_fn

    def f(x):
      return id_nan_grad(x)

    x = constant_op.constant(
        np.random.random_sample((1, 1)), dtype=dtypes.float32)
    error = gradient_checker.max_error(
        *gradient_checker.compute_gradient(f, [x]))
    # Typical test would assert error < max_err, so assert this test would
    # raise AssertionError, since NaN is not < 1.0.
    with self.assertRaisesRegex(AssertionError, "nan not less than 1.0"):
      self.assertLess(error, 1.0)

  def testGradGrad(self):

    def f(x):
      with backprop.GradientTape() as tape:
        tape.watch(x)
        y = math_ops.square(x)
        z = math_ops.square(y)
      return tape.gradient(z, x)

    analytical, numerical = gradient_checker.compute_gradient(f, [2.0])
    self.assertAllEqual([[[48.]]], analytical)
    self.assertAllClose([[[48.]]], numerical, rtol=1e-4)

@test_util.run_all_in_graph_and_eager_modes
class MiniMNISTTest(test.TestCase):

  # Gradient checker for MNIST.
  def _BuildAndTestMiniMNIST(self, param_index, tag):
    # Fix seed to avoid occasional flakiness
    np.random.seed(6)

    # Hyperparameters
    batch = 3
    inputs = 16
    features = 32
    classes = 10

    # Define the parameters
    inp_data = np.random.random_sample(inputs * batch)
    hidden_weight_data = np.random.randn(inputs * features) / np.sqrt(inputs)
    hidden_bias_data = np.random.random_sample(features)
    sm_weight_data = np.random.randn(features * classes) / np.sqrt(features)
    sm_bias_data = np.random.random_sample(classes)

    # special care for labels since they need to be normalized per batch
    label_data = np.random.random(batch * classes).reshape((batch, classes))
    s = label_data.sum(axis=1)
    label_data /= s[:, None]

    # We treat the inputs as "parameters" here
    inp = constant_op.constant(
        inp_data.tolist(),
        shape=[batch, inputs],
        dtype=dtypes.float64,
        name="inp")
    hidden_weight = constant_op.constant(
        hidden_weight_data.tolist(),
        shape=[inputs, features],
        dtype=dtypes.float64,
        name="hidden_weight")
    hidden_bias = constant_op.constant(
        hidden_bias_data.tolist(),
        shape=[features],
        dtype=dtypes.float64,
        name="hidden_bias")
    softmax_weight = constant_op.constant(
        sm_weight_data.tolist(),
        shape=[features, classes],
        dtype=dtypes.float64,
        name="softmax_weight")
    softmax_bias = constant_op.constant(
        sm_bias_data.tolist(),
        shape=[classes],
        dtype=dtypes.float64,
        name="softmax_bias")

    # List all the parameter so that we can test them one at a time
    all_params = [inp, hidden_weight, hidden_bias, softmax_weight, softmax_bias]

    # Now, Building MNIST
    def f(inp, hidden_weight, hidden_bias, softmax_weight, softmax_bias):
      features = nn_ops.relu(
          nn_ops.xw_plus_b(inp, hidden_weight, hidden_bias), name="features")
      logits = nn_ops.xw_plus_b(
          features, softmax_weight, softmax_bias, name="logits")
      labels = constant_op.constant(
          label_data.tolist(),
          shape=[batch, classes],
          dtype=dtypes.float64,
          name="labels")
      cost = nn_ops.softmax_cross_entropy_with_logits(
          labels=labels, logits=logits, name="cost")
      return cost

    def f_restricted(x):
      xs = all_params
      i = param_index
      # use x for the i-th parameter
      xs = xs[0:i] + [x] + xs[i + 1:]
      return f(*xs)

    # Test the gradients.
    err = gradient_checker.max_error(*gradient_checker.compute_gradient(
        f_restricted, [all_params[param_index]], delta=1e-5))

    tf_logging.info("Mini MNIST: %s gradient error = %g", tag, err)
    return err

  def testInputGradient(self):
    self.assertLess(self._BuildAndTestMiniMNIST(0, "input"), 1e-8)

  def testHiddenWeightGradient(self):
    self.assertLess(self._BuildAndTestMiniMNIST(1, "hidden_weight"), 1e-8)

  def testHiddenBiasGradient(self):
    self.assertLess(self._BuildAndTestMiniMNIST(2, "hidden_bias"), 1e-8)

  def testSoftmaxWeightGradient(self):
    self.assertLess(self._BuildAndTestMiniMNIST(3, "softmax_weight"), 1e-8)

  def testSoftmaxBiasGradient(self):
    self.assertLess(self._BuildAndTestMiniMNIST(4, "softmax_bias"), 1e-8)

if __name__ == "__main__":
  test.main()


import requests
import json
import pymongo
import math
from urllib.parse import urljoin
import os
# response = requests.get("http://www.cninfo.com.cn/cninfo-new/js/data/szse_stock.json")

def announcement(orgId,category,code,zwjc,page="1"):
    index = 1
    url = "http://www.cninfo.com.cn/cninfo-new/announcement/query"
    post_data = {
        "stock": "{},{};".format(code,orgId),
        "searchkey": "",
        "plate": "",
        "category": "category_ndbg_szsh;",
        "trade": "",
        "column:szse": "",
        "columnTitle": "历史公告查询",
        "pageNum": page,
        "pageSize": "30",
        "tabName": "fulltext",
        "sortName": "",
        "sortType": "",
        "limit": "",
        "showTitle": "",
        "seDate": "category_ndbg_szsh/category/年度报告;{},{}/stock/{} {}".format(code,orgId,code,zwjc)
    }
    while True:
        try:
            if index>5:
                return False,None
                break
            response = requests.post(url=url, data=post_data)
            content = response.content
            content = json.loads(content)
            return True,[content["totalAnnouncement"], content['announcements']]
        except:
            index += 1

def run():
    client = pymongo.MongoClient(host="192.168.1.36", port=27017)
    db = client["scrapy"]
    col = db["companyDict"]
    col2 = db["annualReport20170516"]
    for item in col.find():
        print(item)
        tag,body= announcement(item["orgId"],item['category'],item['code'],item['zwjc'])
        if not tag:
            continue
        for page in range(math.ceil(int(body[0])/30)):
            tag,body = announcement(item["orgId"], item['category'], item['code'], item['zwjc'],page=page+1)
            if not tag:
                continue
            col2.insert(body[1])
def writeurl():
    client = pymongo.MongoClient(host="192.168.1.36", port=27017)
    db = client["scrapy"]
    col = db["annualReport20170516"]
    fp = open("urls.txt","w")
    for item in col.find():
        atype = item["adjunctType"]
        if atype !="PDF":
            continue
        aurl = item["adjunctUrl"]
        link = urljoin("http://www.cninfo.com.cn",aurl)
        print(link)
        log = "{}****{}****{}\n".format(item["secCode"],link,item["announcementTitle"])
        fp.write(log)
    fp.close()
def downpdf():
    client = pymongo.MongoClient(host="192.168.1.36", port=27017)
    db = client["scrapy"]
    col = db["annualReport20170516"]
    for item in col.find():
        atype = item["adjunctType"]
        if atype !="PDF":
            continue
        aurl = item["adjunctUrl"]
        link = urljoin("http://www.cninfo.com.cn",aurl)
        request = requests.get(link)
        file_tag = os.path.exists(item["secCode"])
        if file_tag:
            pass
        else:
            os.mkdir(item["secCode"])
        file_name = "{}/{}.pdf".format(item['secCode'],item['announcementTitle'])
        with open(file_name, "wb") as code:
            code.write(request.content)
        break

if __name__ == "__main__":
    # run()
    downpdf()


# works with Python 2.7

while True:
    jack, jill = [int(i) for i in raw_input().split()]
    sell=0
    if jack==0 and jill==0:
        break
    jackList =set(int(raw_input()) for _ in range(jack))
    jillList =set(int(raw_input()) for _ in range(jill))
    diff = jackList&jillList
    print(len(diff))


import http.client
import json
import math

CONN = http.client.HTTPSConnection('qiita.com', 443)
USER_ID = 'riekure'
PER_PAGE = 100

class Api:
    # リクエスト結果をJSON形式で返す
    @staticmethod
    def request(http, url) :
        CONN.request(http, url)
        res = CONN.getresponse()
        data = res.read().decode('utf-8')
        return json.loads(data)

    # 投稿数とからページ番号を計算する
    @staticmethod
    def page_count(items_count) :
        return math.floor(items_count / PER_PAGE) + 1

# 投稿数を取得
items_count = Api.request('GET', '/api/v2/users/' + USER_ID)['items_count']
page = Api.page_count(items_count)

# 投稿記事を全て取得
all_article = {}
for i in range(page) :
     article = Api.request('GET', '/api/v2/users/' + USER_ID + '/items?page=' + str(i+1) + '&per_page=' + str(PER_PAGE))
     for j in range(PER_PAGE) :
         try :
             all_article[article[j]['title']] = article[j]['likes_count']
         except IndexError :
             break

# いいね（LGTM）の降順にソート
# items() を使用するので tuple になる
tuple_items = sorted(all_article.items(), key=lambda x:x[1], reverse=True)

# markdown の表形式で表示
print('| 記事タイトル | いいねカウント |')
print('|------------|--------------|')
for title, likes_count in tuple_items:
    print('| ' + title + ' | ' + str(likes_count) + ' |')


from django.urls import path

from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('action/<int:action_id>/', views.action, name='action'),
    path('activity/', views.activity, name='activity'),
    path('activity/<int:activity_id>/task/<task_id>/', views.task, name='task'),
]


# This script creates the index in Elasticsearch
# It can delete the index if it already exists

from elasticsearch import Elasticsearch
from elasticsearch import helpers
import json
import os

DELETE_EXISTING = True
INDEX_NAME = 'reference'

mapping_path = '../json/index_mappings.json'
settings_path = '../json/index_settings.json'

# Setup the main index to recieve the data
if es.indices.exists(index=INDEX_NAME) and DELETE_EXISTING:
    es.indices.delete(index=INDEX_NAME)
if not es.indices.exists(index=INDEX_NAME):
    with open(mapping_path, "r") as file:
                mapping = json.load(file)
    with open(settings_path, "r") as file:
                settings = json.load(file)
    es.indices.create(index=INDEX_NAME,mappings=mapping,settings=settings)


from pathlib import Path
import numpy as np
import pandas as pd

def load_data(datapath: Path) -> pd.DataFrame:
    org_df = pd.read_csv(datapath, delimiter='\t', index_col=0)
    org_df.index = pd.to_datetime(org_df.index)
    return org_df

def get_mean_draw_velocity(sampled_df: pd.DataFrame) -> float:
    distance_pow = (sampled_df - sampled_df.shift(1)).pow(2)
    distance_sum = distance_pow['x'] + distance_pow['y']
    distance = np.sqrt(distance_sum).sum()
    velocity = distance / (sampled_df.index.max() - sampled_df.index.min()).seconds
    return velocity

def calculate_center_of_mass(sampled_df: pd.DataFrame) -> tuple:
    return sampled_df['x'].mean(), sampled_df['y'].mean()

def distance_between_points(p1: tuple, p2: tuple) -> float:
    return np.sqrt((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2)

def get_distance_to_point(sampled_df: pd.DataFrame, point: tuple) -> pd.Series:
    return sampled_df.apply(lambda x: distance_between_points((x['x'], x['y']), point),
                            axis=1)

def metric_system_change():
    length_unit = input('Which unit of length do you want to use:  ')
    return length_unit

def time_unit_system_change():
    unit = input('Which unit do you want it converted to: [default second][ns/ms/s/minutes/hours/days]')
    if unit == "ns":
        ans = 10 * 6
    elif unit == "ms":
        ans = 10 * 3
    elif unit == "minutes":
        ans = 1 / 60
    elif unit == "hours":
        ans = 1 / 60 / 60
    elif unit == "days":
        ans = 1 / 60 / 60 / 24
    else:
        ans = 1
        unit = 's'
    return ans, unit

def length_unit_system_change():
    unit1 = input('Which unit do you want it converted from:  [cm/mm/m/km/ft/inch]')
    unit2 = input('Which unit do you want it converted to: [cm/mm/m/km/ft/inch]')

    if unit1 == "cm" and unit2 == "m":
        ans = 1 / 100

    elif unit1 == "mm" and unit2 == "cm":
        ans = 1 / 10

    elif unit1 == "m" and unit2 == "cm":
        ans = 1 * 100

    elif unit1 == "cm" and unit2 == "mm":
        ans = 1 * 10

    elif unit1 == "mm" and unit2 == "m":
        ans = 1 / 1000

    elif unit1 == "m" and unit2 == "mm":
        ans = 1 * 1000

    elif unit1 == "km" and unit2 == "m":
        ans = 1 * 1000

    elif unit1 == "m" and unit2 == "km":
        ans = 1 / 1000

    elif unit1 == "mm" and unit2 == "km":
        ans = 1 / 1000000

    elif unit1 == "ft" and unit2 == "cm":
        ans = 1 * 30.48

    elif unit1 == "ft" and unit2 == "mm":
        ans = 1 * 304.8

    elif unit1 == "ft" and unit2 == "inch":
        ans = 1 * 12

    elif unit1 == "inch" and unit2 == "cm":
        ans = 1 * 2.54
    elif unit1 == "inch" and unit2 == "mm":
        ans = 1 * 25.4
    else:
        ans, unit2 = 1, unit2

    return ans, unit2

def get_reference_frame():
    px = input('Get axis x reference point')
    py = input('Get axis y reference point')
    px = float(px)
    py = float(py)

    return px, py


#!/usr/bin/python3

def divisible_by_2(my_list=[]):
    result = []  # Create an empty list to store True or False values

    for num in my_list:
        if num % 2 == 0:
            result.append(True)
            # If the integer is a multiple of 2, append True
        else:
            result.append(False)  # If not, append False

    return result