"""Config file and logging related utility functions."""
import configparser
import json
import os
import sys
from pprint import pprint
import yaml

def read_cfg(location, verbose=True):
    """
    Read config file at location using ConfigParser.

    Parameters
    ----------
    location : str
        Where the config file is located
    verbose : bool, optional, defaults to True
        Should print the contents of the read config file.

    Returns
    -------
    ConfigParser
        The python ConfigParser object after reading the cfg.

    """
    if not os.path.exists(location):
        raise ValueError(f"Config file {location} does not exist")

    config = configparser.ConfigParser()
    config.read(location)

    if verbose:
        print_cfg(config, "Program started with configuration")
    return config

def print_cfg(config, msg=""):
    """
    Print the contents of a ConfigParser object.

    Parameters
    ----------
    config : ConfigParser
        The ConfigParser to print the contents of.
    msg: str, optional, defaults to ""
        Message to print before printing the config file.

    Returns
    -------
    None

    """
    if msg != "":
        print(msg)
    config_dict = [{x: tuple(config.items(x))} for x in config.sections()]
    pprint(config_dict, width=120)

def parse_args(parser, verbose=True):
    """
    Parse command line arguments into a Namespace.

    Parameters
    ----------
    verbose : bool, optional, defaults to True
        Should print the values of the command line args.

    Returns
    -------
    Namespace
        Parsed arguments.

    Raises
    ------
    ValueError
        If any arguments are passed which are not used in program.

    """
    args, unparsed = parser.parse_known_args()

    if len(unparsed) != 0:
        raise ValueError(
            "Unrecognised command line arguments passed {}".format(unparsed)
        )

    if verbose:
        if len(sys.argv) > 1:
            print("Command line arguments", args)
    return args

def read_python(path, dirname_replacement=""):
    """
    Execute a python script at path.

    The script is expected to have items visible at global scope,
    which are stored as metadata.

    Note
    ----
    The string "__thisdirname__" is magic and will be replaced by the
    absolute path to the directory containing the script.
    The string "__dirname__" is also magic and will be replaced by
    the value of dirname_replacement.

    Parameters
    ----------
    path : string
        The location of the python script.
    dirname_replacement : string, optional, optional, defaults to None
        What to replace __dirname__ with.
        By default, None will replace __dirname__ with dirname of path.

    Returns
    -------
    dict
        The scripts global scope variables stored in a dictionary.

    """

    def normalise_path(pth):
        s = os.path.abspath(pth)
        s = s.replace(os.sep, "/")
        return s

    path = os.path.realpath(os.path.expanduser(path))
    if not os.path.exists(path):
        raise ValueError("{} does not exist to read".format(path))
    with open(path, "r") as f:
        contents = f.read()
    if dirname_replacement != "":
        contents = contents.replace("__dirname__", normalise_path(dirname_replacement))
    else:
        contents = contents.replace(
            "__dirname__", normalise_path(os.path.dirname(path))
        )
    contents = contents.replace(
        "__thisdirname__", normalise_path(os.path.dirname(path))
    )
    metadata = {}
    try:
        exec(contents, {}, metadata)
    except Exception as e:
        import traceback

        print("QUITTING: An error occurred reading {}".format(path))
        traceback.print_exc()
        exit(-1)
    metadata = {k.lower(): v for (k, v) in metadata.items()}
    return metadata

def read_yaml(path):
    with open(path, "r") as stream:
        parsed_yaml = yaml.safe_load(stream)
    return parsed_yaml

def read_json(path):
    with open(path, "r") as stream:
        parsed_json = json.load(stream)
    return parsed_json

def split_dict(in_dict, index):
    """
    Grab the value at index from each list in the dictionary.

    Parameters
    ----------
    in_dict : dict
        The dictionary to grab from
    index : int
        The index in the lists to pull from

    Returns
    -------
    dict
        The original dictionary but with index values pulled out.

    """
    new_dict = {}
    for key, value in in_dict.items():
        if isinstance(value, list):
            new_dict[key] = value[index]
    return new_dict

def convert_dict_to_string(in_dict, name):
    """
    Convert the underlying parameters dictionary to string.

    Can be useful for printing or writing to a file.
    Does not overwrite default __str__ as the output is quite verbose.

    Parameters
    ----------
    in_dict : dict
        Input dictionary

    Returns
    -------
    str
        The string representation of the dict.

    """

    def _val_to_str(val):
        """
        Convert a value to a string.

        One caveat, if a string is passed, it returns
        the original string wrapped in quotes.

        Parameters
        ----------
        val : object
            The value to convert

        Returns
        -------
        str
            The value as a string.

        """
        return f"'{val}'" if isinstance(val, str) else val

    out_str = ""
    out_str += name + " = {\n"
    for k, v in in_dict.items():
        out_str += f"\t{_val_to_str(str(k))}:"
        if isinstance(v, dict):
            out_str += "\n\t\t{\n"
            for k2, v2 in v.items():
                out_str += "\t\t {}: {},\n".format(
                    _val_to_str(str(k2)), _val_to_str(v2)
                )
            out_str += "\t\t},\n"
        else:
            out_str += f" {_val_to_str(v)},\n"
    out_str += "\t}"
    return out_str


import dash_core_components as dcc
import dash_html_components as html
import dash_bootstrap_components as dbc
import dash
from app import app
from app import server

from apps.gas_monitoring import gas_app

navBar = dbc.NavbarSimple(
    children=[

        dbc.NavItem(dbc.NavLink("Home", href="/")),
        dbc.NavItem(dbc.NavLink("Gas-system", href="/gas-monitoring")),
        dbc.NavItem(dbc.NavLink("Oil-system", href="/oil-monitoring")),

    ],
    brand="G.O.M",
    brand_href="/",
    color="primary",
    dark=True,
)
app.layout = html.Div(
    [
        dcc.Location(id='url', refresh=False),
        navBar,
        html.Div(
        id='content',

       )
    ]
)

error_page = html.Div([
        html.H1("404",style={"textAlign":"center"}),
        html.H3("Page Not Found!",style={"textAlign":"center"})
    ])

index_page = html.Div(
    [
        html.Div([

            html.H2("Welcome to Gas Oil plant monitoring System."),
            html.P("""Lorem ipsum dolor sit amet ac maximusrdiet convallis. Duis rutrum neque consectetur mauris tempor laoreet. Vestibulum quis nulla eu orci efficitur varrisque vel nibh. Integer eu velit eget ex consectetur consectetur sit amet vitae lectus. Mauris egestas purus et mi pulvinar, a posuere justo convallis. Nunc nec laoreet lectus. Mauris purus est, bibendum hendrerit fermentum quis, porttitor at massa.""")
        ],
        style={
            'text-align': 'center',
            'position': 'absolute',
            'top': '50%',
            'left': '50%',
            'transform': 'translate(-50%, -50%)',
            'color': 'white',

        })

    ],

    style={"textAlign":"center",
            'backgroundImage': 'url("assets/images/background.jpg")',
            'backgroundRepeat': 'no-repeat',
            'backgroundPosition': 'center',
            'backgroundSize' : 'cover',
            'height':'50vh',
            'position':'relative',
            },
    )

@app.callback(dash.dependencies.Output('content', 'children'),
              [dash.dependencies.Input('url', 'pathname')])
def display_page(pathname):
    if pathname == '/gas-monitoring':
        return gas_app.layout
    elif pathname == "/oil-monitoring":
        return gas_app.layout
    elif pathname == '/':
        return index_page
    else:
        return error_page

if __name__ == '__main__':
    app.run_server(debug=True)


from . import utils
import time
from .monitor import PostgresMonitor

__all__ = ['QuerySet', 'query', 'update', 'insert']

key = str(time.time())

async def query(sql):
    return await PostgresMonitor.fetch(sql)

async def update(sql):
    return await PostgresMonitor.fetch(sql)

async def insert(sql):
    return await PostgresMonitor.fetchrow(sql)

class QuerySet(object):
    _sql = {
        'get_list': 'SELECT {fields} from {table} {extra} LIMIT {size} OFFSET {offset}',
        'filter': 'SELECT {fields} from {table} WHERE {rule} LIMIT {size} OFFSET {offset}',
        'count': 'SELECT COUNT({field}) FROM {table}',
        'count_on_rule': 'SELECT COUNT({field}) FROM {table} WHERE {rule}',
        'orderby': 'ORDER BY {field}',
        'nearby': 'select {fields} difference from {table} where {rule} and {value} > {column} limit 1',
        'orderby_decr': 'ORDER BY {field} DECR',
        'filter_with_orderby': "SELECT {fields} from {table} WHERE {rule} ORDER BY {sort_key} LIMIT {size} OFFSET {offset};",
        'filter_with_orderby_decr': "SELECT {fields} from {table} WHERE {rule} ORDER BY {sort_key} LIMIT {size} OFFSET {offset};",
        'filter_in': "SELECT {fields} FROM {table} WHERE {key} IN ({targets});",
        'filter_in_range': "SELECT {fields} FROM {table} WHERE {rule} and {key} <= {end} and {key} >= {start};",
        'find_in_range': "SELECT {fields} FROM {table} WHERE {key} <= {end} and {key} >= {start};",
        'find_near': "SELECT {fields} FROM {table} WHERE {key} >= {start};",
        'insert': 'INSERT INTO {table} ({keys}) VALUES ({values}) RETURNING id;',
        'replace': 'REPLACE INTO {table} ({keys}) VALUES ({values})',
        'delete': "DELETE FROM {table} WHERE {rules} RETURNING id",
        'update': "UPDATE {table} SET {key_value_pairs} WHERE {rules} RETURNING id",
        'append_array': "UPDATE {table} SET {key} = array_append({key}, {value}) WHERE id='{id}' RETURNING id",
        'get_via_id': "SELECT {fields} from {table} WHERE id='{id}'",
        'update_via_id': "UPDATE {table} SET {key_value_pairs} WHERE id='{id}' RETURNING id",
        'delete_via_id': "DELETE FROM {table} WHERE id='{id}' RETURNING id",
        'incr': "UPDATE {table} SET {key}={key}+'{num}' WHERE id='{id}' RETURNING id",
        'decr': "UPDATE {table} SET {key}={key}-'{num}' WHERE id='{id}' RETURNING id",
        'search': "SELECT {fields} FROM {table} WHERE {extra} {key} LIKE '%{value}%' LIMIT {size} OFFSET {offset}",
        'insert_or_update': "INSERT INTO {table} ({keys}) VALUES ({values}) ON DUPLICATE KEY UPDATE {key_value_pairs};"
    }

    def __init__(self, table):
        self.table = table
        self.fields = table._fields
        self.tablename = table.__name__

    def format(self, data):
        if not isinstance(data, dict):
            return utils.escape(str(data.encode('utf8')))
        if not all(f in self.fields for f in data.keys()):
            raise Exception("Unknew Fields", set(
                data.keys()) - set(self.fields))
        try:
            res = {k: utils.escape(v) for k, v in data.items()}
            return res
        except:
            raise Exception("Series Failed")

    async def nearby(self, value, column, *args, **kwargs):
        data = self.format(kwargs)
        return await query(self._sql['nearby'].format(**{
            'table': self.tablename,
            'fields': utils.concat(map(utils.wrap_key, self.fields)),
            'value': utils.escape(value),
            'column': utils.escape(column),
            'rule': utils.get_and_seg(data)
        }))

    async def get(self, oid):
        res = await query(self._sql['get_via_id'].format(**{
            'table': self.tablename,
            'fields': utils.concat(map(utils.wrap_key, self.fields)),
            'id': oid
        }))
        return res and dict(res) if res else None

    async def get_by(self, *args, **kwargs):
        data = self.format(kwargs)
        res = await query(self._sql['filter'].format(**{
            'table': self.tablename,
            'rule': utils.get_and_seg(data),
            'size': '1',
            'offset': '0',
            'fields': utils.concat(map(utils.wrap_key, self.fields)),
        }))
        return res and dict(res[0])

    async def search(self, key, value, start, limit, filters=''):
        return await query(self._sql['search'].format(**{
            'table': self.tablename,
            'fields': utils.concat(map(utils.wrap_key, self.fields)),
            'key': self.format(key),
            'value': self.format(value),
            'offset': str(int(start)),
            'size': str(int(limit)),
            'extra': filters and utils.get_pairs(filters) + 'and' or ''
        }))

    async def get_list(self, size=100, offset=0, sort_key='') -> list:
        if isinstance(sort_key, list):
            sort_key = utils.concat(map(utils.set_desc, sort_key))
        else:
            sort_key = sort_key and utils.set_desc(sort_key) or ''

        res = await query(self._sql['get_list'].format(**{
            'table': self.tablename,
            'fields': utils.concat(map(utils.wrap_key, self.fields)),
            'size': str(int(size)),
            'offset': str(int(offset)),
            'extra': sort_key and self._sql['orderby'].format(**{
                'field': sort_key
            }) or ''

        }))
        return [dict(r) for r in res]

    async def find_in(self, key, targets, fields=[]) -> dict:
        return await query(self._sql['filter_in'].format(**{
            'table': self.tablename,
            'fields': utils.concat(map(utils.wrap_key, fields or self.fields)),
            'key': key,
            'targets': utils.concat(map(utils.wrap_value, targets))
        }))

    async def find_near(self, key, start, end, fields=[], *args, **kwargs) -> dict:
        data = self.format(kwargs)
        res = await query(self._sql['find_near'].format(**{
            'table': self.tablename,
            'fields': utils.concat(map(utils.wrap_key, fields or self.fields)),
            'key': key,
            'rule': utils.get_and_seg(data),
            'start': utils.wrap_value(start),
            'end': utils.wrap_value(end)
        }))
        return [dict(r) for r in res]

    async def find_in_range(self, key, start, end, fields=[], *args, **kwargs) -> dict:
        data = self.format(kwargs)
        res = await query(self._sql['filter_in_range'].format(**{
            'table': self.tablename,
            'fields': utils.concat(map(utils.wrap_key, fields or self.fields)),
            'key': key,
            'rule': utils.get_and_seg(data),
            'start': utils.wrap_value(start),
            'end': utils.wrap_value(end)
        }))
        return [dict(r) for r in res]

    async def count(self, field):
        field = utils.escape(field) or '*'
        return await query(self._sql['count'].format(**{
            'table': self.tablename,
            'field': field
        }))

    async def count_on_rule(self, field, rule):
        rule = self.format(rule)
        field = utils.escape(field)
        return await query(self._sql['count_on_rule'].format(**{
            'table': self.tablename,
            'rule': utils.get_and_seg(rule),
            'field': field
        }))

    async def filter(self, limit=100, offset=0, sort_key='', *args, **kwargs):
        data = self.format(kwargs)
        res = await query(self._sql['filter'].format(**{
            'table': self.tablename,
            'rule': utils.get_and_seg(data),
            'size': str(int(limit)),
            'fields': utils.concat(map(utils.wrap_key, self.fields)),
            'offset': str(int(offset))
        }))
        return [dict(r) for r in res]

    async def sortby(self, sort_key='id', offset=0, limit=100, extra="", decr=False, *args, **kwargs):
        data = self.format(kwargs)
        if isinstance(sort_key, list):
            sort_key = utils.concat(map(utils.set_desc, sort_key))
        else:
            sort_key = utils.set_desc(sort_key)
        tmpl = decr and 'filter_with_orderby_decr' or 'filter_with_orderby'
        return await query(self._sql[tmpl].format(**{
            'table': self.tablename,
            'rule': utils.get_and_seg(data),
            'size': str(int(limit)),
            'sort_key': sort_key,
            'offset': str(int(offset)),
            'fields': utils.concat(map(utils.wrap_key, self.fields)),
        }))

    async def insert(self, *args, **kwargs):
        data = self.format(kwargs)
        return await insert(self._sql['insert'].format(**{
            'table': self.tablename,
            'keys': utils.concat(map(utils.wrap_key, data.keys())),
            'values': utils.concat(map(utils.wrap_value, data.values()))
        }))

    async def replace(self, *args, **kwargs):
        data = self.format(kwargs)
        return await insert(self._sql['replace'].format(**{
            'table': self.tablename,
            'keys': utils.concat(map(utils.wrap_key, data.keys())),
            'values': utils.concat(map(utils.wrap_value, data.values()))
        }))

    async def update(self, oid, *args, **kwargs):
        data = self.format(kwargs)
        pairs = utils.get_pairs(data)
        return await update(self._sql['update_via_id'].format(**{
            'id': oid,
            'table': self.tablename,
            'key_value_pairs': pairs
        }))

    async def append_array(self, oid, key, value):
        return await update(self._sql['append_array'].format(**{
            'id': oid,
            'table': self.tablename,
            'key': key,
            'value': value
        }))

    async def insert_or_update(self, *args, **kwargs) -> dict:
        data = self.format(kwargs)
        return await insert(self._sql('insert_or_update').format(**{
            'table': self.tablename,
            'keys': utils.concat(map(utils.wrap_key, data.keys())),
            'values': utils.concat(map(utils.wrap_key, data.values())),
            'key_value_pairs': utils.get_pairs(data)
        }))

    async def update_by(self, rules, *args, **kwargs):
        data = self.format(kwargs)
        rules = self.format(rules)
        return await update(self._sql['update'].format(**{
            'table': self.tablename,
            'rules': utils.get_and_seg(rules),
            'key_value_pairs': utils.get_pairs(data)
        }))

    async def delete(self, oid):
        return await update(self._sql['delete_via_id'].format(**{
            'table': self.tablename,
            'id': oid
        }))

    async def delete_by(self, *args, **kwargs):
        data = self.format(kwargs)
        return await update(self._sql['delete'].format(**{
            'table': self.tablename,
            'rules': utils.get_and_seg(data)
        }))

    async def incr(self, oid, key, num):
        return await update(self._sql['incr'].format(**{
            'id': oid,
            'table': self.tablename,
            'key': key,
            'num': num
        }))

    async def decr(self, oid, key, num):
        return await update(self._sql['decr'].format(**{
            'id': oid,
            'table': self.tablename,
            'key': key,
            'num': num
        }))


def chefWar(h,p):
	while not p <= 0 or not h <= 0:
		if p > 0 and h <= 0:
			return 1
		if p <= 0 and h > 0:
			return 0
		else:
			h -= p
			p /= 2
	return h,p

t = int(input())
for _ in range(t):
	h, p = map(int, input().split())
	print(chefWar(h,p))


import pandas as pd
import scipy.stats as stats
import operator
import numpy as np
from time import sleep as sl
import argparse
from sklearn.metrics import pairwise_distances,pairwise_distances_chunked
from sklearn.cluster import AgglomerativeClustering,DBSCAN
import time
from datetime import timedelta
import sys
from datetime import date
def parseargs():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-e','--mgt_epi_data',help = 'csv file with MGT and epidemiological information')
    parser.add_argument('-m','--mgt_level',help='the level of mgt for clustering, e.g. MGT9 ')
    parser.add_argument('-n','--number_iso_threshold',help = 'the number of isolates threshold for each type in the tested MGT level')
    parser.add_argument('-o','--outpath',help='the path of outfiles, e.g. /srv/scratch')
    parser.add_argument('-f','--mgt_flags',help='the csv file of the flag MGT-STs for meta data')
    parser.add_argument('-p','--prefix',help='the prefix for the outfile names')
    parser.add_argument('-c','--country',help='country for filtering isolates for pairwise distance analysis, e.g. -c Australia')
    parser.add_argument('-t','--transtype',help='international or national cluster, or a list of ODC10 STs in a .txt file without heading, e.g. -t international')
    parser.add_argument('-a','--whole_allele_profile',help='Allele profile of MGT9, e.g. /srv/scratch/mgt9_alleprofile.txt')
    parser.add_argument("-d", "--distances",
                        help="file containing distances corresponding to the alleleprofiles file (from previous run of this script if applicable)")
    parser.add_argument("-l", "--dist_limits",
                        help="comma separated list of cluster cutoffs or range or both i.e 1,2,5 or 1-10 or 1,2,5-10, note ODC0/MGT9 were automatically given",default="1,2,5,10")
    parser.add_argument("-j", "--no_jobs",
                        help="num jobs to split distance calc into", default=1)
    args = parser.parse_args()
    return  args
def main():
    t1 = time.time()
    args = parseargs()
    mgt_epi = pd.read_csv(args.mgt_epi_data)
    threshod_dic = {args.mgt_level: int(args.number_iso_threshold)}
    if args.mgt_flags == None:
        mgt_threshod, type_dic, isolate_dic, type_isolist_dic = transmi_tracking(threshod_dic, mgt_epi)
    if args.mgt_flags != None:
        flags = pd.read_csv(args.mgt_flags)
        mgt_threshod, type_dic, isolate_dic, type_isolist_dic = transmi_tracking_flags(threshod_dic, mgt_epi, flags)
    typedf = pd.DataFrame.from_dict(type_dic, orient='index')
    typedf.index.name = mgt_threshod
    isolate_df = pd.DataFrame.from_dict(isolate_dic, orient='index')
    isolate_df.index.name = 'Accession'
    typedflist = typedf.columns.tolist() + typedf.values.tolist()
    isolate_df.to_csv(args.outpath + '/' + args.prefix + args.mgt_level + '_isolate_transmission_link.csv')
    typedf.to_csv(args.outpath + '/' + args.prefix + args.mgt_level + '_mgttype_transmission_link.csv')

    ###### allele profile getting for pairwise calculation,
    if args.country != None:
        odc_pairwise_list = []
        if args.transtype == 'international':
            typedf2 = typedf[typedf['no_country'] >= 2] ### >=2 is international
            type_country_dic = typedf2['country_detail'].to_dict()

            for type, subdic in type_country_dic.items():
                for c in subdic.keys():
                    if args.country == c :
                        no_iso_country = subdic[c]
                        if no_iso_country>= 1 and args.mgt_level == "ODC10":  ####### to set the threshold of >= 1 for each cluster in Australia.
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
                        if args.mgt_level != "ODC10":
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
            print({"Total No. of types for pairwise distance calculation" : len(odc_pairwise_list)})
            print(odc_pairwise_list)
        if args.transtype == 'national':
            typedf2 = typedf[typedf['no_country'] == 1] ### >=2 is international; == 1 is national
            type_country_dic = typedf2['country_detail'].to_dict()

            for type, subdic in type_country_dic.items():
                for c in subdic.keys():
                    if args.country == c :
                        no_iso_country = subdic[c]
                        if no_iso_country>= 2 and args.mgt_level == "ODC10":  ####### >=2 isolates for national transmission
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
                        if args.mgt_level != "ODC10":
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
            print({"Total No. of types for pairwise distance calculation" : len(odc_pairwise_list)})
            print(odc_pairwise_list)
        if args.transtype == None:
            typedf2 = typedf[typedf['no_country'] >= 1] ### including both international and national
            type_country_dic = typedf2['country_detail'].to_dict()
            for type, subdic in type_country_dic.items():
                for c in subdic.keys():
                    if args.country == c :
                        no_iso_country = subdic[c]
                        if no_iso_country>= 2 and args.mgt_level == "ODC10":  ####### >=2 isolates for national transmission
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
                        if args.mgt_level != "ODC10":
                            if type not in odc_pairwise_list and type != "None":
                                odc_pairwise_list.append(type)
            print({"Total No. of types for pairwise distance calculation" : len(odc_pairwise_list)})
            print(odc_pairwise_list)
        if args.transtype != None and ".txt" in args.transtype:
            odc_pairwise_list=open(args.transtype,'r').read().splitlines()
        # odc_pairwise_list = ['4969']
        for type in odc_pairwise_list :
            if type in type_isolist_dic:
                print(args.mgt_level + '_' + type)
                # time_pw(args, mgt_epi, args.mgt_level, type, args.outpath)
                ### to save the type correlated acc list
                isolatelistfile = open(args.outpath + '/' + args.mgt_level + '_' + type + '_' + args.country + '_correlated_isolatelist.txt','w')
                isolatelistfile.write(args.mgt_level + '_' + type + '\n')
                for acc in type_isolist_dic[type]:
                    isolatelistfile.write(acc + '\n')
                ### to calculate the pairwise distance of alleles
                if args.whole_allele_profile != "":
                    allele_prof = open(args.whole_allele_profile, "r").read().splitlines()
                    allele_proflist = get_part_alleleprofil(allele_prof, type_isolist_dic[type])
                    allele_prof_outfile = open(args.outpath + '/' + args.mgt_level + '_' + type + '_alleleprof.txt', 'w')
                    allele_prof_outlist=[]
                    for a in allele_proflist:
                        allele_prof_outlist.append(a)
                        allele_prof_outfile.write(a + '\n')
                    profs, id_to_strain = process_profiles(allele_prof_outlist)
                    pairw_outfrefix=args.outpath + '/' + args.mgt_level + '_' + type + '_'
                    pairwise_process(args, profs, id_to_strain, pairw_outfrefix)
                    t2 = timecal(t1)
                    ##  iso_process_profiles() and  iso_pairwise_process() are for pairwise distance of isolates in Australia
                    # iso_profs, iso_id_to_strain = iso_process_profiles(allele_prof_outlist)
                    # print(iso_id_to_strain)
                    # iso_pairwise_process(args, iso_profs, iso_id_to_strain, pairw_outfrefix)
def timecal(uptime):
    timespan = time.time() - uptime
    print(timedelta(seconds=timespan))
    return time.time()
def time_metric(a, b):
    match = 0
    missmatch = 0
    a = [int(x) for x in a]
    b = [int(x) for x in b]
    d0 = date(a[0], a[1], a[2])
    d1 = date(b[0], b[1], b[2])
    dayinterv = abs((d1 - d0).days)
    return dayinterv
###note columns have to include 'Accession','Collection Year','Collection Month','Collection Day'.
def time_pw(args, metadf,odc,type,outfrefixout):
    # metadf = pd.read_csv(metapath)
    metadf[odc] = metadf[odc].astype(str)
    metadf = metadf[(metadf[odc]==type) | (metadf[odc]== str(type))]
    timedf = pd.DataFrame(metadf, columns=['Accession','Collection Year','Collection Month','Collection Day'])
    # timedf = pd.read_csv('E:/2018/2019-06-14-Australia_SEN/test/time.csv')
    # # timedf['d'] = pd.to_datetime(timedf['Date'],format = '%Y/%m/%d')
    timedf['Collection Day']=timedf['Collection Day'].replace(np.nan,15)
    timedf = timedf[timedf['Collection Month'].notnull()]
    print({'time_input_dfsize': timedf.shape[0]})
    datedf=pd.DataFrame(timedf,columns=['Collection Year','Collection Month','Collection Day'])
    acclist = timedf['Accession'].values.tolist()
    start_time = time.time()
    if datedf.shape[0]>1:
        dayinterv = pairwise_distances(datedf, metric=time_metric, n_jobs=int(args.no_jobs))
        # pairw_outfrefix = 'E:/2018/2019-06-14-Australia_SEN/test/time_pairwise_'+ type + '_'
        if  len(dayinterv) >=2 :
            print("pairwise distance time", (" --- %s seconds ---" % (time.time() - start_time)))
            np.savetxt(outfrefixout + odc +'_' +type + '_'+"time_pwdistances.txt", dayinterv.astype(int), fmt='%i', header=",".join(acclist), delimiter=",")
def unneg(a):
    if "-" in a:
        return a.split("_")[0][1:]
    else:
        return a
def mgt_dist_metric(a, b):
    match = 0
    missmatch = 0

    for i in range(len(a)):
        aAllele = a[i]
        bAllele = b[i]
        # print(aAllele,bAllele)
        # sl(0.1)
        if aAllele == 0 or bAllele == 0 or aAllele == bAllele:
            match += 1
        else:
            missmatch += 1
            # print(aAllele,bAllele)

    return missmatch
def process_profiles(inprofiles, s=False):
    profs = {}
    id_to_strain = {}

    for line in inprofiles[1:]:
        col = line.split("\t")
        if s:
            if col[0] in s:
                # print(col[0])

                if col[1] not in profs:
                    noneg = [unneg(x) for x in col[3:]]
                    profs[col[1]] = noneg
                    id_to_strain[col[1]] = [str(col[0])]
                else:
                    id_to_strain[col[1]].append(str(col[0]))
        else:
            # print(col[0])
            if col[1] not in profs:
                noneg = [unneg(x) for x in col[3:]]
                profs[col[1]] = noneg
                id_to_strain[col[1]] = [str(col[0])]
            else:
                id_to_strain[col[1]].append(str(col[0]))

    return profs, id_to_strain
def pairwise_process(args,profs,id_to_strain, pairw_outfrefix):
    idlist = list(profs.keys())
    inprofs = [profs[x] for x in idlist]

    dfo = pd.DataFrame(inprofs)
    # distances only calculated if args.distances not set
    lend = ""
    if args.distances:
        # read in distances previosly calculated
        d = np.loadtxt(args.distances)
        lend = len(d) ### number of MGT9 STs in this cluster
    else:
        start_time = time.time()
        d = pairwise_distances(inprofs, metric=mgt_dist_metric, n_jobs=int(args.no_jobs))
        lend = len(d)
        # if  len(d) >=2 :
        print("pairwise distance time", (" --- %s seconds ---" % (time.time() - start_time)))
        np.savetxt(pairw_outfrefix + "mgt9_distances.txt", d.astype(int), fmt='%i', header=",".join(idlist), delimiter=",")
    # distance cutoffs to calculate
    if lend >=2:
        pairw_outfile = open(pairw_outfrefix + 'iso_odc_recal.txt','w')
        diststring = args.dist_limits
        dists = diststring.split(",")
        distances = []
        for i in dists:
            if "-" in i:
                n = i.split("-")
                nlist = list(range(int(n[0]) + 1, int(n[1]) + 2))
                # distance cutoffs seems to be non inclusive i.e. cutoff of 3 means max distance is 2
                # therefore need to add 1 to all values
            else:
                nlist = [int(i) + 1]
            distances += nlist
        clusterlists = {}
        preference = []
        for id in idlist:
            preference.append(len(id_to_strain[id]))
        start_time = time.time()
        for dist in distances:
            clusters = AgglomerativeClustering(n_clusters=None, distance_threshold=dist, affinity="precomputed",
                                               linkage="single").fit_predict(d)
            clusterls = list(clusters)
            clusterlists[dist] = clusterls
        print("clustering time", (" --- %s seconds ---" % (time.time() - start_time)))
        realdists = ["ODC" + str(x - 1) for x in distances]

        pairw_outfile.write("Strain\tMGT9\t{}\n".format("\t".join(realdists)))
        for i in range(len(idlist)):
            id = idlist[i]
            for strain in id_to_strain[id]:
                pairw_outfile.write(strain + '\t' + str(id))
                for d in distances:
                    clust = clusterlists[d][i]
                    pairw_outfile.write("\t" + str(clust + 1))
                pairw_outfile.write("\n")
        pairw_outfile.close()
    if lend < 2:  ### belong to the same MGT9 ST
        pairw_outfile = open(pairw_outfrefix + 'iso_odc_recal.txt','w')
        pairw_outfile.write('Strain' + '\t' + 'MGT9' + '\n')
        for st, isolist in id_to_strain.items():
            for iso in isolist:
                pairw_outfile.write(str(iso) + '\t' + str(st) + '\n')
        return
        ##### pairwise distance calculation
###  iso_process_profiles() and  iso_pairwise_process() are for pairwise distance of isolates in Australia
def iso_process_profiles(inprofiles, s=False):
    profs = {}
    id_to_strain = {}
    for line in inprofiles[1:]:
        col = line.split("\t")
        if s:
            if col[0] in s:
                # print(col[0])
                if col[0] not in profs:
                    noneg = [unneg(x) for x in col[3:]]
                    profs[col[0]] = noneg
                    id_to_strain[col[0]] = [str(col[1])]
                else:
                    id_to_strain[col[0]].append(str(col[1]))
        else:
            # print(col[0])

            if col[0] not in profs:
                noneg = [unneg(x) for x in col[3:]]
                profs[col[0]] = noneg
                id_to_strain[col[0]] = [str(col[1])]
            else:
                id_to_strain[col[0]].append(str(col[1]))
    return profs, id_to_strain
def iso_pairwise_process(args,profs,id_to_strain, pairw_outfrefix):
    idlist = list(profs.keys())
    # print(idlist)
    inprofs = [profs[x] for x in idlist]
    # distances only calculated if args.distances not set
    lend = ""
    if args.distances:
        # read in distances previosly calculated
        d = np.loadtxt(args.distances)
        lend = len(d)
    else:
        start_time = time.time()
        d = pairwise_distances(inprofs, metric=mgt_dist_metric, n_jobs=int(args.no_jobs))
        lend = len(d)
        if  len(d) >=2 :
            print("pairwise distance time", (" --- %s seconds ---" % (time.time() - start_time)))
            np.savetxt(pairw_outfrefix + "iso_distances.txt", d.astype(int), fmt='%i', header=",".join(idlist), delimiter=",")
def epi_filt(mgt_epi,fi_dic):
    col_name = mgt_epi.columns.values.tolist()
    mgt_epi = mgt_epi.values.tolist()
    for key in fi_dic:
        epi_filter_out = []
        for line in mgt_epi:
            line_index = col_name.index(key)
            line_value = str(line[line_index])
            if line_value in fi_dic[key]:
                epi_filter_out.append(line)
        mgt_epi = epi_filter_out
    mgt_epi = pd.DataFrame(mgt_epi)
    mgt_epi.columns = col_name
    print(mgt_epi.shape)
    return mgt_epi
def flag_reprot(flag_input, test_file,key_list):
    flag_input = flag_input.set_index('MGT_type')
    dic_file = flag_input.to_dict()
    mgt_levels_list = test_file.columns.tolist()
    mgt_levels_list = [a for a in mgt_levels_list if "MGT" in a]
    for level in mgt_levels_list:
        test_file[level] = level + test_file[level].astype(str)
    test_list = test_file.values.tolist()
    keyflag_dic = {}
    for k1 in dic_file:
        if k1 in key_list:
            # outfile = open(outpath + '/' + k1 + '.txt','w')
            dic_file2 = {k:v for k, v in dic_file[k1].items() if "nan" not in str(v)}
            mgtst_list = []
            for line in test_list:
                for value in line:
                    if value in dic_file2.keys():
                        mgtst = value
                        mgtst_list.append(mgtst)
                        strain_name = line [0]
                        predict_types = dic_file2[value]
                        # output = "{}\t{}\t{}".format(strain_name,predict_types,mgtst)
                        # print(output)
                        # outfile.write(output + '\n')
            mgtst_ser = pd.Series(mgtst_list)
            keyflag_dic[k1] = mgtst_ser.value_counts().to_dict()
    return keyflag_dic
def transmi_tracking_flags(threshod_dic, mgt_epi,flags):
    for mgt in threshod_dic:
        gp = mgt_epi.groupby([ mgt])['Strain'].count().fillna(0)
        pass_filter_type_dic = gp [gp>= threshod_dic[mgt]].to_dict()
        if 0 in pass_filter_type_dic.keys():
            pass_filter_type_dic.pop(0)
        mgt_threshod = mgt + '_>=_' + str(threshod_dic[mgt])
        type_dic = {}
        type_isolist_dic = {}
        isolate_dic = {}
        interspread = 0
        limited_year = 0
        large_sclale = 0
        for type in pass_filter_type_dic.keys():
            type_isolist_dic[type] = []
            subdf = mgt_epi[mgt_epi[mgt]== type]
            key_list = ['Population_Structure', 'MDR', 'AR2_1', 'Top_MGT4_STs']
            keyflag_dic = flag_reprot(flags, subdf, key_list)
            country_dic = subdf.groupby(['Country'])['Strain'].count().to_dict()
            year_dic = subdf.groupby(['Collection Year'])['Strain'].count().to_dict()
            source_dic = subdf.groupby(['Source Type'])['Strain'].count().to_dict()
            type_dic[type] = keyflag_dic

            # type_dic[type]={"no_isolates":{}}
            type_dic[type]['no_isolates'] = pass_filter_type_dic[type]
            type_dic[type]['country_detail'] = country_dic
            if 'None' in country_dic.keys():
                type_dic[type]['no_country'] = len(country_dic) - 1
            else:
                type_dic[type]['no_country'] = len(country_dic)

            type_dic[type]['year_detail'] = year_dic
            if 'None' in year_dic.keys():
                type_dic[type]['no_year'] = len(year_dic) - 1
            else:
                type_dic[type]['no_year'] = len(year_dic)
            if len(year_dic) <= 2 and  len(year_dic)> 0:
                limited_year =limited_year+1
            if len(country_dic) >= 2 :
                interspread = interspread + 1
            if len(year_dic) > 1 and len(year_dic) > 0 and len(country_dic) >= 2 and pass_filter_type_dic[type] > 50:
                large_sclale = large_sclale + 1

            type_dic[type]['source'] = source_dic
            if 'None' in source_dic.keys():
                type_dic[type]['no_source'] = len(source_dic) - 1
            else:
                type_dic[type]['no_source'] = len(source_dic)
            ########### to product isolate_dic
            acclist = subdf['Accession'].tolist()
            for acc in acclist:
                type_isolist_dic[type].append(acc)
                isolate_dic[acc] = {}
                isolate_dic[acc] = type_dic[type]
                isolate_dic[acc][mgt_threshod] = type

        print("No. of passed types: " + str(len(pass_filter_type_dic)))
        print('No. of potential international spreading clusters: ' + str(interspread))
        print('No. of potential international spreading clusters within years: ' + str(limited_year))
        print('No. of potential international spreading large clusters within years >50: ' + str(large_sclale))

        return mgt_threshod,type_dic, isolate_dic, type_isolist_dic
def transmi_tracking(threshod_dic, mgt_epi):
    for mgt in threshod_dic:
        gp = mgt_epi.groupby([mgt])['Strain'].count().fillna(0)
        pass_filter_type_dic = gp [gp>= threshod_dic[mgt]].to_dict()
        if 0 in pass_filter_type_dic.keys():
            pass_filter_type_dic.pop(0)
        mgt_threshod = mgt + '_>=_' + str(threshod_dic[mgt])
        type_dic = {}
        isolate_dic = {}
        type_isolist_dic ={}
        interspread = 0
        limited_year = 0
        large_sclale = 0
        for type in pass_filter_type_dic.keys():
            type_isolist_dic[type]=[]
            subdf = mgt_epi[mgt_epi[mgt]== type]
            key_list = ['Population_Structure', 'MDR', 'AR2_1', 'Top_MGT4_STs']
            # keyflag_dic = flag_reprot(flags, subdf, key_list)
            country_dic = subdf.groupby(['Country'])['Strain'].count().to_dict()
            year_dic = subdf.groupby(['Collection Year'])['Strain'].count().to_dict()
            source_dic = subdf.groupby(['Source Type'])['Strain'].count().to_dict()
            type_dic[type] = {}
            type_dic[type]['no_isolates'] = pass_filter_type_dic[type]
            type_dic[type]['country_detail'] = country_dic
            if 'None' in country_dic.keys():
                type_dic[type]['no_country'] = len(country_dic) - 1
            else:
                type_dic[type]['no_country'] = len(country_dic)

            type_dic[type]['year_detail'] = year_dic
            if 'None' in year_dic.keys():
                type_dic[type]['no_year'] = len(year_dic) - 1
            else:
                type_dic[type]['no_year'] = len(year_dic)
            if len(year_dic) <= 2 and  len(year_dic)> 0:
                limited_year =limited_year+1
            if len(country_dic) >= 2 :
                interspread = interspread + 1
            if len(year_dic) > 1 and len(year_dic) > 0 and len(country_dic) >= 2 and pass_filter_type_dic[type] > 50:
                large_sclale = large_sclale + 1
            type_dic[type]['source'] = source_dic
            if 'None' in source_dic.keys():
                type_dic[type]['no_source'] = len(source_dic) - 1
            else:
                type_dic[type]['no_source'] = len(source_dic)

            ########### to product isolate_dic
            acclist = subdf['Accession'].tolist()
            for acc in acclist:
                type_isolist_dic[type].append(acc)
                isolate_dic[acc] = {}
                isolate_dic[acc] = type_dic[type]
                isolate_dic[acc][mgt_threshod] = type
        print("No. of passed types: " + str(len(pass_filter_type_dic)))
        print('No. of potential international spreading clusters: ' + str(interspread))
        print('No. of potential international spreading clusters within years: ' + str(limited_year))
        print('No. of potential international spreading large clusters within years >50: '+ str(large_sclale))
        return mgt_threshod,type_dic, isolate_dic, type_isolist_dic
def get_part_alleleprofil(whol_alleprof, isolist):
    outlist = []
    outlist.append(whol_alleprof[0])
    for line in whol_alleprof:
        col = line.split('\t')
        for acc in isolist:
            if acc == col[0]:           # or acc + '_cladeC.fasta' == col[0]:
                outlist.append(line)
    return outlist
if __name__ == "__main__":
    main()


# Take a sample of ten phishing emails (or any text files) and find the most common words in them.

# Using Shakespeare's Hamlet as a sample, find the most common words in the sample.

text=""
for i in range(10):
    with open("./Files/Hamlet/"+str(i)) as f:
        text += f.read()

count = {}
for word in text.split():
    if word in count:
        count[word] += 1
    else:
        count[word] = 1

for word in sorted(count, key=count.get):
    print(word, count[word])


import os
import glob
import pickle
import logging
import argparse
from multiprocessing import Pool

import numpy as np
import pandas as pd

from core.utils import timer, do_job

# PATH
DATA_PATH = os.getenv("DATA_PATH")
PREPROCESSED_DATA_PATH = os.getenv("PREPROCESSED_DATA_PATH")
TXT_DATA_NAME = os.getenv("TXT_DATA_NAME")
print(TXT_DATA_NAME)
DW2V_PATH = os.getenv("DW2V_PATH")
PARAM_PATH = os.getenv("PARAM_PATH")

# Logger
LOGGER = logging.getLogger('JobLogging')
LOGGER.setLevel(10)
fh = logging.FileHandler('job.log')
LOGGER.addHandler(fh)
formatter = logging.Formatter('%(asctime)s:%(lineno)d:%(levelname)s:%(message)s')
fh.setFormatter(formatter)
LOGGER.info("job start")

parser = argparse.ArgumentParser(description='train Dynamic Word Embeddings')
parser.add_argument('--without_preprocess', type=int, default=0, metavar='N',
                    help='if preprocessor is not neccessary, set 1')
parser.add_argument('--n_job', type=str, default="10", metavar='N',
                    help='number of cpu for multiprocessing')
parser.add_argument('--word_freq_min', type=str, default="5", metavar='N',
                    help='minmiun freqency for target word')
args = parser.parse_args()

os.environ["N_JOB"] = args.n_job
os.environ["WORD_FREQ_MIN"] = args.word_freq_min
N_JOB = int(os.getenv("N_JOB"))

if __name__ =="__main__":
    if args.without_preprocess == 0:
        # 前処理
        with do_job("preprocess tweet", LOGGER):
            from core.preprocess_tweet import preprocess_one_day_tweet

            TWEETS_PATHS = glob.glob(DATA_PATH+"alldata_20*")

            if not os.path.exists(PREPROCESSED_DATA_PATH+TXT_DATA_NAME):
                os.mkdir(PREPROCESSED_DATA_PATH+TXT_DATA_NAME)

            with Pool(processes=N_JOB) as p:
                p.map(preprocess_one_day_tweet, TWEETS_PATHS)

    # 単語の共起を確認
    with do_job("make co occ dict", LOGGER):
        from core.make_DW2V import make_unique_word2idx, make_whole_day_co_occ_dict

        TWEETS_PATHS = glob.glob(PREPROCESSED_DATA_PATH+TXT_DATA_NAME+"/*")

        # 全単語のチェック
        make_unique_word2idx(TWEETS_PATHS)

        if not os.path.exists(PREPROCESSED_DATA_PATH+"co_occ_dict_word_count/"):
            os.mkdir(PREPROCESSED_DATA_PATH+"co_occ_dict_word_count/")

        TWEETS_PATHS = glob.glob(PREPROCESSED_DATA_PATH+TXT_DATA_NAME+"/*")
        make_whole_day_co_occ_dict(TWEETS_PATHS)

    # PPMIの計算
    with do_job("make PPMI", LOGGER):
        from core.make_DW2V import make_whole_day_ppmi_list

        TWEETS_PATHS = sorted(glob.glob(PREPROCESSED_DATA_PATH+TXT_DATA_NAME+"/*"))
        DICTS_PATHS = sorted(glob.glob(PREPROCESSED_DATA_PATH+"co_occ_dict_word_count/*"))
        PATH_TUPLES = [(tweet_p, dict_p) for tweet_p, dict_p in zip(TWEETS_PATHS, DICTS_PATHS)]

        make_whole_day_ppmi_list(PATH_TUPLES)

    # DW2Vの計算
    with do_job("make DW2V", LOGGER):
        from core.make_DW2V import make_DW2V

        make_DW2V(PARAM_PATH+"params_0803.json")


from tokenize import TokenInfo, DEDENT
from bones.bones_tree import BonesNode

from bones.token_parser import parse
from bones.tests.conftest import tokens_from_string
from bones.suppressors.known_mutants import FUNCTION

def test_module_tokens_are_put_in_root_node():

    given = tokens_from_string('''\
from somewhere import rainbow
friends = ['dog']

def a_journey(friends):
    import yellow.brick

    return 'home'

destination = a_journey(friends)

''')

    expected_tokens = _generate_tokens_from_string('''\
from somewhere import rainbow
friends = ['dog']

## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
destination = a_journey(friends)

''')

    module = parse(given)

    assert module.tokens == expected_tokens

def test_function_tokens_are_put_in_function_blocks():
    # given
    given = tokens_from_string('''\
from newer import better
old = 'bad'

def new_test(thing):
    from special_date_module import is_new
    return is_new(thing)

class SneakyClass():
    """just making sure nothing here show up where it shouldn't"""
    pass

def old_test(thing):
    from special_date_module import is_old
    return is_old(thing)

good = new_test(thing) and not old_test(thing)
''')

    expected_function_1_tokens =  _generate_node_tokens_from_string('''\
## delete from test ##
## delete from test ##
## delete from test ##
def new_test(thing):
    from special_date_module import is_new
    return is_new(thing)

''')
    func_block_1 = _build_func_block(expected_function_1_tokens)

    expected_function_2_tokens =  _generate_node_tokens_from_string('''\
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
## delete from test ##
def old_test(thing):
    from special_date_module import is_old
    return is_old(thing)

''')
    func_block_2 = _build_func_block(expected_function_2_tokens)

    # when
    module = parse(given)

    # then
    assert module.children[0].tokens[:-1] == func_block_1.tokens[:-1]
    # The dedent token (the last token) will look different because if way the test is generated
    assert module.children[0].tokens[-1] == TokenInfo(type=DEDENT, string='', start=(8, 0), end=(8, 0), line='class SneakyClass():\n')

    assert module.children[2].tokens[:-1] == func_block_2.tokens[:-1]
    # The dedent token (the last token) will look different because if way the test is generated
    assert module.children[2].tokens[-1] == TokenInfo(type=DEDENT, string='', start=(16, 0), end=(16, 0), line='good = new_test(thing) and not old_test(thing)\n')

def test_class_tokens_are_put_in_class_block():
    given = tokens_from_string('''\
def a_func(thing):
    pass

class ImportantClass():
    """I'm parsed correctly"""

    def __init__(self):
        pass

def another_func():
    pass
''')

    expected_tokens = _generate_node_tokens_from_string('''\
## delete from test ##
## delete from test ##
## delete from test ##
class ImportantClass():
    """I'm parsed correctly"""

## delete from test ##
## delete from test ##
''')

    module = parse(given)

    assert module.children[1].tokens[:-1] == expected_tokens[:-1]
    # The dedent token (the last token) will look different because if way the test is generated
    assert module.children[1].tokens[-1] == TokenInfo(type=DEDENT, string='', start=(10, 0), end=(10, 0), line='def another_func():\n')

def _generate_tokens_from_string(s):
    toks = list(tokens_from_string(s))
    return _remove_placeholder_tokens(toks)

def _remove_placeholder_tokens(toks):
    # In order to generate expected_tokens that will match the parser's behavior
    # some empty lines must be used to make tok.start and tok.end values the same.
    reduced_toks = []
    for tok in toks:
        if tok.line != '## delete from test ##\n':
            reduced_toks.append(tok)

    return reduced_toks

def _build_func_block(expected_tokens):
    func_block = BonesNode(block_type=FUNCTION, parent=None)
    func_block.tokens = expected_tokens
    return func_block

def _generate_node_tokens_from_string(s):
    # A funcion or class node in the bones tree will not have the
    #   - the module's final ENDMARKER token (that token belongs in the root node)
    # so, remove it here too.
    return _generate_tokens_from_string(s)[:-1]


# -*- coding: utf-8 -*-
import psycopg2
# the module that connects to the database

"""

The task is to create a reporting tool that prints out reports (in plain text)
based on the data in the database.
1.What are the most popular three articles of all time?
Which articles have been accessed the most? Present this information as a sorted list
with the most popular article at the top.
2.Who are the most popular article authors of all time?
That is, when you sum up all of the articles each author has written,
which authors get the most page views? Present this as a sorted list with the most
popular author at the top.
3.On which days did more than 1% of requests lead to errors?
The log table includes a column status that indicates the HTTP status code that
the news site sent to the user's browser. (Refer back to this lesson
if you want to review the idea of HTTP status codes.)

"""
DBNAME = "news"

# Open and connect to database; Run the query; Return database cursor objects
def query(user_query):
	DB = psycopg2.connect(database = DBNAME)
	cursor = DB.cursor()
	cursor.execute(user_query)
	result = cursor.fetchall()
	DB.close()
	return result

# 1. popular article
def pop_article():
	top_article = query("select title, count(*) from articles "
              "join log on path like CONCAT('%',slug) group by title "
              "order by count(*) desc limit 3")
	print("The most popular three articles are:")
	for title, views in top_article:
		print(" \"{}\" -- {} views".format(title, views))
# 2. popular author
def pop_author():
	top_authors = query("select name, count(path) from authors "
            "join articles on authors.id = author join log "
            "on path like CONCAT('%', slug) group by name order by count(path) desc limit 4")
	print('The most popular authors are:')
	for name, views in top_authors:
		print(" {} -- {} views".format(name, views))

# 3. error
def error_day():
    errorday = query("select date, avg from ("
            "select date, (sum(error) / (select count(*) "
            "from log where (time::date) = date)) as avg "
            "from (select (time::date) as date, count(*) as error "
            "from log where status like '4%' group by date) "
            "as error_percentage group by date order by avg  desc) as final "
            "where avg >= .01")
    print('Days with more than 1%  of requests lead to errors')
    for res in errorday:
        print (str(res[0]) + " — " + str(round((res[1]*100), 2)) +
               '%')

if __name__ == '__main__':
	pop_article()
	pop_author()
	error_day()


#!/usr/bin/python3

# USE THIS WHEN IN NOTEBOOK -> %python

# CHANGE ACCORDINGLY: the field XXX

import sys
import time

from azure.identity import ClientSecretCredential
from azure.storage.filedatalake import DataLakeServiceClient,FileSystemClient

ACCOUNT_NAME = "XXX"
FILE_SYSTEM = "XXX"
TARGET_DIR = "XXX"

def set_permission(path,acl):
    # Directories and files need to be handled differently
    if path.is_directory:
        directory_client = filesystem.get_directory_client(directory=path.name)
        resp = directory_client.set_access_control(acl=acl)
        print(f'\tApplied Directory ACL to {path.name}')
    else:
        file_client = filesystem.get_file_client(path.name)
        # Need to remove "Default" ACL segments from ACL string because that can't be applied to files
        resp = file_client.set_access_control(acl=acl[:acl.find('default')-1])
        print(f'\tApplied File ACL to {path.name}')
    return resp

def main(target_dir,filesystem):

    # Get the target directory, subdirectories and permissions
    paths = filesystem.get_paths(path=target_dir)
    directory_client = filesystem.get_directory_client(directory=target_dir)
    acl = directory_client.get_access_control()
    target_acl_dir = acl['acl']

    for path in paths:
      set_permission(path,target_acl_dir)

if __name__ == '__main__':

    # Clients
    credential = "XXX" # the master account key.
    service = DataLakeServiceClient(account_url=f'https://{ACCOUNT_NAME}.dfs.core.windows.net/', credential=credential)
    filesystem = service.get_file_system_client(file_system=FILE_SYSTEM)

    print('*'*20)
    print(f'Storage Account Name: {ACCOUNT_NAME}')
    print(f'File System Name: {FILE_SYSTEM}')
    print('*'*20)
    print(f'Running: Setting ACLs for all child paths (subdirectories and files) in TARGET_DIR to match parent.')
    total_start = time.time() # Start Timing
    main(TARGET_DIR,filesystem)
    total_end = time.time() # End Timing
    print("Complete: Recursive ACL configuration took {} seconds.".format(str(round(total_end - total_start,2))))


def main():

    # Getting input from the user. (1-8 only)
    height = get_height_int()

    # Pyramid
    for row in range(height):
        # Left section of the pyramid.
        for col in range(height - row):
            print(" ", end="")
        for hash in range(height - col):
            print("#", end="")

        # 2 spaces (dots) in middle of 2 pyramids.
        for space in range(2):
            print(" ", end="")

        # Right section of the pyramid.
        for hash in range(height - col):
            print("#", end="")

        print("")

def get_height_int():
    while True:
        height = int(input("Height: "))

        if height in range(1, 9):
            return height

main()


#!/usr/bin/env python
#encoding=utf8

from json import dumps

def get_node(tree, name):
    if tree.label == name:
        return True, [tree.label]
    if not tree.children:
        return False, None
    for child in tree.children:
        found, addr = get_node(child, name)
        if found:
            return True, [tree.label] + addr
    return False, None

def goto_node(tree, desc):
    assert tree.label == desc[0]
    node = tree
    for name in desc[1:]:
        nodes = [n for n in node.children if n.label == name]
        if not nodes:
            return False, None
        node = nodes[0]
    return True, node

def recreate_node(orig, desc):
    """Recreate item in orig under desc."""
    tree = Tree(desc[-1], [])
    success, node = goto_node(orig, desc)
    if not node.children:
        return tree

    for child in node.children:
        success, _ = goto_node(orig, desc + [child.label])
        if not success:
            child_node = Tree(child.label, [])
        else:
            child_node = recreate_node(orig, desc + [child.label])
        tree.children.append(child_node)
    return tree

class Tree(object):
    def __init__(self, label, children=[]):
        self.label = label
        self.children = children

    def __dict__(self):
        return {self.label: [c.__dict__() for c in sorted(self.children)]}

    def __str__(self, indent=None):
        return dumps(self.__dict__(), indent=indent)

    def __lt__(self, other):
        return self.label < other.label

    def __eq__(self, other):
        return self.__dict__() == other.__dict__()

    def from_pov(self, from_node):
        found, desc = get_node(self, from_node)
        if not found:
            raise ValueError("Node {} not found.".format(from_node))

        last_label = desc[-1]
        node = recreate_node(self, desc)
        last_node = node
        reverse_desc = [last_label]
        for name in reversed(desc[:-1]):
            desc_ = get_node(self, name)[1]
            parent = recreate_node(self, desc_)

            last_node.children.append(parent)
            parent.children = [
                child for child in parent.children
                if child.label != last_label
            ]
            last_label = desc_[-1]
            last_node = parent
        return node

    def path_to(self, from_node, to_node):
        tree = self.from_pov(from_node)
        found, desc = get_node(tree, to_node)
        if not found:
            raise ValueError("Dest node {} not found.".format(to_node))
        return desc


import pandas as pd
import argparse
from gtfparse import read_gtf

parser = argparse.ArgumentParser()
parser.add_argument('--phenotype', type=str, required=True)
# parser.add_argument('--ncRNA', type=str, required=True)

if __name__ == '__main__':
    args = parser.parse_args()
    phenotype = args.phenotype

    gtf = read_gtf('ReferenceGenome/Annotations/gencode.v34.chromasomal.annotation.gtf')

    ncRNA_genes = gtf.loc[(gtf.gene_type.isin(['snoRNA', 'snRNA',
                                               'lncRNA', 'unprocessed_pseudogene',
                                               'transcribed_unprocessed_pseudogene',
                                               'pseudogene', 'rRNA_pseudogene',
                                               'transcribed_processed_pseudogene',
                                               'transcribed_unitary_pseudogene',
                                               'transcribed_unprocessed_pseudogene',
                                               'translated_processed_pseudogene',
                                               'translated_unprocessed_pseudogene',
                                               'unprocessed_pseudogene'
                                              ])) & (gtf.feature == 'gene')].gene_id

    counts = pd.read_csv('featureCounts/{phenotype}/Counts.txt'.format(phenotype=phenotype),
                                 sep='\t', skiprows=1, index_col=0)

    ncRNA_counts = counts.loc[ncRNA_genes]

    ncRNA_counts.to_csv('featureCounts/{phenotype}_annotated_ncRNA/Counts.txt'.format(phenotype=phenotype),
                        sep='\t', index=True, header=True)


import dlib
from imutils import face_utils
dlib_path = "dlibb/shape_predictor_68_face_landmarks.dat"
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(dlib_path)
import argparse
import pickle
import cv2
import os
import mpmath
import numpy as np

# face_classifier =  cv2.CascadeClassifier('harcascades/haarcascade_frontalface_default.xml')
src_path = ("O:\\Nama_College\\FYP\\MY_FYP_CODE\\MY_FYP_CODE\\MY_CODE\\TESTING_DATASET\\")
predict = []
features_vector = []
pickle_in = open("O:\\Nama_College\\FYP\\MY_FYP_CODE\\MY_FYP_CODE\\MY_CODE\\dlib_normalized.pickle","rb")
# pickle_in = open("O:\\Nama_College\\FYP\\MY_FYP_CODE\\MY_FYP_CODE\\MY_CODE\\dlib_normalized_full.pickle","rb")
model = pickle.load(pickle_in)
cap = cv2.VideoCapture(0)
B= 0
while (True):
    ret, frame = cap.read()
    gray = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
    B += 1
    if B % 5 == 0:
        print(B)
        face = detector(gray,0)
        for (J, rect) in enumerate(face):
            shap = predictor(gray, rect)
            xlist = []
            ylist = []
            shap = face_utils.shape_to_np(shap)
            Centre = (shap[30])
            centre_x = Centre[0]
            centre_y = Centre[1]
            shap =  shap[18:68]
            for i in shap:
                xlist.append(i[0])
                ylist.append(i[1])
            forx = []
            fory = []
            for x in xlist:
                forx.append((x - centre_x) ** 2)
            for y in ylist:
                fory.append((y - centre_y) ** 2)
            listsum = [sum(x) for x in zip(forx, fory)]
            features = []
            for i in listsum:
                k = mpmath.sqrt(float(i))
                features.append(float(k))
            maxx = (max(features))
            final = []
            for i in features:
                if (i == 0.0):
                    continue
                F = i / maxx
                final.append(F)
            # print(final)
            numpy_array = np.array(final)
            prediction = model.predict([numpy_array])[0]

            # predict.append(prediction)
            (x, y, w, h) = face_utils.rect_to_bb(rect)
            cv2.rectangle(frame, (x, y), (x + w, y + h),(0, 255, 0), 2)
        # display the image and the prediction
        #     cv2.putText(frame, "FACE ({})".format(J+ 1) + " " + prediction, (x , y ), cv2.FONT_HERSHEY_COMPLEX, 0.5,
        #             (0, 255, 0), 2)
            cv2.putText(frame, prediction, (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5,
                        (0, 255, 0), 2)
        # cv2.putText(tasveer,  prediction, (x-5 , y-5 ), cv2.FONT_HERSHEY_COMPLEX, 1.2,
        #             (0, 0, 255),4)
            print(prediction)
            cv2.circle(frame, (centre_x, centre_y), 1, (0, 0, 0), 5)
            for (x, y) in shap:
                cv2.circle(frame, (x, y), 1, (0, 0, 255), 2)

                cv2.imshow("Image", frame)
                cv2.waitKey(1)
                if k == 'q':
                    break
cap.release()
cv2.destroyAllWindows()


candies, multiple = [int(x) for x in input().split()]
primes = [True for i in range(candies + 1)]
primes[0] = False
primes[1] = False
combos = 0
for p in range(2, candies + 1):
    if primes[p]:
        combos += (candies - p) // multiple + 1
        combos += (candies - p - 1) // multiple + 1

        for i in range(p * 2, candies + 1, p):
            primes[i] = False
# print(primes)
print(combos)


#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Custom filters for use in openshift_aws
'''

from ansible import errors

class FilterModule(object):
    ''' Custom ansible filters for use by openshift_aws role'''

    @staticmethod
    def scale_groups_serial(scale_group_info, upgrade=False):
        ''' This function will determine what the deployment serial should be and return it

          Search through the tags and find the deployment_serial tag. Once found,
          determine if an increment is needed during an upgrade.
          if upgrade is true then increment the serial and return it
          else return the serial
        '''
        if scale_group_info == []:
            return 1

        scale_group_info = scale_group_info[0]

        if not isinstance(scale_group_info, dict):
            raise errors.AnsibleFilterError("|filter plugin failed: Expected scale_group_info to be a dict")

        serial = None

        for tag in scale_group_info['tags']:
            if tag['key'] == 'deployment_serial':
                serial = int(tag['value'])
                if upgrade:
                    serial += 1
                break
        else:
            raise errors.AnsibleFilterError("|filter plugin failed: deployment_serial tag was not found")

        return serial

    @staticmethod
    def scale_groups_match_capacity(scale_group_info):
        ''' This function will verify that the scale group instance count matches
            the scale group desired capacity

        '''
        for scale_group in scale_group_info:
            if scale_group['desired_capacity'] != len(scale_group['instances']):
                return False

        return True

    @staticmethod
    def build_instance_tags(clusterid):
        ''' This function will return a dictionary of the instance tags.

            The main desire to have this inside of a filter_plugin is that we
            need to build the following key.

            {"kubernetes.io/cluster/{{ openshift_aws_clusterid }}": "{{ openshift_aws_clusterid}}"}

        '''
        tags = {'clusterid': clusterid,
                'kubernetes.io/cluster/{}'.format(clusterid): clusterid}

        return tags

    def filters(self):
        ''' returns a mapping of filters to methods '''
        return {'build_instance_tags': self.build_instance_tags,
                'scale_groups_match_capacity': self.scale_groups_match_capacity,
                'scale_groups_serial': self.scale_groups_serial}


PHANTOM_SYS_INFO_URL = "{url}rest/system_info"
PHANTOM_ASSET_INFO_URL = "{url}rest/asset/{asset_id}"

URL_GET_CODE = 'https://login.salesforce.com/services/oauth2/authorize'
URL_GET_TOKEN = 'https://login.salesforce.com/services/oauth2/token'

URL_GET_CODE_TEST = 'https://test.salesforce.com/services/oauth2/authorize'
URL_GET_TOKEN_TEST = 'https://test.salesforce.com/services/oauth2/token'

API_ENDPOINT_DESCRIBE_GLOBAL = '{version}/sobjects/'
API_ENDPOINT_GET_UPDATED = '{version}/sobjects/{sobject}/updated/'
API_ENDPOINT_OBJECT_ID = '{version}/sobjects/{sobject}/{id}/'
API_ENDPOINT_RUN_QUERY = '{version}/{query_type}/'
API_ENDPOINT_OBJECT = '{version}/sobjects/{sobject}/'
API_ENDPOINT_GET_LISTVIEWS = '{version}/sobjects/{sobject}/listviews/'
API_ENDPOINT_GET_LISTVIEW_LOCATOR = '{version}/sobjects/{sobject}/listviews/{locator}/'
API_ENDPOINT_BATCH_REQUEST = '{version}/composite/batch/'
API_ENDPOINT_GET_LISTVIEWS_FROM_OBJECT = '{version}/ui-api/list-records/{sobject}/{view_name}'

CASE_FIELD_MAP = {
    'parent_case_id': 'ParentId',
    'subject': 'Subject',
    'priority': 'Priority',
    'description': 'Description',
    'status': 'Status',
    'closed': 'IsClosed',
    'escalated': 'IsEscalated'
}
SALESFORCE_INVALID_INTEGER = 'Please provide non-zero positive integer in "{parameter}"'
SALESFORCE_UNKNOWN_ERR_MSG = "Unknown error occurred. Please check the asset configuration and|or action parameters."
SALESFORCE_ERR_CODE_UNAVAILABLE = "Error code unavailable"
SALESFORCE_DEFAULT_TIMEOUT = 30


import re
from datetime import date
from typing import Optional

import docx  # type: ignore

from adaptive_hockey_federation.parser.user_card import BaseUserInfo

NAME = '[И|и][М|м][Я|я]'
SURNAME = '[Ф|ф][А|а][М|м][И|и][Л|л][И|и][Я|я]'
PATRONYMIC = '[О|о][Т|т]?[Ч|ч][Е|е][С|с][Т|т][В|в][О|о]'
DATE_OF_BIRTH = '[Д|д][А|а][Т|т][А|а] [Р|р][О|о].+'
TEAM = '[К|к][О|о][М|м][А|а][Н|н][Д|д][А|а]'
PLAYER_NUMBER = '[И|и][Г|г][Р|р][О|о][В|в][О|о][Й|й]'
POSITION = '[П|п][О|о][З|з][И|и][Ц|ц][И|и][Я|я]'
NUMERIC_STATUS = '[Ч|ч].+[С|с][Т|т].+'
PLAYER_CLASS = '[К|к][Л|л][А|а][С|с][С|с]'

def read_file_columns(file: docx) -> list[docx]:
    """Функция находит таблицы в файле и возвращает список объектов
    docx с данными каждого столбца.
    """
    return [
        column
        for table in file.tables
        for index, column in enumerate(table.columns)
    ]

def read_file_text(file: docx) -> list[str]:
    """Функция находит текстовые данные в файле и возвращает список объектов
    docx с найденными данными.
    """
    return [
        run.text
        for paragraph in file.paragraphs
        for run in paragraph.runs
    ]

def get_counter_for_columns_parser(
        columns: list[docx]
) -> int:
    count = 0
    for column in columns:
        for index, cell in enumerate(column.cells):
            if re.search(r'п/п', cell.text):
                for cell in column.cells[index + 1:]:
                    if cell.text and len(cell.text) < 4:
                        count += 1
                    else:
                        break
            else:
                if count > 0:
                    break
    return count

def columns_parser(
        columns: list[docx],
        regular_expression: str,
) -> list[Optional[str]]:
    """Функция находит столбец по названию и списком выводит содержимое
    каждой ячейки этого столбца.
    """
    output = [
        text if text
        else None
        for column in columns
        if re.search(
            regular_expression,
            list(cell.text for cell in column.cells)[0]
        )
        for text in list(cell.text for cell in column.cells)[1:]
    ]
    if not output:
        count = get_counter_for_columns_parser(columns)
        for column in columns:
            for index, cell in enumerate(column.cells):
                if re.search(regular_expression, cell.text):
                    for cell in column.cells[index + 1:index + 1 + count]:
                        output.append(cell.text)

    return output

def find_names(columns: list[docx], regular_expression: str) -> list[str]:
    """Функция парсит в искомом столбце имена. Опирается на шаблон ФИО
    (имя идет после фамилии на втором месте).
    """
    names_list = columns_parser(columns, regular_expression)
    return [
        name.split()[1].rstrip()
        for name in names_list
        if name
    ]

def find_surnames(columns: list[docx], regular_expression: str) -> list[str]:
    """Функция парсит в искомом столбце фамилии. Опирается на шаблон ФИО
    (фамилия идет на первом месте).
    """
    surnames_list = columns_parser(columns, regular_expression)
    return [
        surname.split()[0].rstrip()
        for surname in surnames_list
        if surname
    ]

def find_patronymics(
        columns: list[docx],
        regular_expression: str,
) -> list[str]:
    """Функция парсит в искомом столбце отчества. Опирается на шаблон ФИО
    (отчество идет на последнем месте).
    """
    patronymics_list = columns_parser(columns, regular_expression)
    return [
        patronymic.replace('/', ' ').split()[2].rstrip().rstrip(',')
        if patronymic and len(patronymic.split()) > 2
        else 'Отчество отсутствует'
        for patronymic in patronymics_list
    ]

def find_dates_of_birth(
        columns: list[docx],
        regular_expression: str,
) -> list[date]:
    """Функция парсит в искомом столбце дату рождения
    и опирается на шаблон дд.мм.гггг.
    """
    dates_of_birth_list = columns_parser(columns, regular_expression)
    dates_of_birth_list_clear = []
    for date_of_birth in dates_of_birth_list:
        if date_of_birth:
            try:
                for day, month, year in [
                    re.sub(r'\D', ' ', date_of_birth).split()
                ]:
                    if len(year) == 2:
                        if int(year) > 23:
                            year = '19' + year
                        else:
                            year = '20' + year
                    dates_of_birth_list_clear.append(
                        date(int(year), int(month), int(day))
                    )
            except ValueError or IndexError:  # type: ignore
                dates_of_birth_list_clear.append(date(1900, 1, 1))
        else:
            dates_of_birth_list_clear.append(date(1900, 1, 1))

    return dates_of_birth_list_clear

def find_team(
        text: list[str],
        columns: list[docx],
        regular_expression: str,
) -> str:
    """Функция парсит название команды.
    """
    text_clear = ' '.join(text)
    text_clear = re.sub(
        r'\W+|_+|ХК|СХК|ДЮСХК|Хоккейный клуб|по незрячему хоккею'
        '|по специальному хоккею|Спец хоккей|по специальному|по следж-хоккею',
        ' ',
        text_clear
    ).split()  # type: ignore
    try:
        return [
            'Молния Прикамья'
            if text_clear[index + 2] == 'Прикамья'
            else 'Ак Барс'
            if text_clear[index + 1] == 'Ак'
            else 'Снежные Барсы'
            if text_clear[index + 1] == 'Снежные'
            else 'Хоккей Для Детей'
            if text_clear[index + 1] == 'Хоккей'
            else 'Дети-Икс'
            if text_clear[index + 1] == 'Дети'
            else 'СКА-Стрела'
            if text_clear[index + 1] == 'СКА'
            else 'Сборная Новосибирской области'
            if text_clear[index + 2] == 'Новосибирской'
            else 'Атал'
            if text_clear[index + 3] == 'Атал'
            else 'Крылья Мечты'
            if text_clear[index + 2] == 'мечты'
            else 'Огни Магнитки'
            if text_clear[index + 1] == 'Огни'
            else 'Энергия Жизни Краснодар'
            if text_clear[index + 3] == 'Краснодар'
            else 'Энергия Жизни Сочи'
            if text_clear[index + 4] == 'Сочи'
            else 'Динамо-Москва'
            if text_clear[index + 1] == 'Динамо'
            else 'Крылья Советов'
            if text_clear[index + 2] == 'Советов'
            else 'Красная Ракета'
            if text_clear[index + 2] == 'Ракета'
            else 'Красная Молния'
            if text_clear[index + 2] == 'молния'
            else 'Сахалинские Львята'
            if text_clear[index + 1] == 'Сахалинские'
            else 'Мамонтята Югры'
            if text_clear[index + 1] == 'Мамонтята'
            else 'Уральские Волки'
            if text_clear[index + 1] == 'Уральские'
            else 'Нет названия команды'
            if text_clear[index + 1] == 'Всего'
            else text_clear[index + 1].capitalize()
            for index, txt in enumerate(text_clear)
            if re.search(regular_expression, txt)
        ][0]
    except IndexError:
        for column in columns:
            for cell in column.cells:
                if re.search(regular_expression, cell.text):
                    txt = re.sub(r'\W', ' ', cell.text)
                    return txt.split()[1].capitalize()

        return 'Название команды не найдено'

def find_players_number(
        columns: list[docx],
        regular_expression: str,
) -> list[int]:
    """Функция парсит в искомом столбце номер игрока.
    """
    players_number_list = columns_parser(columns, regular_expression)
    players_number_list_clear = []
    for player_number in players_number_list:
        if player_number:
            try:
                players_number_list_clear.append(
                    int(re.sub(r'\D', '', player_number)[:2])
                )
            except ValueError:
                players_number_list_clear.append(0)
        else:
            players_number_list_clear.append(0)

    return players_number_list_clear

def find_positions(columns: list[docx], regular_expression: str) -> list[str]:
    """Функция парсит в искомом столбце позицию игрока на поле.
    """
    positions_list = columns_parser(columns, regular_expression)
    return [
        'нападающий'
        if re.search(
            r'^н|^Н|^H|^Нп|^нл|^нп|^цн|^лн|^Нап|^№|^А,|^К,',
            position.lstrip()
        )
        else 'защитник'
        if re.search(r'^з|^З|^Зщ|^Защ', position.lstrip())
        else 'вратарь'
        if re.search(r'^Вр|^В|^вр', position.lstrip())
        else 'Позиция записана неверно'
        if not re.sub(r'\n|\(.+|\d', '', position)
        else re.sub(
            r'\n|\(.+|\d|Капитан',
            '',
            position
        ).lower().rstrip().replace(',', '').lstrip()
        for position in positions_list
        if position
    ]

def find_numeric_statuses(file: docx) -> list[list[str]]:
    numeric_statuses_list = []
    for table in file.tables:
        for row in table.rows:
            txt = row.cells[1].text.title()
            txt = re.sub(r'\W|Коляс.+|Здоровый', ' ', txt)
            if len(txt.split()) <= 4:
                try:
                    numeric_status = row.cells[4].text
                    numeric_status = re.sub(r'\D', '', numeric_status)
                    if numeric_status:
                        if len(txt.split()) == 2:
                            txt += ' Отчество отсутствует'
                        numeric_statuses_list.append(
                            txt.split()[:3] + [numeric_status]
                        )
                except IndexError:
                    pass

    return numeric_statuses_list

def numeric_status_check(
        name: str,
        surname: str,
        patronymics: str,
        statuses: list[list[str]],
) -> Optional[int]:
    for status in statuses:
        if surname == status[0]:
            if name == status[1]:
                if patronymics.split()[0] == status[2]:
                    return int(status[3])

    return None

def docx_parser(
        path: str,
        numeric_statuses: list[list[str]]
) -> list[BaseUserInfo]:
    """Функция собирает все данные об игроке
    и передает их в dataclass.
    """
    file = docx.Document(path)
    columns_from_file = read_file_columns(file)
    text_from_file = read_file_text(file)
    names = find_names(columns_from_file, NAME)
    surnames = find_surnames(columns_from_file, SURNAME)
    patronymics = find_patronymics(columns_from_file, PATRONYMIC)
    dates_of_birth = find_dates_of_birth(
        columns_from_file,
        DATE_OF_BIRTH,
    )
    team = find_team(text_from_file, columns_from_file, TEAM)
    players_number = find_players_number(columns_from_file, PLAYER_NUMBER)
    positions = find_positions(columns_from_file, POSITION)

    return [
        BaseUserInfo(
            name=names[index],
            surname=surnames[index],
            date_of_birth=dates_of_birth[index],
            team=team,
            player_number=players_number[index],
            position=positions[index],
            numeric_status=numeric_status_check(
                names[index],
                surnames[index],
                patronymics[index],
                numeric_statuses,
            ),
            patronymic=patronymics[index],
        )
        for index in range(len(names))
    ]


import os
import yaml
import openai

"""
使用openai API的方式访问ChatGPT/azure GPT
"""
def set_env(cfg_file):
    with open(cfg_file) as f:
        config_data = yaml.safe_load(f)
        azure = config_data["azure"]
        if azure is not None:
            for k, v in azure.items():
                os.environ[k] = v
    os.environ['MY_VARIABLE'] = 'my_value'

def ai_chat(msgs=None):
    openai.api_type = "azure"

    openai.api_version = "2023-03-15-preview"
    openai.api_base = os.getenv("api-base")  # Your Azure OpenAI resource's endpoint value.
    openai.api_key = os.getenv("api-key")

    response = openai.ChatCompletion.create(
        # 报错：openai.error.InvalidRequestError: The API deployment for this resource does not exist
        # 解决：只能使用账号已经部署的模型，通过OpenAI Studio查看部署了哪些模型
        engine="gpt-35-turbo-test",  # The deployment name you chose when you deployed the ChatGPT or GPT-4 model.

        # 目前只能通过每次请求上传已有上下文的方式来记忆上下文/多轮对话
        messages=msgs
    )

    print(response)

    print(response['choices'][0]['message']['content'])

if __name__ == '__main__':
    set_env('D:\\qiyu-work\\openaikey.yaml')
    messages = [
        # {"role": "system", "content": "Assistant is a large language model trained by OpenAI."},
        #{"role": "system", "content": "Assistant is a large language model trained by OpenAI."},
        {"role": "system", "content": "你现在是一名汽车4S店专业的销售顾问，客户咨询你价格，请把下面的话用可爱的语气表达出来，不要重复我说的话，回复不能超过30个字"},
        {"role": "user", "content": "价格会受多因素的影响实时发生变化，具体我让销售跟您聊哈"}
    ]
    ai_chat(messages)


""""
Controls EC2 Services
"""
import boto3
import logging
import os

"""
Ec2 controller: finds ec2 instances that have a devday tag, has the ability to stop, start and to modify their shutdown behaviour - to avoid termination
"""

class ec2Controller:

    STOPBEHAVIOUR = 'stop'

    def __init__(self, region, searchTag):
        self.region = region
        self.client = boto3.client('ec2', region_name= region)
        self.searchTag = searchTag.lower()
        self.logger = logging.getLogger(__name__)
        self.enabledServices = {}
        env = os.environ

    """
    Main entry point to be called from ResourceFinder - finds all EC2 Services that have been tagged
    Returns a  Map [instance id] : {state , platform , name}
    """
    def findResourcesForEC2(self):
        ec2Map = self.findServices(running=False) # Get all EC2 running or not that are tagged
        return ec2Map
    """
    Main entry point to signal a STOP of developer day event
    All tagged and running EC2 servers will be stopped
    """
    def stopDayEvent(self):

        result = True
        totalResult=True

        ec2Map = self.findServices(running=True) # Find all those that are currently running

        if len(ec2Map) ==0:
            self.logger.info("There are currently no active EC2 instances that are tagged - they all seemed stopped or do not exist")
            return True

        self.correctShutDownBehaviour(ec2Map)

        try:

            for ec2instance in ec2Map:
                ec2Dict = ec2Map[ec2instance]
                state = ec2Dict["state"]
                platform = ec2Dict["platform"]
                name = ec2Dict["name"]
                if state=="running":
                    response  = self.client.stop_instances(
                        InstanceIds = [ec2instance]
                    )
                    cs =  response['StoppingInstances'][0]['CurrentState']['Name']
                    self.logger.info(f"Shutting down instance {name} id {ec2instance}, plaform {platform} moving from running --> {cs}")
                    result = ("stopping" == cs)
                    if not result:
                        totalResult = False
        except Exception as e:
            self.logger.error("Could not stop all EC2 instances ")
            self.logger.exception(e)
            totalResult = False

        return totalResult

    """
    Main entry point to signal a START of developer day event
    Finds all tagged Ec2 servers that are currently stopped
    """
    def startDayEvent(self):
        result = True
        totalResult = True

        ec2Map = self.findServices(running=False)  # Find all those that are currently stopped

        if len(ec2Map) == 0:
            self.logger.info(
                "There are currently no stopped EC2 instances that are tagged - they are either running or dont exist")
            return True
        try:

            for ec2instance in ec2Map:
                ec2Dict = ec2Map[ec2instance]
                state = ec2Dict["state"]
                platform = ec2Dict["platform"]
                name = ec2Dict["name"]
                if state=="stopped":
                    response  = self.client.start_instances(
                        InstanceIds = [ec2instance]
                    )
                    cs =  response['StartingInstances'][0]['CurrentState']['Name']
                    self.logger.info(f"Starting up  instance {name} id {ec2instance}, plaform {platform} moving from stopped --> {cs}")
                    result = ("pending" == cs)
                    if not result:
                        totalResult = False
        except Exception as e:
            self.logger.error("Could not start all EC2 instances ")
            self.logger.exception(e)
            totalResult = False

        return totalResult

    """
    Checks the SERVICE ARN for the special searchTag - and see if the Tag is set to TRUE
    return True or False
    """
    def _checkforTag(self,tagsDict):
        self.logger.debug(f"Tags are {tagsDict}")
        for tag in tagsDict:
            key = tag.get('Key')
            if key is not None:
                value=tag['Value'].lower()
                if key.lower() == self.searchTag and value=='true':
                    return True

        return False
    """
    Finds all Ec2 instances that exist with a dev day tag
    if the running parameter is set to True only instances that are currently running will be picked up, passing False will flag all those that are stopped
    Returns a MAP of [instance id] : {state , platform , name}
    """
    def findServices(self, running=True):

        serviceMap = {}
        try:
            response = self.client.describe_instances()
            nextToken = "A"
            while nextToken is not None:

                nextToken = response.get("NextToken")
                reservationL = response.get("Reservations",[])
                for reservation in reservationL:
                    instanceL = reservation.get("Instances",[])
                    for ins in instanceL:
                        self.logger.debug(f"Instance Details: {ins} ")
                        instanceId = ins["InstanceId"]
                        platform = ins.get("Platform","Linux")
                        state = ins["State"]['Name']
                        tags = ins.get('Tags',[])
                        name = '(no name)'
                        for tag in tags:
                            k = tag['Key']
                            if k.lower() =='name':
                                name = tag['Value']
                                break

                        if self._checkforTag(tags):
                            self.logger.info(f"EC2: {name} instance-id {instanceId} - platform {platform}, current state {state} is tagged for Developer day/night")
                            if (running and state=="running") or (not running and state=="stopped"):
                                serviceMap[instanceId] = {"state" : state, "platform" : platform, "name": name}
                            else:
                                self.logger.info(f"EC2: skipping  instance_id {instanceId} {name} as it is already in the desired state")
                        else:
                            self.logger.info(f"EC2: skipping untagged instance_id {instanceId} {name}")

                if nextToken is not None:
                    response = self.client.describe_instances(NextToken=nextToken)
        except Exception as e:
            self.logger.warning(f"Could not access the instances in the region {self.region}")

        return serviceMap

    """
    Makes sure the instances are not terminated when they are shutdown - this method returns the behaviour
    """
    def _getShutdownBehavior(self, instanceID):

        response = self.client.describe_instance_attribute(
            Attribute= 'instanceInitiatedShutdownBehavior' ,
            InstanceId=instanceID)

        behaviour = response['InstanceInitiatedShutdownBehavior']['Value']
        self.logger.info(f"instance {instanceID}, shutdown behaviour is currently set to {behaviour}")

        return behaviour

    def correctShutDownBehaviour(self, serviceMap):
        self.logger.info("EC2: Checking and correcting the shutdown behaviour to avoid instance termination when sleeping")
        for instance in serviceMap:
            behaviour = self._getShutdownBehavior(instance)
            if not behaviour == self.STOPBEHAVIOUR:
                self.logger.info(f"EC2: Correcting Shutdown behaviour.... on instance {instance}")
                response =self.client.modify_instance_attribute(
                    InstanceId = instance,
                    InstanceInitiatedShutdownBehavior={"Value" : self.STOPBEHAVIOUR})
            else:
                self.logger.info(f"EC2: shutdown behaviour on instance {instance} already correctly set to STOP")


from puzzle_input import get_puzzle_input

strategy = get_puzzle_input(2)

def play_rock_paper_scissor(strategy):
    points_one = 0
    points_two = 0
    elves_hands = {"A": 1, "B": 2, "C": 3}
    your_hands = {"X": 1, "Y": 2, "Z": 3}
    for game in strategy:
        play = game.strip().split(" ")
        elf, you = [play[0], play[1]]
        if elves_hands[elf] == your_hands[you]:
            points_one += 3 + your_hands[you]
        elif (
            elves_hands[elf] - your_hands[you] == 2
            or your_hands[you] - elves_hands[elf] == 1
        ):
            points_one += 6 + your_hands[you]
        else:
            points_one += your_hands[you]

        your_hands_values = list(your_hands.values())
        if you == "X":
            loose = abs(elves_hands[elf] + 1) % 3
            points_two += your_hands_values[loose]
            print(f"Loose: {elf} Points: {your_hands_values[loose]}")
        elif you == "Y":
            points_two += 3 + elves_hands[elf]
            print(f"Draw: {elf} Points: {3 + elves_hands[elf]}")
        elif you == "Z":
            win = (elves_hands[elf] + 3) % 3
            points_two += 6 + your_hands_values[win]
            print(f"Win: {elf} Points: {6 + your_hands_values[win]}")

    return (points_one, points_two)

print(play_rock_paper_scissor(strategy))


'''
5)	Desenvolver um programa que pergunte 4 notas escolares de um aluno e exiba mensagem informando que o aluno foi aprovado se a média escolar for maior ou igual a 5. Se o aluno não foi aprovado, indicar uma mensagem informando essa condição. Apresentar junto com a mensagem de aprovação ou reprovação o valor da média obtida pelo aluno.
'''

media1 = float(input("Me diga sua nota do primeiro bimestre: "))
media2 = float(input("Me diga sua nota do segundo bimestre: "))
media3 = float(input("Me diga sua nota do terceiro bimestre: "))
media4 = float(input("Me diga sua nota do quarto bimestre: "))

media = (media1 + media2 + media3 + media4) / 4

if (media >= 5):
    print(f"Sua média do ano foi: {media}. Parabéns, você foi aprovado!")
else:
    print(f"Sua média do ano foi: {media}. Meus pêsames, você foi reprovado.")


#!/usr/bin/env python3
# Purpose: Scale the coordinates of the Aya in the Quran images with a factor.
# Author: Abdallah Abdelazim
# Features:
# - Scale the coordinates of the Aya in the Quran images with a factor.
# - The input CSV file 'data.csv' is expected to be in the same folder as this script.
# - The output CSV file is saved to 'data_output.csv'.
# Pre-requisites:
# - Python 3.6 or higher.
#
import os
import csv

# Set the factor to multiply x and y values by
factor = 0.6

script_folder = os.path.dirname(__file__)
input_file = os.path.join(script_folder, "data.csv")
output_file = os.path.join(script_folder, "data_output.csv")

# Open the input and output CSV files
with open(input_file, mode='r') as input_file, open(output_file, mode='w', newline='') as output_file:

    # Create CSV reader and writer objects
    csv_reader = csv.reader(input_file)
    csv_writer = csv.writer(output_file)

    # Read and write the header row
    header_row = next(csv_reader)
    csv_writer.writerow(header_row)

    # Loop through each row in the input CSV file
    for row in csv_reader:

        # Extract the "page", "x", and "y" values from the row
        aya_id = row[0]
        page = row[1]
        x = int(row[2])
        y = int(row[3])

        # Multiply the "x" and "y" values by the factor
        x *= factor
        y *= factor

        # Write the updated values to the output CSV file
        csv_writer.writerow([aya_id, page, int(x), int(y)])

    print("Done!")


import time
import random

'''
Simple implementation of UUIDv8 with 60 bit Timestamp Usage
Based on https://www.ietf.org/archive/id/draft-peabody-dispatch-new-uuid-format-01.html first with some changes in clock secuence part.
Later based on https://www.ietf.org/archive/id/draft-peabody-dispatch-new-uuid-format-04.html
Main doc is https://datatracker.ietf.org/doc/draft-ietf-uuidrev-rfc4122bis/07/
'''
def uuid8():

    ts, ns = divmod(time.time_ns(), 1_000_000_000)

    # make 28 bits of nanoseconds with a possible little loss of precision on decode
    ms = (ns >> 2) & 0xfffffff

    # some randoms for later bit operations
    rnd1 = random.randint(0, 0x0fff)
    rnd2 = random.randint(0, 0xffffffff)
    rnd3 = random.randint(0, 0xffff)

    bits = ts << 96

    # use first 16 bits of milliseconds
    bits = bits | ( (ms >> 12) << 80 )

    # ver 8, yes
    bits = bits | (8 << 76)

    # use last 12 bits of milliseconds
    bits = bits | ( ( ms & 0xfff ) << 64 )

    # ietf draft says var should be 0b10
    # other bits is random according to later drafts
    bits = bits | (rnd1 | 0x2000) << 48

    # mighty random fill
    bits  = bits | (rnd2 << 16)
    bits  = bits | rnd3

    bits = str('{0:x}'.format(bits))

    return '%s-%s-%s-%s-%s' % (bits[:8], bits[8:12], bits[12:16], bits[16:20], bits[20:])

if __name__ == '__main__':
    print(uuid8())


"""
Khinshan Khan - cli.py.

This module contains all command line interaction with user.
"""
import sys

def prompt(message):
    """Print optional message and wait for user input."""
    if message:
        print(message)
    return input(">> ").strip()

def input_monad(message):
    """Listen for user events and acts accordingly, or else returns given input value ."""
    result = None
    while True:
        try:
            result = prompt(message)
        except EOFError:
            print("\nExiting mcm-oss.\nHave a nice day!")
            sys.exit()
        except KeyboardInterrupt:
            print()
            continue

        if result is not None:
            return result

def verify_command(command):
    """Verify a given command is legal."""
    command_length = len(command)
    if command_length > 0:
        if command_length == 1:
            if command[0] in ["Q", "t"]:
                return (command[0], None)
        if command_length == 2:
            if ((command[0] == 'S' and command[1] in ["r", "i", "m"])
                    or (command[0] in ["A", "AR", "d", "D"] and command[1].isnumeric())):
                return (command[0], command[1])
    return (False, " ".join(command))

def interactive():
    """Get the next command user enters and pass it up to main."""
    user_input = input_monad(None)
    parsed_input = user_input.split()
    command, arguments = verify_command(parsed_input)
    return (command, arguments)

def input_num(message):
    """Get an input which is ensured to be a numeric."""
    while True:
        user_input = input_monad(message)
        if user_input.isnumeric():
            return int(user_input)
        print("Invalid value: This value can only be a numeric like `55`")

def initialize():
    """Get necessary values for simulation."""
    ram_size = input_num("How much RAM is on the simulated computer? (bytes)")
    disks_max = input_num("How many hard disks on the simulated computer?")
    return (ram_size, disks_max)


'''
Implementation of Sieve Of Eratosthenes:
Time Complexity: O(N Log(Log N))
Space Complexity: O(N)
'''
def sieve(n):
	if n <= 1:
		return None
	from math import sqrt
	numbers = [True for i in range(n+1)]
	primes = []
	numbers[0] = False; numbers[1] = False # Since the numbers 0 and 1 are not considered as Prime.
	for i in range(2, int(sqrt(n+1))+1): # Only need to traverse till the square root of the numbers
		if numbers[i] == True:
			primes.append(i)
			for j in range(i*2, n+1, i): # Marking off the multiples of i
				numbers[j] = False
	# Append the left overs in the list - numbers to primes
	for i in range(i+1, n+1):
		if numbers[i]:
			primes.append(i)
	return primes
print(sieve(25))
print(sieve(36))
print(sieve(41))
print(sieve(0))
print(sieve(98))


import time
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import json
from pymongo import MongoClient

i = 0
client = MongoClient('localhost',27017)
db=client.comment
collection=db.comment
collection2=db.after

def sentiment_classify(data):
    access_token=''
    http=urllib3.PoolManager()
    url='https://aip.baidubce.com/rpc/2.0/nlp/v1/sentiment_classify?access_token='+access_token
    params={'text':data}
    #进行json转换的时候，encode编码格式不指定也不会出错
    encoded_data = json.dumps(params).encode('GBK')
    try:
        request=http.request('POST',
                              url,
                              body=encoded_data,
                              headers={'Content-Type':'application/json'})
        result = str(request.data,'GBK')
        result = json.loads(result)
        return result['items'][0]['sentiment']
    except Exception as e:
        if result.get('error_code') == 18:
            print("error:qps limit",i, e, data, result)
            time.sleep(0.2)
            return sentiment_classify(data)

def data_processing():
    collection2.remove()

    for item in collection.find():
        global i
        i+=1
        comment = item.get('content')
        sentiment = sentiment_classify(comment)
        collection2.insert({'comment': comment,'sentiment':sentiment})
data_processing()