"""
Compile some information from a table
"""

import xlrd
import xlwt
import os

def frequencies_by_place(xls, entities_sheet, entities_id, entities_place,
                                 data_sheet, output, null=None, out_cols_basename=None,
                                 entities_filter=None, cols_exclusion=None):
    """
    Entities table:
    ID_Aluno | codigo_postal
    xxxxxxxx | xxxx-xxx
    xxxxxxxx | xxxx-xxx

    data_sheet:
    ID_Aluno | disciplina_1 | disciplina_2 | disciplina_n
    xxxxxxxx |    nota_1    |    xxxxxxx   |   xxxxxxx
    xxxxxxxx |    nota_2    |    xxxxxxx   |   xxxxxxx
    xxxxxxxx |    nota_3    |    xxxxxxx   |   xxxxxxx

    Output tables (one for each disciplina and other for the total)
                    |  nota_1  |  nota_2  |  nota_3  | Mean | Variance
    total           | NrAlunos | NrAlunos | NrAlunos |  x   |    y
    codigo_postal_1 | NrAlunos | NrAlunos | NrAlunos |  x   |    y
    codigo_postal_n | NrAlunos | NrAlunos | NrAlunos |  x   |    y

    In the entities_sheet and data_sheet, there must be only one referente
    for an entity (duplicated entities are not allowed)

    Filtering entities properties is possible
    """

    import numpy
    from decimal           import Decimal
    from glass.tbl.xls.fld  import get_columns_position
    from glass.tbl.xls.summ import list_unique_values_column
    from glass.tbl.xls.summ import count_values_column
    from glass.tbl.xls.summ import count_values_column_if_entity_exists
    from glass.tbl.xls.summ import count_values_column_by_entity_property

    """
    Aux method to estimate some generic statistics based on a "Dict Histogram"
    (total, mean and standard deviation)
    """
    def map_dict(histogram):
        if 'mean' in histogram.keys():
            del histogram['mean']

        if 'stdesviation' in histogram.keys():
            del histogram['stdesviation']

        if 'total' in histogram.keys():
            del histogram['total']

        # Get the total of entities
        histogram['total'] = sum(histogram.values())
        # Get mean
        numerator = 0
        denominator = 0
        std_sample = []
        for v in histogram:
            if type(v) != str:
                numerator += Decimal(v) * Decimal(histogram[v])
                denominator += histogram[v]
                std_sample += [v for x in range(histogram[v])]

        if numerator and denominator:
            histogram['mean'] = numerator / Decimal(denominator)
            histogram['stdesviation'] = Decimal(numpy.std(std_sample))

        return histogram

    # Open xls file
    __xls_file = xlrd.open_workbook(xls, on_demand=True)

    # Map entities with location
    """
    d = {
        entitie_id : location
    }
    """
    if entities_filter:
        filters = entities_filter.split(';')
        filters_map = {}
        for flt in filters:
            col_filter, val_filter = flt.split('=')
            filters_map[col_filter] = val_filter

        interest_col = [entities_id, entities_place] + filters_map.keys()

    else:
        filters_map = 0
        interest_col = [entities_id, entities_place]

    __entities_sheet = __xls_file.sheet_by_name(entities_sheet)
    id_place_position = get_columns_position(
        __entities_sheet, interest_col
    )

    if entities_id not in id_place_position.keys():
        raise ValueError('Couldn\'t find the given column id')
    elif entities_place not in id_place_position.keys():
        raise ValueError('Couldn\'t find the position of the place id')

    entities_map = {}
    for row in range(1, __entities_sheet.nrows):
        __id = __entities_sheet.cell(
            row, id_place_position[entities_id]).value

        location = __entities_sheet.cell(
            row, id_place_position[entities_place]).value

        if filters_map:
            c = 0
            for col_filter in filters_map:
                __filter = __entities_sheet.cell(
                    row, id_place_position[col_filter]).value

                if __filter != val_filter:
                    c += 1
                    break

            if c:
                continue

        entities_map[__id] = location

    # Count Entities by value in each column
    """
    d = {
        col_1 : {
            total : {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            },
            location_1: {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            },
            location_2: {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            }
        },
        col_2 : {
            total : {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            },
            location_1: {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            },
            location_2: {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            }
        },
        col_n : {
            ...
        }
    }
    """
    data_sheet = __xls_file.sheet_by_name(data_sheet)
    data = {}

    cols_exclusion = cols_exclusion if type(cols_exclusion) == list else \
        [cols_exclusion] if type(cols_exclusion) == str else 0

    for col in range(1, data_sheet.ncols):
        column_name = data_sheet.cell(0, col).value
        if cols_exclusion:
            if column_name in cols_exclusion:
                continue
        # List unique values
        values = list_unique_values_column(data_sheet, col)

        # Del NoData identifier if defined
        if null or null == 0:
            if null in values:
                values.remove(null)

        # Count occourences of a value in that column
        val_count = count_values_column_if_entity_exists(
            data_sheet, col, entities_map.keys(), values
        )

        for v in values:
            if v not in val_count:
                val_count[v] = 0

        data[column_name] = {'total' : map_dict(val_count)}

        # Do the same for each location
        locations = list(set(entities_map.values()))

        for place in locations:
            val_count = count_values_column_by_entity_property(
                data_sheet, col, place, entities_map,
                values
            )

            for v in values:
                if v not in val_count:
                    val_count[v] = 0

            data[column_name].update({place : map_dict(val_count)})

    """
    Update data dict with one combination of all keys/values data
    d.update({
        general : {
            total : {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            },
            location_1: {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            },
            location_2: {
                total   : total_count,
                value_1 : value_1_count,
                value_2 : value_2_count,
                value_n : value_n_count
            }
        }
    })
    """
    general = {}
    for col_sheet in data:
        for k in data[col_sheet]:
            if k not in general:
                general[k] = {}
                for _k in data[col_sheet][k]:
                    general[k][_k] = data[col_sheet][k][_k]
            else:
                for _k in data[col_sheet][k]:
                    if _k in general[k]:
                        general[k][_k] += data[col_sheet][k][_k]
                    else:
                        general[k][_k] = data[col_sheet][k][_k]
    data['general'] = general

    for __dict in data['general']:
        with_stats = map_dict(data['general'][__dict])
        data['general'][__dict] = with_stats

    # Write data in a new xls_file
    out_xls = xlwt.Workbook()
    for col_sheet in data:
        new_sheet = out_xls.add_sheet(col_sheet)

        if out_cols_basename:
            cols_basename = ['{b}_{_k}'.format(
                b=out_cols_basename, _k=str(k)
                ) for k in data[col_sheet]['total'].keys()]

        else:
            cols_basename = data[col_sheet]['total'].keys()

        cols_name = data[col_sheet]['total'].keys()

        cols_basename.sort()
        cols_name.sort()

        # Write columns names
        for c in range(len(cols_name)):
            new_sheet.write(0, c+1, cols_basename[c])

        # Write lines
        lnh = 1
        lines_name = data[col_sheet].keys()
        lines_name.sort()
        for line in lines_name:
            # Write line name
            new_sheet.write(lnh, 0, line)

            # Write counting data
            for cln in range(len(cols_name)):
                if cols_name[cln] in data[col_sheet][line].keys():
                    new_sheet.write(
                        lnh, cln + 1, data[col_sheet][line][cols_name[cln]]
                    )

            lnh += 1

    out_xls.save(output)

    __xls_file.release_resources()
    del __xls_file

def run_freq_by_place_by_sheet(xls, sheets_map, output_fld, null=None,
                               out_cols_basename=None, entities_filter=None,
                               cols_exclusion=None):
    """
    Execute frequencies_by_place for each group of entities and data sheets

    sheets_map should be something like this:
    sheets_map = {
        'entities_sheet_name' : {
            fid   : 'entities_field_id_name',
            place : 'entities_place_field_name',
            data  : 'data_sheet_name'
        },
        ...
    }
    """

    for entity_sheet in sheets_map:
        frequencies_by_place(
            xls, entity_sheet, sheets_map[entity_sheet]['fid'],
            sheets_map[entity_sheet]['place'],
            sheets_map[entity_sheet]['data'],
            output,
            null=null, out_cols_basename=out_cols_basename,
            entities_filter=entities_filter,
            cols_exclusion=cols_exclusion
        )

def frequencies_by_entity_attr(xls, entities_sheet, entities_id, attr_sheet,
                            interest_values, output, entities_filter=None,
                            attr_exclusion=None):
    """
    Count entities with a number of attributes with a set of specific values

    Example:
    specific_values = A, S
    Entities table:
    entity_id | some_data
    xxxxxxxx  | xxxx-xxx
    xxxxxxxx  | xxxx-xxx

    attributes_sheet:
    From tables as:
    entity | attr_1 | attr_2 | ... | attr_n
       0   |    S   |   S    | ... |   S
       1   |    A   |   B    | ... |   S
       2   |    B   |   B    | ... |   A
       3   |    S   |   A    | ... |   S
       4   |    A   |   S    | ... |   B

    We came to the following result:
    values  | entities_count | attribute_count
      S;A   |        2       |        3
      S;A   |        2       |        2
      S;A   |        1       |        1

    Filtering entities properties is possible... Many could be used...
    When many fields are used, the AND operator logic will be applied

    E.g of application:
    Number of students with a number of negatives
    """

    from glass.tbl.xls.fld import get_columns_position

    # Open file
    __xls = xlrd.open_workbook(xls, on_demand=True)

    # Map entities
    if entities_filter:
        filters = entities_filter.split(';')
        filters_map = {}
        for flt in filters:
            col_filter, val_filter = flt.split('=')
            filters_map[col_filter] = val_filter

        interest_col = [entities_id] + filters_map.keys()

    else:
        filters_map = 0
        interest_col = entities_id

    __entities_sheet = __xls.sheet_by_name(entities_sheet)
    cols_position = get_columns_position(__entities_sheet, interest_col)

    if entities_id not in cols_position:
        raise ValueError('Couldn\'t find the given column id')

    entities_map = {}
    for row in range(1, __entities_sheet.nrows):
        __id = __entities_sheet.cell(row, cols_position[entities_id]).value

        if filters_map:
            c = 0
            for col_filter in filters_map:
                __filter = __entities_sheet.cell(
                    row, cols_position[col_filter]).value

                if __filter != filters_map[col_filter]:
                    c += 1
                    break

            if c:
                continue

        entities_map[__id] = 0

    # Count occurences of the specific/interes values by entitie
    data_sheet = __xls.sheet_by_name(attr_sheet)

    interest_values = interest_values if type(interest_values) == list else \
        [interest_values] if type(interest_values) == str else 0

    if not interest_values: raise ValueError(
        'interest_values should be a list or string')

    for col in range(1, data_sheet.ncols):
        column_name = data_sheet.cell(0, col).value
        if attr_exclusion and type(attr_exclusion) == list:
            if column_name in attr_exclusion:
                continue

        for row in range(1, data_sheet.nrows):
            __id = data_sheet.cell(row, 0).value
            value = data_sheet.cell(row, col).value

            if value in interest_values:
                if __id in entities_map:
                    entities_map[__id] += 1

    # Count the number entities with the same number of occurences of the
    # interest/specific values
    occurence_count = entities_map.values()
    occurence_unique = list(set(occurence_count))

    entities_countv = {}
    for countv in occurence_unique:
        entities_count = occurence_count.count(countv)
        entities_countv[entities_count] = countv

    # Write output
    out_xls = xlwt.Workbook()
    new_sheet = out_xls.add_sheet(os.path.splitext(os.path.basename(output))[0])

    # colums
    c = ['values', 'entities_number', 'values_occurences']
    for i in range(len(c)):
        new_sheet.write(0, i, c[i])

    # values
    l = 1
    for i in entities_countv:
        new_sheet.write(l, 0, ';'.join([str(x) for x in interest_values]))
        new_sheet.write(l, 1, i)
        new_sheet.write(l, 2, entities_countv[i])
        l+=1

    out_xls.save(output)

    __xls.release_resources()
    del __xls

def frequencies_table(xls, data_sheet, output,
                           entities_sheet=None, entities_id=None,
                           entities_filter=None, values_filter=None):
    """
    Count values occurences in each attribute column

    input table:
    entity | attr_1 | attr_2 | ... | attr_n
       0   |   a    |   b    | ... |   a
       1   |   b    |   c    | ... |   b
       2   |   b    |   c    | ... |   c
       3   |   a    |   f    | ... |   d
       4   |   c    |   a    | ... |   f

    output table:
           | a | b | c | d | f |
    attr_1 | 2 | 2 | 1 | 0 | 0 |
    attr_2 | 1 | 1 | 2 | 0 | 1 |
    attr_n | 1 | 1 | 1 | 1 | 1 |

    It is possible filter entities and attribute values
    """

    from glass.tbl.xls.fld  import get_columns_position
    from glass.tbl.xls.summ import count_values_column
    from glass.tbl.xls.summ import count_values_column_if_entity_exists

    # Open file
    __xls = xlrd.open_workbook(xls, on_demand=True)

    if entities_sheet and entities_id and entities_filter:
        filters = entities_filter.split(';')
        filters_map = {}
        for flt in filters:
            col_filter, val_filter = flt.split('=')
            filters_map[col_filter] = val_filter

        __entities_sheet = __xls.sheet_by_name(entities_sheet)

        int_col = [entities_id] + filters_map.keys()
        cols_position = get_columns_position(
            __entities_sheet, int_col
        )

        entities_map = []
        for row in range(1, __entities_sheet.nrows):
            __id = __entities_sheet.cell(
                row, cols_position[entities_id]).value

            c = 0
            for col_filter in filters_map:
                __filter = __entities_sheet.cell(
                    row, cols_position[col_filter]).value

                if __filter != filters_map[col_filter]:
                    c += 1
                    break

            if c:
                continue

            entities_map.append(__id)

    else:
        entities_map = None

    attr_sheet = __xls.sheet_by_name(data_sheet)

    data = {}
    registed_values = []
    values_filter = values_filter if type(values_filter) == list else \
        [values_filter] if type(values_filter) == str else None

    for col in range(1, attr_sheet.ncols):
        col_name = attr_sheet.cell(0, col).value

        if not entities_map:
            values_count = count_values_column(attr_sheet, col, values_filter)
        else:
            values_count = count_values_column_if_entity_exists(
                attr_sheet, col, entities_map, values_filter
            )

        registed_values.extend(values_count.keys())

        data[col_name] = values_count

    registed_values = list(set(registed_values))

    # Write output
    out_xls = xlwt.Workbook()
    new_sheet = out_xls.add_sheet(
        os.path.splitext(os.path.basename(output))[0]
    )

    # columns
    for i in range(len(registed_values)):
        new_sheet.write(0, i+1, registed_values[i])

    # Write values
    r=1
    for row in data:
        new_sheet.write(r, 0, row)
        for col in range(len(registed_values)):
            if registed_values[col] in data[row]:
                new_sheet.write(r, col+1, data[row][registed_values[col]])
        r += 1

    out_xls.save(output)

    __xls.release_resources()
    del __xls


import psycopg2
import pandas as pd
from sqlalchemy import create_engine
from Services.ConfigParser import ConfigParser

class Postgres:
    def __init__(self):
        config = ConfigParser()
        db_config = config.DbConfigSettings()
        self.connection_string = "postgresql://" + db_config["username"] + ":" + db_config["password"] + "@" + db_config["host"] + "/" + db_config["databaseName"]

    def LoadFromDataFrame(self, data, tableName, boolReplace):
        if (boolReplace):
            exists = 'replace'
        else:
            exists = 'append'
        db = create_engine(self.connection_string)
        conn = db.connect()
        df = pd.DataFrame(data)
        df.to_sql(tableName, con=conn, if_exists=exists, index=False)
        conn = psycopg2.connect(self.connection_string)
        conn.autocommit = True
        conn.close()

    def GetData(self, query):
        conn = psycopg2.connect(self.connection_string)
        cur = conn.cursor()
        cur.execute(query)
        records = cur.fetchall()
        conn.commit()
        conn.close()
        return records


'''Using recurssion'''
def dec_to_bin(n):
    if n>1:
        dec_to_bin(n//2)
    print(n%2,end=' ')

if __name__=='__main__':
    dec_to_bin(16)

# n = int(input("Enter a decimal number: "))
# n1 = []
# while n!=0:
#     rem = n % 2
#     n1.append(rem)
#     n//=2
# n2 = n1.reverse()
# for i in n1:
#     print(i,end=' ')


#coding=utf-8
import json
import queue
import random
import re
import time
import traceback
import urllib
import urllib.request
import uuid

import requests
import wipe_off_html_tag
from bs4 import BeautifulSoup

import data_storager

class ParentChild(object):
    def __init__(self):
        self._parent_id = None
        self._parent_title = None
        self._parent_url = None
        self._parent_layer = None
        self._child_id = None
        self._child_title = None
        self._child_url = None
        self._child_layer = None

class Spider(object):
    """
    layer：第一层，首页， 科学百科
           第二层，航空航天
           第三层，3·8马来西亚航班失踪事件
           第四层，马来西亚航空公司
           第h层
    """
    def __init__(self):
        self._post_lemmas_url = 'https://baike.baidu.com/wikitag/api/getlemmas'
        self._get_zhixin_url = 'https://baike.baidu.com/wikiui/api/zhixinmap?lemmaId='
        self._get_guess_like_url = 'https://baike.baidu.com/api/wikiui/guesslike?url='
        self._parent_childs = queue.Queue()  # relationship队列
        self._crawl_layers = 4-1  # 需要爬取的最后一层的上一层

    def execute(self):
        # 录入首页，科学百科
        root_url = 'https://baike.baidu.com/science'
        root_title = r'科学百科'
        root_id = str(uuid.uuid3(uuid.NAMESPACE_DNS, str(root_url)))
        # data_storager.insert_webpage(root_id, root_title, root_url, None)
        # print("wxm entity:", root_title, root_url)

        # 录入首页下二级页面
        first_page_html = self.get_firstpage_content("https://baike.baidu.com/science")
        first_page_urls = self.get_firstpage_url(first_page_html)  # 10条二级页面信息
        tag_ids = self.get_firstpage_tagid(first_page_urls)  # 9条二级页面信息
        first_page_urls_slice = first_page_urls[1:2]  # 获取前半部分URL
        tag_ids_slice = tag_ids[1:2]  # 获取前半部分Tag

        for page_id in range(0, 61):
            try:
                for i in range(0, len(tag_ids_slice)):
                    # 处理每一个二级页面，如航空航天，天文学，环境生态
                    print("page_id:" + str(page_id) + ", tagids：" + str(tag_ids_slice[i]))
                    time.sleep(random.choice(range(1, 3)))
                    title = first_page_urls_slice[i].get_text()
                    url = first_page_urls_slice[i].get('href')
                    id = str(uuid.uuid3(uuid.NAMESPACE_DNS, str(url)))  # 为当前网页生成UUID
                    data_storager.insert_relationship(root_id, root_title, id, title)  # 插入首页与二级页面的relationship，如<科学百科_id, 航空航天_id>
                    data_storager.insert_webpage(id, title, url, None)  # 插入二级页面webpage信息,页面内容为空，如航空航天
                    print("wxm entity:",title, url)
                    # 录入所有的<二级页面，三级页面>配对信息到缓存中,如航空航天下面的当前page_id下的所有条目,如马航,911事件
                    self.get_second_page_url(id, title, url, tag_ids_slice[i], page_id)
                while (self._parent_childs):  # 从第三层（马航,911事件，位于缓存对象的child_xx中）开始，处理队列每一个页面
                    pc = self._parent_childs.get(0)
                    self.deal_with_child_webpage(pc)  # 用layer控制爬虫爬取深度，调用deal_with_child_webpage方法即插入数据库
            except:
                print(traceback.format_exc())
                print("wxmException，page_id：" + str(page_id))
            print("wxmmmmmmmmmmmmmm， tag_ids_slice:" + str(tag_ids_slice[i]))
        print("wxmmmmmmmmmmmmmm， page_id：" + str(page_id))

    def deal_with_child_webpage(self, pc):
        """ 处理当前缓存队列中的对象中的child对象，该方法始终以child为中心处理，parent在前面处理了；从第三层开始的 """
        try:
            htmL_content = self.get_sub_page_content(pc._child_url)  # 获取当前孩子页面的html内容
            if htmL_content == None:
                return
            real_content = self.get_web_real_content(htmL_content) # 获取网页的真正需要入库的content
            if len(real_content)>100 and real_content!=None:
                real_content=real_content[:100]
            data_storager.insert_relationship(pc._parent_id, pc._parent_title, pc._child_id, pc._child_title)  # 插入二级页面与三级页面的relationship，如<航空航天_id, 马航_id>
            data_storager.insert_webpage(pc._child_id, pc._child_title, pc._child_url, real_content)  # 插入当前孩子页面信息，如马航 todo, 解析出htmlcontent中汉字
            print("wxm entity:",pc._child_title, pc._child_url)
            self.batch_insert_webpage_attributes(pc._child_id, pc._child_title, htmL_content)  # 插入当前页面的所有属性信息，如马航下的中文名 3·8马来西亚航班失踪事件
            self._crawl_layers = random.choice(range(3, 7))
            if pc._child_layer <= self._crawl_layers:  # 3<=4,仅从第三层爬到第四层
                self.get_sub_urls(pc._child_id, pc._child_title, pc._child_url, pc._child_layer, htmL_content)  # 获取当前网页下所有连接，传入第三层的id，title，url，获取第四层信息，加入到缓存队列中
        except:
            print(traceback.format_exc())
            print("wxmException, ", pc._child_id, pc._child_title, pc._child_url)

    def batch_insert_webpage_attributes(self, id, title, htmL_content):
        soup_findattribute = BeautifulSoup(htmL_content, 'html.parser')
        data_findattribute = soup_findattribute.find_all('div', {'class': 'basic-info cmn-clearfix'})
        for attributes in data_findattribute:
            attribute_names = attributes.find_all('dt', {'class': 'basicInfo-item name'})
            names_list = []
            for a in attribute_names:
                attribute_name = a.get_text()
                names_list.append(attribute_name)
            attribute_values = attributes.find_all('dd', {'class': 'basicInfo-item value'})
            values_list = []
            for b in attribute_values:
                attributes_value = b.get_text()
                values_list.append(attributes_value)
            nvs = zip(names_list, values_list)
            nvDict = dict((str(names_list), str(values_list)) for names_list, values_list in nvs)
            for names_lists, values_lists in nvDict.items():
                names_lists_insert = names_lists
                value_lists_insert = values_lists.strip("\n")
                data_storager.insert_attributes(id, title, names_lists_insert, value_lists_insert)

    def get_web_real_content(self, htmL_content):
        soup = BeautifulSoup(htmL_content, 'html.parser')
        main_content = soup.find('div', {'class': 'main-content'})
        return wipe_off_html_tag.filter_tags(str(main_content)).strip(r'收藏\n查看我的收藏\n').\
            strip(r'有用+1\n已投票\n').strip(r'编辑\n锁定\n').strip(r'审核\n。')

    def get_sub_urls(self, id, title, url, layer, content):
        soup = BeautifulSoup(content, 'html.parser')
        body = soup.body  # 获取body部分

        # 爬取para中连接词条
        try:
            link_list1 = {}
            link_body1 = body.find_all('div', {'class': 'para', 'label-module': "para"})
            for link_body in link_body1:
                result_list = link_body.find_all('a', {'target': '_blank'})
                if result_list:
                    for link in result_list:
                        fourth_page_url_postfix = link.get('href')
                        pattern = re.compile(r'/pic')
                        match = pattern.match(fourth_page_url_postfix)
                        if not match:
                            sub_title = link.get_text()
                            sub_url = "http://baike.baidu.com" + fourth_page_url_postfix
                            link_list1[sub_title] = sub_url
        except:
            print(traceback.format_exc())
            print("wxmException, 爬取para中连接词条 todo print 出错更详细信息:" + id + ", " + title + ", " + url)

        # 爬取相关信息词条
        try:
            link_list2 = {}
            link_body2 = body.find('div', {'class': 'zhixin-box'})
            newlemmaid = link_body2['data-newlemmaid']
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36'
            }
            data = {"lemmaId": newlemmaid}
            request_url = self._get_zhixin_url + str(newlemmaid)
            content = requests.get(request_url, params=data, headers=headers).content
            response = content.decode('unicode-escape').replace("\/", "/")
            js = json.loads(response)
            for i in range(len(js)):
                title_urls = js[i]['data']
                for j in range(len(title_urls)):
                    sub_title = js[i]['data'][j]['title']
                    sub_url = js[i]['data'][j]['url']
                    link_list2[sub_title] = sub_url
        except:
            print(traceback.format_exc())
            print("wxmException, 爬取相关信息词条 todo print 出错更详细信息:" + id + ", " + title + ", " + url)

        # 爬取猜你喜欢词条
        try:
            link_list3 = {}
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36'
            }
            data = {
                "url": sub_url,
                "lemmaTitle": sub_title,
                "eid": 202
            }
            request_url = self._get_guess_like_url + str(sub_url) + "&lemmaTitle=" + sub_title + "&eid=202"
            content = requests.get(request_url, params=data, headers=headers).content
            try:
                jso1 = json.loads(content)
                ad_place_data = jso1["ad_place_list"][0]["ad_place_data"]
                ad_place_data2 = ad_place_data.replace("\/", "/")
                jso2 = json.loads(ad_place_data2)
                ads = jso2["ads"]
                for ad in ads:
                    sub_title = ad['title']
                    sub_url = ad['url']
                    link_list3[sub_title] = sub_url
            except:
                print(traceback.format_exc())
                print("wxmException, 爬取猜你喜欢词条 json 解析失败")
        except:
            print(traceback.format_exc())
            print("wxmException, 爬取猜你喜欢词条 todo print 出错更详细信息:" + id + ", " + title + ", " + url)
        try:
            dictMerged2 = {}
            dictMerged1 = dict(link_list1, **link_list2)
            dictMerged2 = dict(dictMerged1, **link_list3)
        except:
            print(traceback.format_exc())
            print("wxmException, 合并dict失败")

        for sub_title, sub_url in dictMerged2.items():
            pc = ParentChild()
            pc._parent_id = id
            pc._parent_title = title
            pc._parent_url = url
            pc._parent_layer = layer
            child_id = str(uuid.uuid3(uuid.NAMESPACE_DNS, str(sub_url)))  # 为当前孩子页面生成UUID
            pc._child_id = child_id
            pc._child_title = sub_title
            pc._child_url = sub_url
            pc._child_layer = pc._parent_layer + 1
            self._parent_childs.put(pc)

    def get_sub_page_content(self, url3, data=None):
        header = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Encoding': 'gzip,deflate,sdch',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Connection': 'keep-alive',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.143 Safari/537.36'
        }
        timeout = 180
        try:
            result2 = requests.get(url3, headers=header, timeout=timeout)
            result2.encoding = 'utf-8'
            return result2.text
        except urllib.request.HTTPError as e:
            print(traceback.format_exc())
            print('wxmException,  1:', e.code)
        except urllib.request.URLError as e:
            print(traceback.format_exc())
            print('wxmException,  2:', e.reason)
        except:
            print(traceback.format_exc())
            print('wxmException,  0')
        return None

    def get_second_page_url(self, id, title, url, tagId, page):
        """
            父亲id是传入的，孩子id是生成的
        """
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36'
        }
        data = {
            "limit": "24",
            "timeout": 3000,
            "filterTags": [],
            "tagId": tagId,
            "fromLemma": False,
            "contentLength": 40,
            "page": page
        }
        try:
            content = requests.post(self._post_lemmas_url, data=data, headers=headers).content
            response = content.decode('unicode-escape').replace("\/", "/")
            js = json.loads(response)
            for i in range(len(js['lemmaList'])):
                sub_title = (js['lemmaList'][i]['lemmaTitle'])
                sub_url = (js['lemmaList'][i]['lemmaUrl'])
                pc = ParentChild()
                pc._parent_id = id
                pc._parent_title = title
                pc._parent_url = url
                pc._parent_layer = 2
                child_id = str(uuid.uuid3(uuid.NAMESPACE_DNS, str(sub_url)))  # 为当前孩子页面生成UUID
                pc._child_id = child_id
                pc._child_title = sub_title
                pc._child_url = sub_url
                pc._child_layer = 3
                self._parent_childs.put(pc)
        except:
            print(traceback.format_exc())
            print("wxmException, ", id, title, url, tagId, page)  # todo 描述

    def get_firstpage_content(self, url, data=None):
        header = {
            'Accept': 'text/css,*/*;q=0.1',
            'Accept-Encoding': 'gzip,deflate,sdch',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Connection': 'keep-alive',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.143 Safari/537.36'
        }
        timeout = random.choice(range(80, 180))
        while True:
            try:
                rep = requests.get(url, headers=header, timeout=timeout)
                rep.encoding = 'utf-8'
                req = urllib.request.Request(url, data, header)
                response = urllib.request.urlopen(req, timeout=timeout)
                response.close()
                break
            except urllib.request.HTTPError as e:
                print(traceback.format_exc())
                print('wxmException,  1:', e)
                time.sleep(random.choice(range(1,3)))
            except urllib.request.URLError as e:
                print(traceback.format_exc())
                print('wxmException,  2:', e)
                time.sleep(random.choice(range(1,3)))
            except:
                print(traceback.format_exc())
                print('wxmException 0')
        return rep.text

    def get_firstpage_url(self, html):
        bs = BeautifulSoup(html, "html.parser")
        body = bs.body  # 获取body部分
        data = body.find_all('div', {'class': 'category-info'})  # 找到id为7d的div
        urls_list = []
        for a in data:
            ul = a.find('h5')
            url = ul.find('a')
            urls_list.append(url)
        return urls_list

    def get_firstpage_tagid(self, first_page_urls):
        links_list = []
        for i in first_page_urls:
            links = i.get('href')
            links_list.append(links)
        tagid_list = []
        for b in links_list:
            tagid = b[46:51]
            tagid_list.append(tagid)
        # 这里可以del 多个first_page_urls，将其分成多部分，供两台机器一起跑
        # 可以用切片，将tagid_list选择出对应的部分来
        del first_page_urls[0]
        del tagid_list[0]
        return tagid_list  # 检测出返回值最好跟你要输出的内容保持一直，出错率比较低

if __name__ == '__main__':
    spider = Spider()
    spider.execute()


from flask import Flask, request, jsonify
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)

# Replace 'your-connection-string' with your actual connection string
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://john1:Lucban2101@35.239.116.222/john1'
db = SQLAlchemy(app)

class Item(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(255), nullable=False)
    description = db.Column(db.String(255))

@app.route('/items', methods=['GET'])
def get_items():
    items = Item.query.all()
    result = []
    for item in items:
        result.append({
            'id': item.id,
            'name': item.name,
            'description': item.description
        })
    return jsonify(result)

@app.route('/items', methods=['POST'])
def create_item():
    data = request.get_json()
    new_item = Item(name=data['name'], description=data.get('description'))
    db.session.add(new_item)
    db.session.commit()
    return jsonify({'message': 'Item created successfully'})

@app.route('/items/<int:item_id>', methods=['PUT'])
def update_item(item_id):
    item = Item.query.get(item_id)
    if item is None:
        return jsonify({'message': 'Item not found'}), 404
    data = request.get_json()
    item.name = data['name']
    item.description = data.get('description')
    db.session.commit()
    return jsonify({'message': 'Item updated successfully'})

@app.route('/items/<int:item_id>', methods=['DELETE'])
def delete_item(item_id):
    item = Item.query.get(item_id)
    if item is None:
        return jsonify({'message': 'Item not found'}), 404
    db.session.delete(item)
    db.session.commit()
    return jsonify({'message': 'Item deleted successfully'})

if __name__ == '__main__':
    app.run(debug=True)


import numpy as np
from random import random, choice

class Agent:
    def __init__(self, alpha, gamma, eps, env):
        self.alpha = alpha
        self.gamma = gamma
        self.eps = eps
        self.env = env
        self.qtable = np.zeros((env.observation_space.n, env.action_space.n))

    def update_qtable(self, curr_state, action, step_reward, next_state):
        curr_q = self.qtable[curr_state, action]
        max_next_q = np.max(self.qtable[next_state])

        self.qtable[curr_state, action] = curr_q + self.alpha * (
            step_reward + self.gamma * max_next_q - curr_q
        )

    def __best_action(self, state_qtable):
        all_best_actions = np.argwhere(state_qtable == np.amax(state_qtable))
        return choice(all_best_actions.flatten().tolist())

    def explore(self, curr_state):
        if random() < self.eps:
            return self.env.action_space.sample()
        else:
            return self.__best_action(self.qtable[curr_state])

    def choose_action(self, curr_state, random_action):
        if random_action:
            return self.env.action_space.sample()
        else:
            return self.__best_action(self.qtable[curr_state])


from __future__ import annotations

from datetime import datetime
from typing import Any
from gym_utilities import in_week, create_offering_dict

# The additional pay per hour that instructors receive for each certificate they
# hold.
BONUS_RATE = 1.50

class WorkoutClass:
    """A workout class that can be offered at a gym.

    === Public Attributes ===
    name: The name of the workout class.

    === Private Attributes ===
    _required_certificates: The certificates that an instructor must hold to
        teach this WorkoutClass.
    """
    name: str
    _required_certificates: list[str]

    def __init__(self, name: str, required_certificates: list[str]) -> None:
        """Initialize a new WorkoutClass called <name> and with the
        <required_certificates>.

        >>> workout_class = WorkoutClass('Kickboxing', ['Strength Training'])
        >>> workout_class.name
        'Kickboxing'
        """
        self.name = name
        self._required_certificates = required_certificates[:]

    def get_required_certificates(self) -> list[str]:
        """Return all the certificates required to teach this WorkoutClass.

        >>> workout_class = WorkoutClass('Kickboxing', ['Strength Training'])
        >>> needed = workout_class.get_required_certificates()
        >>> needed
        ['Strength Training']
        >>> needed.append('haha')
        >>> try_again = workout_class.get_required_certificates()
        >>> try_again
        ['Strength Training']
        """
        # Make a copy of the list to avoid aliasing
        return self._required_certificates[:]

    def __eq__(self, other: Any) -> bool:
        """Return True iff this WorkoutClass is equal to <other>.

        Two WorkoutClasses are considered equal if they have the same name and
        the same required certificates.

        >>> workout_class = WorkoutClass('Kickboxing', ['Strength Training'])
        >>> workout_class2 = WorkoutClass('Kickboxing', ['Strength Training'])
        >>> workout_class == workout_class2
        True
        >>> d = {1: 17}
        >>> workout_class == d
        False
        """
        if not isinstance(other, WorkoutClass):
            return False
        return (self.name == other.name
                and self._required_certificates == other._required_certificates)

class Instructor:
    """
    An instructor at a Gym.

    === Public Attributes ===
    name: The name of this instructor.

    === Private Attributes ===
    _id: The id representing this instructor.
    _qualifications: A list of qualifications this instructor holds.
    """

    name: str
    _id: int
    _qualifications: list[str]

    def __init__(self, _id: int, name: str) -> None:
        """
        Initialize an Instructor with <name>, <id>,
        and qualifications if the instructor
        has any qualifications.
        """
        self._id = _id
        self.name = name
        self._qualifications = []

    def add_certificate(self, q: str) -> bool:
        """
        Appends certificate <q> to self._qualifications.
        """
        if q not in self._qualifications:
            self._qualifications.append(q)
            return True
        return False

    def get_id(self) -> int:
        """
        Returns the Instructor's id.
        """
        temp = self._id
        return temp

    def get_certificates(self) -> list[str]:
        """
        Returns a copy of the Instructor's _qualifications.
        """
        return self._qualifications[:]

class Gym:
    """A gym that hosts workout classes taught by instructors.

    All offerings of workout classes start on the hour and are 1 hour long.
    If a class starts at 7:00 pm, for example, we say that the class is "at"
    the timepoint 7:00, or just at 7:00.

    === Public Attributes ===
    name: The name of the gym.

    === Private Attributes ===
    _instructors: The instructors who work at this Gym.
        Each key is an instructor's ID and its value is the Instructor object
        representing them.
    _workouts: The workout classes that are taught at this Gym.
        Each key is the name of a workout class and its value is the
        WorkoutClass object representing it.
    _room_capacities: The rooms and capacities in this Gym.
        Each key is the name of a room and its value is the room's capacity,
        that is, the number of people who can register for a class in the room.
    _schedule: The schedule of classes offered at this gym.
        Each key is a date and time and its value is a nested dictionary
        describing all offerings that start then. In the nested dictionary,
        each key is the name of a room that has an offering scheduled then,
        and its value is a tuple describing the offering. The tuple elements
        record, in order:
            - the instructor teaching the class,
            - the workout class itself, and
            - a list of registered clients. Each client is represented in the
              list by a unique string.

    === Representation Invariants ===
    - All instructors in _schedule are in _instructors (the reverse is not
      necessarily true).
    - All workout classes in _schedule are in _workouts (the reverse is not
      necessarily true).
    - All rooms recorded in _schedule are also recorded in _room_capacities (the
      reverse is not necessarily true).
    - Two workout classes cannot be scheduled at the same time in the same room.
    - No instructor is scheduled to teach two workout classes at the same time.
      I.e., there does not exist timepoint t, and rooms r1 and r2 such that
          _schedule[t][r1][0] == _schedule[t][r2][0]
    - No client can take two workout classes at the same time.
      I.e., there does not exist timepoint t, and rooms r1 and r2 such that
          c in _schedule[t][r1][2] and c in _schedule[t][r2][2]
    - If an instructor is scheduled to teach a workout class, they have the
      necessary qualifications.
    - If there are no offerings scheduled at date and time <d>, then <d>
      does not occur as a key in _schedule.
    - If there are no offerings scheduled at date and time <d> in room <r> then
      <r> does not occur as a key in _schedule[d]
    - Each list of registered clients for an offering is ordered with the most
      recently registered client at the end of the list.
    """
    name: str
    _instructors: dict[int, Instructor]
    _workouts: dict[str, WorkoutClass]
    _room_capacities: dict[str, int]
    _schedule: dict[datetime,
                    dict[str, tuple[Instructor, WorkoutClass, list[str]]]]

    def __init__(self, gym_name: str) -> None:
        """Initialize a new Gym with <name>. Initially, this gym has no
        instructors, workout classes, rooms, or offerings.

        >>> ac = Gym('Athletic Centre')
        >>> ac.name
        'Athletic Centre'
        """
        self.name = gym_name
        self._instructors = {}
        self._workouts = {}
        self._room_capacities = {}
        self._schedule = {}

    def add_instructor(self, instructor: Instructor) -> bool:
        """Add a new <instructor> to this Gym's roster iff the <instructor> does
        not have the same id as another instructor at this Gym.

        Return True iff the id has not already been added to this Gym's roster.

        >>> ac = Gym('Athletic Centre')
        >>> diane = Instructor(1, 'Diane')
        >>> ac.add_instructor(diane)
        True
        """
        for _id in self._instructors:
            if instructor.get_id() == _id:
                return False

        self._instructors[instructor.get_id()] = instructor
        return True

    def add_workout_class(self, workout_class: WorkoutClass) -> bool:
        """Add a <workout_class> to this Gym iff the <workout_class> does not
        have the same name as another WorkoutClass at this Gym.

        Return True iff the workout class has not already been added this Gym.

        >>> ac = Gym('Athletic Centre')
        >>> kickboxing = WorkoutClass('Kickboxing', ['Strength Training'])
        >>> ac.add_workout_class(kickboxing)
        True
        """
        for workout_name in self._workouts:
            if workout_class.name == workout_name:
                return False

        self._workouts[workout_class.name] = workout_class
        return True

    def add_room(self, name: str, capacity: int) -> bool:
        """Add a room with <name> and <capacity> to this Gym iff there is not
         already a room with <name> at this Gym.

        Return True iff the room has not already been added to this Gym.

        >>> ac = Gym('Athletic Centre')
        >>> ac.add_room('Dance Studio', 50)
        True
        """
        for room_name in self._room_capacities:
            if name == room_name:
                return False

        self._room_capacities[name] = capacity
        return True

    def schedule_workout_class(self, time_point: datetime, room_name: str,
                               workout_name: str, instr_id: int) -> bool:
        """Add an offering to this Gym at <time_point> iff: the room with
        <room_name> is available, the instructor with <instr_id> is qualified
        to teach the workout class with <workout_name>, and the instructor is
        not teaching another workout class at the same <time_point>.

        A room is available iff it does not already have another workout class
        scheduled at that day and time.

        The added offering starts with no registered clients.

        Return True iff the offering was added.

        Preconditions:
            - The room has already been added to this Gym.
            - The Instructor has already been added to this Gym.
            - The WorkoutClass has already been added to this Gym.

        >>> ac = Gym('Athletic Centre')
        >>> jacqueline = Instructor(1, 'Jacqueline Smith')
        >>> ac.add_instructor(jacqueline)
        True
        >>> jacqueline.add_certificate('Cardio 1')
        True
        >>> diane = Instructor(2, 'Diane Horton')
        >>> ac.add_instructor(diane)
        True
        >>> ac.add_room('Dance Studio', 18)
        True
        >>> ac.add_room('lower gym', 50)
        True
        >>> boot_camp = WorkoutClass('Boot Camp', ['Cardio 1'])
        >>> ac.add_workout_class(boot_camp)
        True
        >>> tap = WorkoutClass('Intro Tap', [])
        >>> ac.add_workout_class(tap)
        True
        >>> sep_9_2022_12_00 = datetime(2022, 9, 9, 12, 0)
        >>> ac.schedule_workout_class(sep_9_2022_12_00, 'lower gym',\
        boot_camp.name, jacqueline.get_id())
        True
        >>> ac.schedule_workout_class(sep_9_2022_12_00, 'Dance Studio',\
        tap.name, diane.get_id())
        True
        """

        if time_point in self._schedule:
            for _class in self._schedule[time_point]:
                if self._schedule[time_point][_class][0].get_id() == instr_id:
                    return False

        for q in self._workouts[workout_name].get_required_certificates():
            if q not in self._instructors[instr_id].get_certificates():
                return False

        if time_point in self._schedule:
            for room in self._schedule[time_point]:
                if room_name == room:
                    return False

        if self._schedule.get(time_point) is None:
            self._schedule[time_point] \
                = {room_name: (self._instructors[instr_id],
                               self._workouts[workout_name],
                               [])}
            return True
        self._schedule[time_point][room_name] \
            = (self._instructors[instr_id], self._workouts[workout_name], [])
        return True

    def register(self, time_point: datetime, client: str, workout_name: str) \
            -> bool:
        """Add <client> to the WorkoutClass with <workout_name> that is being
        offered at <time_point> iff the client has not already been registered
        in any course (including <workout_name>) at <time_point>, and the room
        is not full.

        If the WorkoutClass is being offered in more than one room at
        <time_point>, then add the client to the room that has the most clients
        already registered but still has available space. In the case of a tie,
        register <client> in any of the tied classes.

        Return True iff the client was added.

        Precondition: the WorkoutClass with <workout_name> is being offered in
            at least one room at <time_point>.

        >>> ac = Gym('Athletic Centre')
        >>> diane = Instructor(1, 'Diane')
        >>> diane.add_certificate('Cardio 1')
        True
        >>> ac.add_instructor(diane)
        True
        >>> ac.add_room('Dance Studio', 50)
        True
        >>> boot_camp = WorkoutClass('Boot Camp', ['Cardio 1'])
        >>> ac.add_workout_class(boot_camp)
        True
        >>> sep_9_2022_12_00 = datetime(2022, 9, 9, 12, 0)
        >>> ac.schedule_workout_class(sep_9_2022_12_00, 'Dance Studio',\
        boot_camp.name, diane.get_id())
        True
        >>> ac.register(sep_9_2022_12_00, 'Philip', 'Boot Camp')
        True
        >>> ac.register(sep_9_2022_12_00, 'Philip', 'Boot Camp')
        False
        """
        if time_point not in self._schedule:
            return False

        for _class in self._schedule[time_point]:
            if client in self._schedule[time_point][_class][2]:
                return False

        listy = []

        for room_name in self._schedule[time_point]:
            if self._schedule[time_point][room_name][1].name == workout_name:
                listy.append((self._room_capacities[room_name]
                              - len(self._schedule[time_point][room_name][2]),
                              room_name))

        listy.sort()
        listy.reverse()

        for item in listy:
            if item[0] != 0:
                self._schedule[time_point][item[1]][2].append(client)
                return True

        return False

    def instructor_hours(self, time1: datetime, time2: datetime) -> \
            dict[int, int]:
        """Return a dictionary reporting the hours worked by instructors
        teaching classes that start at any time between <time1> and <time2>,
        inclusive.

        Each key is an instructor ID and its value is the total number of hours
        worked by that instructor between <time1> and <time2>. Both <time1> and
        <time2> specify the start time for an hour when an instructor may have
        taught.

        Precondition: time1 <= time2

        >>> ac = Gym('Athletic Centre')
        >>> diane = Instructor(1, 'Diane')
        >>> david = Instructor(2, 'David')
        >>> diane.add_certificate('Cardio 1')
        True
        >>> ac.add_instructor(diane)
        True
        >>> ac.add_instructor(david)
        True
        >>> ac.add_room('Dance Studio', 50)
        True
        >>> boot_camp = WorkoutClass('Boot Camp', ['Cardio 1'])
        >>> ac.add_workout_class(boot_camp)
        True
        >>> t1 = datetime(2019, 9, 9, 12, 0)
        >>> ac.schedule_workout_class(t1, 'Dance Studio', boot_camp.name, 1)
        True
        >>> t2 = datetime(2019, 9, 10, 12, 0)
        >>> ac.instructor_hours(t1, t2) == {1: 1, 2: 0}
        True
        >>> ac.schedule_workout_class(t2, 'Dance Studio', boot_camp.name, 1)
        True
        >>> ac.instructor_hours(t1, t2) == {1: 2, 2: 0}
        True
        """
        hours_dict = {}
        for _id in self._instructors:
            hours_dict[self._instructors[_id].get_id()] = 0

        if not self._schedule:
            return hours_dict

        for time in self._schedule:
            if time1 <= time <= time2:
                for room_name in self._schedule[time]:
                    ins_id = self._schedule[time][room_name][0].get_id()
                    hours_dict[ins_id] += 1

        return hours_dict

    def payroll(self, time1: datetime, time2: datetime, base_rate: float) \
            -> list[tuple[int, str, int, float]]:
        """Return a sorted list of tuples reporting pay earned by each
        instructor teaching classes that start any time between <time1> and
        <time2>, inclusive. The list should be sorted in ascending order of
        instructor ids.

        Each tuple contains 4 elements, in this order:
        - an instructor's ID,
        - the instructor's name,
        - the number of hours worked by the instructor between <time1> and
          <time2>, and
        - the instructor's total wages earned between <time1> and <time2>.
        The returned list is sorted by instructor ID.

        Both <time1> and <time2> specify the start time for an hour when an
        instructor may have taught.

        Each instructor earns a <base_rate> per hour plus an additional
        BONUS_RATE per hour for each certificate they hold.

        Precondition: time1 <= time2

        >>> ac = Gym('Athletic Centre')
        >>> diane = Instructor(1, 'Diane')
        >>> david = Instructor(2, 'David')
        >>> diane.add_certificate('Cardio 1')
        True
        >>> ac.add_instructor(david)
        True
        >>> ac.add_instructor(diane)
        True
        >>> ac.add_room('Dance Studio', 50)
        True
        >>> boot_camp = WorkoutClass('Boot Camp', ['Cardio 1'])
        >>> ac.add_workout_class(boot_camp)
        True
        >>> t1 = datetime(2019, 9, 9, 12, 0)
        >>> ac.schedule_workout_class(t1, 'Dance Studio', boot_camp.name,
        ... 1)
        True
        >>> t2 = datetime(2019, 9, 10, 12, 0)
        >>> ac.payroll(t1, t2, 25.0)
        [(1, 'Diane', 1, 26.5), (2, 'David', 0, 0.0)]
        """
        payroll_list = []

        if not self._schedule:
            return payroll_list

        for time in self._schedule:
            if time1 <= time <= time2:
                for room_name in self._schedule[time]:
                    ins_id = self._schedule[time][room_name][0].get_id()
                    hrs = self.instructor_hours(time1, time2)[ins_id]
                    ins_name = self._schedule[time][room_name][0].name
                    ins = self._instructors[ins_id]
                    payroll_list.append((ins_id,
                                         ins_name,
                                         hrs,
                                         (base_rate + len(
                                             ins.get_certificates())
                                          * BONUS_RATE)
                                         * hrs
                                         ))

        for _id in self._instructors:
            temp = True
            for item in payroll_list:
                if item[0] == _id:
                    temp = False
            if temp:
                payroll_list.append((_id, self._instructors[_id].name, 0, 0.0))

        payroll_list.sort()

        return payroll_list

    def _is_instructor_name_unique(self, instructor: Instructor) -> bool:
        """Return True iff the name of <instructor> is used by <= 1 instructor
        in the Gym.

        >>> ac = Gym('Athletic Centre')
        >>> first_hire = Instructor(1, 'Diane')
        >>> ac.add_instructor(first_hire)
        True
        >>> ac._is_instructor_name_unique(first_hire)
        True
        >>> second_hire = Instructor(2, 'Diane')
        >>> ac.add_instructor(second_hire)
        True
        >>> ac._is_instructor_name_unique(first_hire)
        False
        >>> ac._is_instructor_name_unique(second_hire)
        False
        >>> third_hire = Instructor(3, 'Tom')
        >>> ac._is_instructor_name_unique(third_hire)
        True
        """
        count = 0
        for _id in self._instructors:
            if self._instructors[_id].name == instructor.name:
                count += 1

        if count <= 1:
            return True

        return False

    def offerings_at(self, time_point: datetime) -> list[dict[str, str | int]]:
        """Return a list of dictionaries, each representing a workout offered
        at this Gym at <time_point>.

        The offerings should be sorted by room name, in alphabetical ascending
        order.

        Each dictionary must have the following keys and values:
            'Date': the weekday and date of the class as a string, in the format
                'Weekday, year-month-day' (e.g., 'Monday, 2022-11-07')
            'Time': the time of the class, in the format 'HH:MM' where
                HH uses 24-hour time (e.g., '15:00')
            'Class': the name of the class
            'Room': the name of the room
            'Registered': the number of people already registered for the class
            'Available': the number of spots still available in the class
            'Instructor': the name of the instructor
                If there are multiple instructors with the same name, the name
                should be followed by the instructor ID in parentheses
                e.g., "Diane (1)"

        If there are no offerings at <time_point>, return an empty list.

        NOTE:
        - You MUST use the helper function create_offering_dict from
          gym_utilities to create the dictionaries, in order to make sure you
          match the format specified above.
        - You MUST use the helper method _is_instructor_name_unique when
          deciding how to format the instructor name.

        >>> ac = Gym('Athletic Centre')
        >>> diane1 = Instructor(1, 'Diane')
        >>> diane1.add_certificate('Cardio 1')
        True
        >>> diane2 = Instructor(2, 'Diane')
        >>> david = Instructor(3, 'David')
        >>> david.add_certificate('Strength Training')
        True
        >>> ac.add_instructor(diane1)
        True
        >>> ac.add_instructor(diane2)
        True
        >>> ac.add_instructor(david)
        True
        >>> ac.add_room('Dance Studio', 50)
        True
        >>> ac.add_room('Room A', 20)
        True
        >>> boot_camp = WorkoutClass('Boot Camp', ['Cardio 1'])
        >>> ac.add_workout_class(boot_camp)
        True
        >>> kickboxing = WorkoutClass('KickBoxing', ['Strength Training'])
        >>> ac.add_workout_class(kickboxing)
        True
        >>> t1 = datetime(2022, 9, 9, 12, 0)
        >>> ac.schedule_workout_class(t1, 'Dance Studio', boot_camp.name, 1)
        True
        >>> ac.schedule_workout_class(t1, 'Room A', kickboxing.name, 3)
        True
        >>> ac.offerings_at(t1) == [
        ... { 'Date': 'Friday, 2022-09-09', 'Time': '12:00',
        ... 'Class': 'Boot Camp', 'Room': 'Dance Studio', 'Registered': 0,
        ... 'Available': 50, 'Instructor': 'Diane (1)' },
        ... { 'Date': 'Friday, 2022-09-09', 'Time': '12:00',
        ... 'Class': 'KickBoxing', 'Room': 'Room A', 'Registered': 0,
        ... 'Available': 20, 'Instructor': 'David' }
        ... ]
        True
        """
        listy = []

        if not self._schedule[time_point]:
            return []

        for room_name in self._schedule[time_point]:
            temp = self._schedule[time_point][room_name][0]
            if self._is_instructor_name_unique(temp):
                ins_name = self._schedule[time_point][room_name][0].name
            else:
                ins_name = self._schedule[time_point][room_name][0].name + \
                    f" ({self._schedule[time_point][room_name][0].get_id()})"

            listy.append(
                create_offering_dict(
                    time_point.strftime('%A, %Y-%m-%d'),
                    time_point.strftime('%H:%M'),
                    self._schedule[time_point][room_name][1].name,
                    room_name,
                    len(self._schedule[time_point][room_name][2]),
                    self._room_capacities[room_name]
                    - len(self._schedule[time_point][room_name][2]),
                    ins_name
                )
            )

        listy.sort(key=lambda x: x['Room'])
        return listy

    def to_schedule_list(self, week: datetime = None) \
            -> list[dict[str, str | int]]:
        """Return a list of dictionaries for the Gym's entire schedule, with
        each dictionary representing a workout offered (in the format specified
        by the docstring for offerings_at).

        The dictionaries should be in the list in ascending order by their date
        and time (not the string representation of the date and time).
        Offerings occurring at exactly the same date and time should
        be in alphabetical order based on their room names.

        If <week> is specified, only return the events that occur between the
        date interval (between a Monday 0:00 and Sunday 23:59) that contains
        <week>.

        Hint: The helper function <in_week> can be used to determine if one
        datetime object is in the same week as another.

        >>> ac = Gym('Athletic Centre')
        >>> diane1 = Instructor(1, 'Diane')
        >>> diane1.add_certificate('Cardio 1')
        True
        >>> diane2 = Instructor(2, 'Diane')
        >>> david = Instructor(3, 'David')
        >>> david.add_certificate('Strength Training')
        True
        >>> ac.add_instructor(diane1)
        True
        >>> ac.add_instructor(diane2)
        True
        >>> ac.add_instructor(david)
        True
        >>> ac.add_room('Studio 1', 20)
        True
        >>> boot_camp = WorkoutClass('Boot Camp', ['Cardio 1'])
        >>> ac.add_workout_class(boot_camp)
        True
        >>> kickboxing = WorkoutClass('KickBoxing', ['Strength Training'])
        >>> ac.add_workout_class(kickboxing)
        True
        >>> t1 = datetime(2022, 9, 9, 12, 0)
        >>> ac.schedule_workout_class(t1, 'Studio 1', boot_camp.name, 1)
        True
        >>> t2 = datetime(2022, 9, 8, 13, 0)
        >>> ac.schedule_workout_class(t2, 'Studio 1', kickboxing.name, 3)
        True
        >>> ac.to_schedule_list() == [
        ... { 'Date': 'Thursday, 2022-09-08', 'Time': '13:00',
        ... 'Class': 'KickBoxing', 'Room': 'Studio 1', 'Registered': 0,
        ... 'Available': 20, 'Instructor': 'David' },
        ... { 'Date': 'Friday, 2022-09-09', 'Time': '12:00',
        ... 'Class': 'Boot Camp', 'Room': 'Studio 1', 'Registered': 0,
        ... 'Available': 20, 'Instructor': 'Diane (1)' },
        ... ]
        True
        """
        listy = []
        if not self._schedule:
            return listy

        schedule_keys = list(self._schedule.keys())
        schedule_keys.sort(reverse=False)

        if week:
            for time in schedule_keys:
                if in_week(time, week):
                    listy = self.__add_o(listy, time)

        else:
            for time in schedule_keys:
                listy = self.__add_o(listy, time)

        return listy

    def __add_o(self, listy: list[dict[str, str | int]], t: datetime) \
            -> list[dict[str, str | int]]:
        """
        Helper function for to_schedule_list, appends all offerings at
        <time> to <listy>.
        """
        if len(self.offerings_at(t)) == 1:
            listy.append(self.offerings_at(t)[0])
            return listy

        for item in self.offerings_at(t):
            listy.append(item)
            return listy

    def __eq__(self, other: Gym) -> bool:
        """Return True iff this Gym is equal to <other>.

        Two gyms are considered equal if they have the same name,
        instructors, workouts, room capacities, and schedule.

        >>> ac = Gym('Athletic Centre')
        >>> ac2 = Gym('Athletic Centre')
        >>> ac == ac2
        True
        """
        return self.name == other.name \
            and self._instructors == other._instructors \
            and self._workouts == other._workouts \
            and self._room_capacities == other._room_capacities \
            and self._schedule == other._schedule

    def to_webpage(self) -> None:
        """
        Deleted this method because I was having an error with pandas on
        my computer. It said I had no module _bz2, I tried to install
        using the command line and pip, and also tried to build
        python again, but it didn't work, so I removed the method entirely,
        along with any other methods that called pandas. The method header
        is still here because there is a test on MarkUs that checks if it is.
        """

if __name__ == '__main__':
    pass


"""Module for the cli argument parser."""
import sys
from typing import List
from configargparse import ArgumentParser, HelpFormatter
from .globals import (
    CONFIG_PATHS,
    ALLOWED_SHOW_VALUES,
    EXT_LOG_DEFAULT,
    EXT_OUT_DEFAULT,
    EXT_ERR_DEFAULT,
    TOLERATED_USAGE,
    BAD_USAGE,
    ARGUMENT_ERROR
)

class CustomFormatter(HelpFormatter):
    """

    Custom formatter for setting argparse formatter_class.

    Identical to the default formatter,
     except that very long option strings are split into two
    lines.

    Solution discussed on: https://bit.ly/32CkCWK
    """

    def _format_action_invocation(self, action):
        """Reformats long argument help descriptions."""
        if not action.option_strings:
            metavar, = self._metavar_formatter(action, action.dest)(1)
            return metavar

        parts = []
        # if the Optional doesn't take a value, format is:
        #    -s, --long
        if action.nargs == 0:
            parts.extend(action.option_strings)
        # if the Optional takes a value, format is:
        #    -s ARGS, --long ARGS
        else:
            default = action.dest.upper()
            args_string = self._format_args(action, default)
            for option_string in action.option_strings:
                # parts.append('%s %s' % (option_string, args_string))
                parts.append(f"{option_string}, {args_string}")

        if sum(len(s) for s in parts) < self._width - (len(parts) - 1) * 2:
            return ', '.join(parts)
        # else
        return ',\n  '.join(parts)

class CLIArgumentParser(ArgumentParser):
    """
    Parser based on configargparse ArgumentParser to be able
    to ignore config files.
    """

    _ignore_configs = None

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._ignore_configs = False

    def set_default_config_paths(self, config_paths):
        """Set default config paths."""
        self._default_config_files = config_paths

    def ignore_configs(self):
        """Mainly for testing, ignore configs every time"""
        self._ignore_configs = True

    def get_params(self, args: List = None):
        """Get params from args."""
        if args is None:
            args = []
        # add config paths if help wanted
        if "-h" in args or "--help" in args:
            self.set_default_config_paths(CONFIG_PATHS)
            self.parse_args(["-h"])
            # exit

        # else check if --ignore-config is set
        prio_args = self.parse_args(args)
        if not prio_args.ignore_config and not self._ignore_configs:
            # if not set add config paths
            self.set_default_config_paths(CONFIG_PATHS)

        return self.parse_args(args)

    def error(self, message: str):
        self.print_usage(sys.stderr)
        self.exit(ARGUMENT_ERROR, f"{self.prog}: error: {message}\n")

def setup_parser() -> CLIArgumentParser:
    """
    Define parser with all arguments listed below.

    :return: parser
    """

    parser = CLIArgumentParser(
        formatter_class=CustomFormatter,
        # Todo: Change when fixed:
        #       https://github.com/bw2/ConfigArgParse/issues/217
        allow_abbrev=False,
        description="Analyze or summarize HTCondor-Joblogs",
    )
    parser.add_argument(
        "paths",
        nargs="*",
        help="Directory of file paths for log files"
    )
    parser.add_argument(
        "-r", "--recursive",
        action="store_true",
        default=False,
        help="Recursive search through directory hierarchy"
    )
    parser.add_argument(
        "--version",
        help="Get the current version of this script",
        action="store_true"
    )
    parser.add_argument(
        "-v", "--verbose",
        help="Print out extended execution details",
        action="store_true",
        default=False
    )
    parser.add_argument(
        "--analyze",
        action="store_true",
        default=False,
        help="Analyze given files one by one"
    )
    parser.add_argument(
        "--ext-log",
        help="Suffix of HTCondor job logs (default: none)",
        default=EXT_LOG_DEFAULT
    )
    parser.add_argument(
        "--ext-out",
        help="Suffix of job out logs (default: .out)",
        default=EXT_OUT_DEFAULT
    )
    parser.add_argument(
        "--ext-err",
        help="Suffix of job error logs (default: .err)",
        default=EXT_ERR_DEFAULT
    )
    allowed_show_vals = ALLOWED_SHOW_VALUES[:]  # copying
    allowed_show_vals.append('')  # needed so empty list are valid in config
    parser.add_argument(
        "--show",
        nargs="+",
        default=[],
        dest="show_list",
        choices=allowed_show_vals,
        help="Show more details"
    )
    parser.add_argument(
        "--rdns-lookup",
        action="store_true",
        default=False,
        help="Resolve the ip-address of an execution nodes"
             " to their dns entry"
    )
    parser.add_argument(
        "--tolerated-usage",
        type=float,
        help="Threshold to warn the user, "
             "when a given percentage is exceeded "
             "between used and requested resources",
        default=TOLERATED_USAGE
    )
    parser.add_argument(
        "--bad-usage",
        type=float,
        help="Threshold to signal overuse/waste of resources, "
             "when a given percentage is exceeded "
             "between used and requested resources",
        default=BAD_USAGE
    )

    action = parser.add_mutually_exclusive_group()
    action.add_argument(
        "-c", "--config",
        is_config_file=True,
        help="Path to config file"
    )
    action.add_argument(
        "--ignore-config",
        action="store_true",
        help="Do not search for config"
    )

    return parser


import logging
import json

from django.core.management.base import BaseCommand

from ...models import Category, Tag

logger = logging.getLogger('library')

INDUSTRIES_CATEGORY = 'Industries'
EXO_ATTRIBUTES_CATEGORY = 'ExO Attributes'
TECH_CATEGORY = 'Technologies'

INDUSTRIES_LIST = [
    'Accommodations', 'Accounting', 'Advertising', 'Aerospace',
    'Agriculture & Agribusiness', 'Air Transportation', 'Aircraft',
    'Alcohol', 'Apparel & Accessories', 'Auto', 'Aviation', 'Banking',
    'Beauty & Cosmetics', 'Big Data', 'Biotechnology', 'Biotechnology',
    'Chemical', 'Communications', 'Computer', 'Construction',
    'Consulting', 'Consumer Products', 'Cosmetic', 'Cybersecurity',
    'Diamond',
    'Economy', 'Education', 'Electronics', 'Employment', 'Energy', 'Ethics',
    'Entertainment & Recreation',
    'Fashion', 'Financial Services', 'Food & Beverage', 'Future',
    'Glass', 'Health',
    'Hospitality',
    'Information',
    'Insurance', 'Inspiration', 'Internet of Things', 'Investment and Trading',
    'Journalism & News',
    'Legal Services', 'Life Sciences', 'Manufacturing',
    'Media & Broadcasting', 'Medical', 'Metal', 'Military',
    'Motion Pictures & Video', 'Music', 'Nanotechnology',
    'Neuroscience', 'Nuclear', 'Packaging', 'Paint & Coatings',
    'Oil and Gas',
    'Petrochemicals', 'Pharmaceutical', 'Plastics', 'Privacy', 'Private Spaceflight',
    'Publishing', 'Pulp & Paper', 'Rail', 'Real Estate', 'Recycling',
    'Retail', 'Robotics', 'Security', 'Service', 'Shipping', 'Shipyards',
    'Society', 'Solar', 'Space', 'Space-based Economy', 'Specialty Drugs',
    'Sporting Goods', 'Sports', 'Steel', 'Sustainability',
    'Telecommunications', 'Television', 'Textile', 'Tire', 'Tobacco', 'Toy', 'Transport',
    'Utilities',
    'Video Game',
    'Waste', 'Web Services',
]

EXO_ATTRIBUTES_LIST = [
    "Staff on Demand", "Community & Crowd", "Algorithms",
    "Leveraged Assets", "Engagement", "Interfaces", "Dashboards",
    "Experimentation", "Autonomy", "Social Technologies", 'MTP'
]

DEFAULT_FILTERS = [
    'Consulting', 'Education', 'Computer', 'Communications',
    'Blockchain', 'Internet of Things', 'Artificial general intelligence',
    'Cryptocurrency', 'Virtual Reality'
]

class Command(BaseCommand):
    def handle(self, *args, **kwargs):
        TECHNOLOGIES = []
        with open('resource/management/commands/technologies.json') as json_data:
            techs = json.load(json_data)

        for tech in techs:
            TECHNOLOGIES.append(tech.get("title"))

        CATEGORIES = [{
            'name': EXO_ATTRIBUTES_CATEGORY,
            'values': EXO_ATTRIBUTES_LIST
        }, {
            'name': INDUSTRIES_CATEGORY,
            'values': INDUSTRIES_LIST
        }, {
            'name': TECH_CATEGORY,
            'values': TECHNOLOGIES
        }]

        for category in CATEGORIES:
            category_name = category.get("name")
            cat, _ = Category.objects.get_or_create(name=category_name)
            Tag.objects.get_or_create(name=category.get("name"))
            default_show_filter = False

            for tag in category.get("values"):
                default_show_filter = False
                if category_name == EXO_ATTRIBUTES_CATEGORY or tag in DEFAULT_FILTERS:
                    default_show_filter = True

                defaults = {
                    'category': cat,
                    'default_show_filter': default_show_filter
                }
                Tag.objects.update_or_create(name=tag, defaults=defaults)


def fibonacci(sequencia=None):
    # Se sequencia=None -> sequencia = [0, 1]
    sequencia = sequencia or [0, 1]
    sequencia.append(sequencia[-1] + sequencia[-2])

    return sequencia

if __name__ == "__main__":
    inicio = fibonacci()
    print(inicio, id(inicio))
    print(fibonacci(inicio))
    restart = fibonacci() # Não cria uma nova lista
    print(restart, id(restart))
    # inicio e restart tem o mesmo id - são o mesmo objeto em memória
    assert restart == [0, 1, 1]

Python 3 - Curso Completo do Básico ao Avançado/Seção 11 - Funções/128. Correção Param. Padrão Mutável.py


# while True:
# import random
# # # def freaken_game():
# num1 = random.randrange(100)
# num2 = random.randrange(100)
# sum2 = random.randrange(100)
# sum1 = num1 + num2
# print(num1,"+",num2,"=",sum2)

# userinput = input("Answer the question : ")
# point=0
# if userinput == sum1:
#     num1 = random.randrange(100)
#     num2 = random.randrange(100)
# print(num1)
# print(num2)
# print(sum1)
while True:
    def addition():
        num1 = random.randint(-100,100)
        num2 = random.randint(-100,100)
        num3 = random.randint(-1,1)
        sum1 = num1 + num2
        sum2 = sum1 + num3
        # print(num1)
        # print(num2)
        # print(sum2)

    import random
    num1 = random.randint(-100,100)
    num2 = random.randint(-100,100)
    num3 = random.randint(-1,1)
    sum1 = num1 + num2
    sum2 = sum1 + num3
    print("O for true; X for false")
    print(num1,"+",num2,"=",sum2)
    userinput= input("Your answer: ")
    if userinput == "O":
        if sum1 == sum2:
             addition()
             print("True")
             print()
        elif sum1 != sum2:
             print("Game Over")
             break
    elif userinput == "X":
        if sum1 != sum2:
             addition()
             print("True")
             print()
        elif sum1 == sum2:
             print("Game Over")
             break

    def Subtraction():
        num1 = random.randint(-100,100)
        num2 = random.randint(-100,100)
        num3 = random.randint(-1,1)
        sub1 = num1 - num2
        sub2 = sub1 + num3
        # print(num1)
        # print(num2)
        # print(sub2)
        # print(num1,"-",num2,"=",sub2)

    def Multiplication():
        num1 = random.randint(-50,50)
        num2 = random.randint(-50,50)
        num3 = random.randint(-1,1)
        mult1 = num1 * num2
        mult2 = mult1 + num3
        print(num1)
        print(num2)
        print(mult2)
        print(num1,"x",num2,"=",mult2)

    def chia():
        num1 = random.randint(-50,50)
        num2 = random.randint(-50,50)
        num3 = random.randint(-1,1)
        chia1 = num1 / num2
        chia2 = chia1 + num3
        print(num1)
        print(num2)
        print(chia2)
        print(num1,":",num2,"=",chia2)

    # addition()
    # print()
    # Subtraction()
    # print()
    # Multiplication()
    # print()
    # chia()


from setuptools import setup

PACKAGES = [
    'ladder_network',
    'ladder_network.ops',
]

def setup_package():
    setup(
        name="LadderNetwork",
        version='0.1.0',
        description="TensorFlow implementation of Rasmus et. al's Ladder Network",
        author='Joshua D. Loyal',
        url='https://github.com/joshloyal/LadderNetwork',
        license='MIT',
        install_requires=['numpy', 'tensorflow'],
        packages=PACKAGES,
    )

if __name__ == '__main__':
    setup_package()


import requests
import json
import urllib
from requests.exceptions import HTTPError

def getContact(email, apikey):
    contact = {}
    try:
        #print("- Downloading Contact from Hubspot API...")
        url= 'https://api.hubapi.com/contacts/v1/contact/email/' + email + '/profile?hapikey=' + apikey
        response = requests.get(url)
        response.raise_for_status()
        jsonResponse = response.json()
        #print(jsonResponse['properties'])
        if 'phone' in jsonResponse['properties']:
            contact['phone'] = jsonResponse['properties']['phone']['value']
        else:
            contact['phone'] = ""
        if 'firstname' in jsonResponse['properties']:
            contact['name'] = jsonResponse['properties']['firstname']['value']
        else:
            contact['name'] = ""
        if 'lastname' in jsonResponse['properties']:
            contact['lastname'] = jsonResponse['properties']['lastname']['value']
        else:
            contact['lastname'] = ""

    except HTTPError as http_err:
        if response.status_code == 404:
            print("------ ERROR: contact " + email + " might not exist. Maybe it was deleted ------")
        else:
            print(f"--- HTTP error occurred: {response}")
        contact['phone'] = 'error'
        contact['name'] = 'error'
        contact['lastname'] = 'error'
        return contact
    except Exception as err:
        print(f'------- Other error occurred: {err}')
        contact['phone'] = 'error'
        contact['name'] = 'error'
        contact['lastname'] = 'error'

    return contact

def processEvent(e, statusDict):
    #print(e)
    email = e['recipient']
    #print ("Event read for: " + email)

    if email in statusDict.keys():
        #print("- already in statusDict.keys")
        #  IF CURRENT EVENT TYPE IS  FINAL  OR  EQUAL TO CURRENT STATE, DO NOTHING.  ELSE, SAVE IT IF IT'S A MORE RECENT TYPE IN THE WORKFLOW
        if statusDict[email] == 'DROPPED' or statusDict[email] == 'BOUNCE' or statusDict[email] == 'SPAMREPORT' or statusDict[email] == e['type']:
            #print("--- already " + e['type'] + ". Nothing to do.")
            pass
        else:
            if e['type'] == 'CLICK':
                statusDict[email] = e['type']
            elif e['type'] == 'OPEN' and statusDict[email] not in ['CLICK']:
                statusDict[email] = e['type']
            elif e['type'] == 'DELIVERED' and statusDict[email] not in ['CLICK', 'OPEN']:
                statusDict[email] = e['type']
            elif e['type'] == 'PROCESSED' and statusDict[email] not in ['CLICK', 'OPEN', 'DELIVERED']:
                statusDict[email] = e['type']
            elif e['type'] == 'SENT' and statusDict[email] not in ['CLICK', 'OPEN', 'DELIVERED', 'PROCESSED']:
                statusDict[email] = e['type']
            elif e['type'] == 'STATUSCHANGE' or e['type'] == 'DEFERRED':
                statusDict[email] = e['type']
    else:
        #  FIRST TYPE STATE IS THE FIRST RECEIVED
        #print("- NOT in statusDict.keys, adding contact and setting to " + e['type'])
        statusDict[email] = e['type']

def main():

    apikey = input('What\'s is your API Key? ')
    campaignId = input('What\'s the campaignId? ')
    statusDict =	{}

    try:
        print("- Downloading Events from Hubspot API for Campaign " + campaignId + " ...")
        url = 'https://api.hubapi.com/email/public/v1/events?hapikey=' + apikey + '&campaignId=' + campaignId
        #print( ">>> GET: " , url)
        response = requests.get(url)
        response.raise_for_status()
        jsonResponse = response.json()
        for e in jsonResponse['events']:
            processEvent(e, statusDict)

        hasMore = jsonResponse['hasMore']

        while hasMore:
            url = 'https://api.hubapi.com/email/public/v1/events?hapikey=' + apikey + '&campaignId=' + campaignId + '&offset=' + jsonResponse['offset']
            #print( ">>> GET: " , url)
            response = requests.get(url)
            response.raise_for_status()
            jsonResponse = response.json()
            for e in jsonResponse['events']:
                processEvent(e, statusDict)
            #print("<<<< Response: ")
            #print(jsonResponse)
            hasMore = jsonResponse['hasMore']

        print("")
        print("- All events read from API. Saving CSV with each Contact email,status for this campaign")
        print("")
        file_object = open('contactsCampaign.csv', 'w')
        for email in statusDict:
            contact = getContact(email, apikey)
            print (email + "," + contact['phone'] + "," + contact['name'] + "," + contact['lastname'] + "," + statusDict[email])
            file_object.write(email + "," + contact['phone'] + "," + contact['name'] + "," + contact['lastname'] + "," + statusDict[email] + "\n")
        file_object.close()

    except HTTPError as http_err:
        print(f'HTTP error occurred: {http_err}')
    except Exception as err:
        print(f'Other error occurred: {err}')

if __name__ == '__main__':
    main()


import torch
import torch.nn as nn
import torch.optim as optim
import os

from omegaconf import OmegaConf, DictConfig

def save_model(model, path=None):
    if path:
        new_path = os.path.join(model.save_path,path)
        model_file_path = os.path.join(
                new_path,
                model.name)
    else:
        new_path = model.save_path
        model_file_path = os.path.join(model.save_path, model.name)

    if not os.path.exists(new_path):
        os.makedirs(new_path)
    torch.save(model.get_weights(), model_file_path)

def load_model(model, save_path, checkpoint):
    model_file_path = os.path.join(save_path,checkpoint, model.name)
    model.load_state_dict(torch.load(model_file_path))

class Actor(nn.Module):
    '''
    Actor network for DDPG
    '''
    def __init__(self, config: DictConfig, target=False, worker=False):
        super().__init__()
        self.config = config
        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
        if self.config.running_on == 'M1':
            self.device = 'mps' if torch.backends.mps.is_available() else 'cpu'
        if worker:
            self.device = 'cpu'

        self.name = 'actor'
        if target:
            self.name = 'target_actor'
        self.save_path = config.agent.save_path
        self.learning_rate = config.agent.actor_lr
        self.state_dimension = config.env.state_dimension
        self.action_dimension = config.env.action_dimension

        self.linear_block = nn.Sequential(
                nn.Linear(self.state_dimension, 400),
                #nn.BatchNorm1d(400),
                nn.LayerNorm(400),
                nn.ReLU(),
                nn.Linear(400, 300),
                #nn.BatchNorm1d(300),
                nn.LayerNorm(300),
                nn.ReLU(),
                nn.Linear(300, self.action_dimension),
                nn.Tanh()
                )
        self.optimizer = optim.Adam(
                self.parameters(), lr=self.learning_rate
                )
        self.to(self.device)

    def forward(self, x):
        x = self.linear_block(x)
        return x

    def get_weights(self):
        return {k: v.cpu() for k,v in self.state_dict().items()}

    def set_weights(self, weights):
        self.load_state_dict(weights)

    def save_model(self, path=None):
        save_model(self, path)

    def load_model(self, path, checkpoint):
        load_model(self, path, checkpoint)

class Critic(nn.Module):
    '''
    Critic network for DDPG
    '''
    def __init__(self, config: DictConfig, target=False, worker=False):
        super().__init__()
        self.config = config
        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
        if self.config.running_on == 'M1':
            self.device = 'mps' if torch.backends.mps.is_available() else 'cpu'
        if worker:
            self.device = 'cpu'

        self.name = 'critic'
        if target:
            self.name = 'target_critic'
        self.save_path = config.agent.save_path
        self.learning_rate = config.agent.critic_lr
        self.state_dimension = config.env.state_dimension
        self.action_dimension = config.env.action_dimension

        self.batch_size = self.config.memory.batch_size

        self.linear_block = nn.Sequential(
                nn.Linear(self.state_dimension + self.action_dimension, 400),
                #nn.BatchNorm1d(400),
                nn.LayerNorm(400),
                nn.ReLU(),
                nn.Linear(400, 300),
                #nn.BatchNorm1d(300),
                nn.LayerNorm(300),
                nn.ReLU(),
                nn.Linear(300, 1),
                )
        self.optimizer = optim.Adam(
                self.parameters(), lr=self.learning_rate
                )
        self.to(self.device)

    def forward(self, x,y):
        if not(len(x.shape) == 1):
            q = self.linear_block(torch.cat([x,y],1))
            return q
        else:
            q = self.linear_block(torch.cat([x,y],0))
            return q

    def get_weights(self):
        return {k: v.cpu() for k,v in self.state_dict().items()}

    def set_weights(self, weights):
        self.load_state_dict(weights)

    def save_model(self,path=None):
        save_model(self, path)

    def load_model(self, path, checkpoint):
        load_model(self, path, checkpoint)

# Adapted from:
# https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On-Second-Edition/blob/master/Chapter17/lib/model.py
class D4PGCritic(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.obs_size = self.config.env.state_dimension
        self.act_size = self.config.env.action_dimension
        self.n_atoms = self.config.agent.n_atoms
        self.v_min = self.config.agent.v_min
        self.v_max = self.config.agent.v_max

        self.obs_net = nn.Sequential(
            nn.Linear(self.obs_size, 400),
            nn.ReLU(),
        )

        self.out_net = nn.Sequential(
            nn.Linear(400 + self.act_size, 300),
            nn.ReLU(),
            nn.Linear(300, self.n_atoms)
        )

        delta = (self.v_max - self.v_min) / (self.n_atoms - 1)
        self.register_buffer("supports", torch.arange(
            self.v_min, self.v_max + self.delta, self.delta))

    def forward(self, x, a):
        obs = self.obs_net(x)
        return self.out_net(torch.cat([obs, a], dim=1))

    def distr_to_q(self, distr):
        weights = F.softmax(distr, dim=1) * self.supports
        res = weights.sum(dim=1)
        return res.unsqueeze(dim=-1)


#!/usr/bin/env python
# coding: utf-8
import cv2
import numpy as np
import glob
import os

rows = 6
cols = 9

objp = np.zeros((rows * cols, 3), np.float32)
objp[:, :2] = np.mgrid[0:rows, 0:cols].T.reshape(-1, 2)

# define the path
path = os.getcwd() + '/images/task_1/'
output_path = os.getcwd() + '/output/task_1/'
# 3d point in real world space
obj_left = []
# 2d points in image plane.
img_l = []

images = glob.glob(os.path.join(path, "left_*.png"))
for img_name in images:
    img_color = cv2.imread(img_name)
    img = cv2.cvtColor(img_color, cv2.COLOR_BGR2GRAY)
    ret, corners = cv2.findChessboardCorners(img, (rows, cols), None)
    obj_left.append(objp)
    img_l.append(corners)
    if ret == True:
        corners2 = cv2.cornerSubPix(img, corners, (11,11), (-1,-1), (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_COUNT, 30, 0.01))
        cv2.drawChessboardCorners(img_color, (rows, cols), corners2, ret)
        cv2.imshow('img', img_color)
        cv2.imwrite(output_path + '/corners.png', img_color)

#Calibration
ret, mtx_left, dist, emp, emp = cv2.calibrateCamera(obj_left, img_l, img.shape[::-1], None, None)

# Undistortion for left_2 png
img = cv2.imread(path  + "left_2.png")

h, w = img.shape[:2]
new_mtx_left_matrix, roi = cv2.getOptimalNewCameraMatrix(mtx_left, dist, (w,h), 0)

mapx, mapy = cv2.initUndistortRectifyMap(mtx_left, dist, None, new_mtx_left_matrix, (w,h), 5)
dst = cv2.remap(img, mapx, mapy, cv2.INTER_LINEAR)
x, y, w, h = roi
dst = dst[y:y+h, x:x+w]

if dst[0].size > 0:
    cv2.imwrite(output_path + '/left_2_undistorted.png', dst)
    undistorted_image = cv2.imread(os.path.join(output_path + '/left_2_undistorted.png'))
    cv2.imshow('undistorted image', undistorted_image)

param_path = os.getcwd() + '/parameters'

s = cv2.FileStorage('{0}/left_camera_intrinsics.xml'.format(param_path), cv2.FileStorage_WRITE)
s.write('mtx_left', mtx_left)
s.write('distCoeffs_left', dist)
s.release()

# RIght image

rows = 9
cols = 6

objp = np.zeros((rows * cols, 3), np.float32)
objp[:, :2] = np.mgrid[0:rows, 0:cols].T.reshape(-1, 2)

obj_right = [] # 3d point in real world space
img_r = [] # 2d points in image plane.

images = glob.glob(os.path.join(path, "right_*.png"))

for img_name in images:
    img_color = cv2.imread(img_name)
    img = cv2.cvtColor(img_color, cv2.COLOR_BGR2GRAY)

    ret, corners = cv2.findChessboardCorners(img, (rows, cols), None)
    obj_right.append(objp)
    img_r.append(corners)
    if ret == True:
        corners2 = cv2.cornerSubPix(img, corners, (5,5), (-1,-1), (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_COUNT, 30, 0.01))
        cv2.drawChessboardCorners(img_color, (rows, cols), corners2, ret)
        cv2.imshow('img', img_color)

#Calibration

ret, mtx_right, dist_right, emp, emp = cv2.calibrateCamera(obj_right, img_r, img.shape[::-1], None, None)

img = cv2.imread(path + "right_2.png")

h, w = img.shape[:2]

new_mtx_right_matrix, roi = cv2.getOptimalNewCameraMatrix(mtx_right, dist_right, (w,h), 0)

mapx, mapy = cv2.initUndistortRectifyMap(mtx_right, dist, None, new_mtx_right_matrix, (w,h), 5)
dst = cv2.remap(img, mapx, mapy, cv2.INTER_LINEAR)
x, y, w, h = roi
dst = dst[y:y+h, x:x+w]
if dst[0].size > 0:
    cv2.imwrite(output_path + '/right_2_undistorted.png', dst)
    undistorted_image = cv2.imread(output_path + '/right_2_undistorted.png')
    cv2.imshow('undistorted image', undistorted_image)

s = cv2.FileStorage('{0}/right_camera_intrinsics.xml'.format(param_path), cv2.FileStorage_WRITE)
s.write('mtx_right', mtx_right)
s.write('distCoeffs_right', dist_right)
s.release()


import struct
import socket
from prettytable import PrettyTable
class IPv4(object):
    def __init__(self):
        self.version = None     # 4bit version
        self.headerlen = None      #4bit header length
        self.tos = None         # 8bit type of service
        self.totalLen = None    # 16bit total length
        self.identification = None  # 16bit header identification
        self.fragment = None        # 16bit others and fragment offset
        self.ttl = None             # 8bit time to live
        self.protocol = None        # 8bit type of protocol
        self.checksum = None    # 16bit header checksum
        self.srcIP = None       # 32bit src IP address
        self.dstIP = None       # 32bit dst IP address
    def decodeIP(self,buffer):
        (versionAndLen,self.tos,self.totalLen,self.identification,self.fragment,
         self.ttl,self.protocol,self.checksum,self.srcIP,self.dstIP) = struct.unpack('>cBHHHBBHII',buffer)
        self.version = str(versionAndLen.hex())[0]
        self.headerlen = str(versionAndLen.hex())[1]
        self.dstIP = socket.inet_ntoa(struct.pack('I',socket.ntohl(self.dstIP)))
        self.srcIP = socket.inet_ntoa(struct.pack('I',socket.ntohl(self.srcIP)))
    def info(self):
        row = PrettyTable()
        row.add_row(['version', self.version])
        row.add_row(['header length', self.headerlen])
        row.add_row(['type of service', self.tos])
        row.add_row(['total length', self.totalLen])
        row.add_row(['header identification', self.identification])
        row.add_row(['others and fragment offset', self.fragment])
        row.add_row(['time to live', self.ttl])
        row.add_row(['type of protocol', self.protocol])
        row.add_row(['header checksum', self.checksum])
        row.add_row(['src IP address', self.srcIP])
        row.add_row(['dst IP address', self.dstIP])
        return row
    def data(self):
        data = []
        data.append(self.version)
        data.append(self.headerlen)
        data.append(self.tos)
        data.append(self.totalLen)
        data.append(self.identification)
        data.append(self.fragment)
        data.append(self.ttl)
        data.append(self.protocol)
        data.append(self.checksum)
        data.append(self.srcIP)
        data.append(self.dstIP)
        return data


import os
import sys
import json

from applnlayer.ApplnMessageTypes import ResponseMessage

def serialization(rm):

    json_buf={
        "code":rm.code,
        "contents":rm.contents
        }
    return json.dumps(json_buf)

def deserialization (buf):

    json_buf = json.loads(buf)

    rm=ResponseMessage(0,None)
    rm.code=json_buf["code"]
    rm.contents=json_buf["contents"]

    return rm


from selenium import webdriver
import csv
def Skill():
    driver = webdriver.Chrome()
    driver.maximize_window()
    driver.implicitly_wait(120)
    url="https://www.jobs.af/"
    driver.get(url)

    href=[]
    skill_description=[]
    elm=driver.find_elements_by_xpath('//div[@class="item-header"]//h2//a')
    for i in elm:
        try:
            href.append(i.get_attribute('href'))
        except Exception as e:
            print(e)

    for i in href:

        driver.get(i)

        try:
            t=driver.find_elements_by_xpath('//*[./preceding-sibling::h3="Skills Description:"]//p')

        except:
            t=''

        for i in t:
            skill_description.append(i.text)

    with open('submission.csv','w') as f:
        f.write("Url , Title \n")
    with open('submission.csv', 'a') as data_file:

        for i in range(0,12):

            data_file.write(str(href[i])+ "," + str(skill_description[i]) + "\n")

    driver.close()
if __name__ == '__main__':
    Skill()


from flask import Flask, request
import sqlite3
import pickle
import numpy as np

app = Flask(__name__)
app.config["Debug"] = True

@app.route("/", methods = ["GET"])
def entrada():
    return("¡Bienvenido a la página web de Antonio!")

@app.route("/predict", methods=['GET'])
def prediction():
    tv = request.args.get("tv", None)
    radio = request.args.get("radio", None)
    newspaper = request.args.get("newspaper", None)
    with open("advertising_model", "rb") as f:
        modelo = pickle.load(f)
    prediccion = modelo.predict([[tv, radio, newspaper]])
    return str(prediccion)

@app.route("/ingest_data", methods=['GET'])
def agregar():
    tv = request.args.get("tv", None)
    radio = request.args.get("radio", None)
    newspaper = request.args.get("newspaper", None)
    sales = request.args.get("sales", None)
    connection = sqlite3.connect("Advertising.db")
    cursor = connection.cursor()
    cursor.execute(str("INSERT INTO Advertising(tv, radio, newspaper, sales) VALUES(" + tv + "," + radio + "," + newspaper + "," + sales + ")"))
    num = str(len(cursor.execute("select * from advertising").fetchall()))
    connection.commit()
    connection.close()
    return str("el número de registros de la base de datos ahora es: " + num)

@app.route("/retrain", methods=['GET'])
def reentrenar():
    connection = sqlite3.connect("Advertising.db")
    cursor = connection.cursor()
    with open("advertising_model", "rb") as f:
        modelo = pickle.load(f)
    x = np.array(cursor.execute("select * from advertising").fetchall())[:, :-1]
    y = np.array(cursor.execute("select * from advertising").fetchall())[:, 3]
    modelo.fit(x,y)
    connection.close()
    return "El modelo ha sido reentrenado con los valores añadidos"


from socket import *
import sys
import select
import os

host= "192.168.0.76"
port = 9999
s = socket(AF_INET, SOCK_DGRAM)
s.bind((host,port))

addr = (host,port)
buf = 1024

count=0
data,addr = s.recvfrom(buf)
#how much recv -> buf
print ("file recv start from",host)

file_name = data.decode()

print("File Name : ", file_name)
data,addr = s.recvfrom(buf)
f = open(file_name, 'wb')#file create

file_size = data.decode()
print("File Size : ", file_size)

data,addr = s.recvfrom(buf)

try:
    while(data):
        count = count + len(data.decode())
        print("current_size / total_size =", count,"/",int(file_size),",",count/int(file_size)*100,"%")
        f.write(data)
        s.settimeout(2)
        data,addr = s.recvfrom(buf)
except timeout:
    f.close()
    s.close()
    print("File Downloaded")