# -*- coding: utf-8 -*-
from telebot import types
import sqlite3 as sq
from datetime import date
import time
import datetime
import requests
from bs4 import BeautifulSoup as Bs
from telebot import TeleBot
from fake_useragent import UserAgent
import random
from threading import Thread
from config import *
def get_proxy(proxy_list):
    proxy = random.choice(proxy_list)
    proxies = {"http": f'http://{proxy}', "https": f'http://{proxy}'}
    return proxies
def get_useragent():
    my_browser = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
    try:
        ua = UserAgent(fallback=my_browser)
        my_agent = ua.random

        return my_agent

    except Exception as e:
        print(e)
        return my_browser

headers = {'User-Agent': get_useragent()}

# Фильтрация обьявлений на от агентство и от частника
def filter_AN(link_user):
    r = requests.get(link_user, headers=headers,proxies=get_proxy(proxy_list), timeout=5)
    soup = Bs(r.text, features="html.parser")
    ads = (soup.findAll('a', class_='adTile-mainInfo'))

    n = 0
    for i in ads:
        i = i.get("href")

        if 'komnat' in str(i):
            n = n + 1

    print(n)
    if int(n) < 3:
        return None

    else:
        return True

# Добавление в базу новых обьявлений
def base_add(rooms, series, ploshad, remont, etaj, etaj_iz, rayon, prodaves, price, add_link, poslednyi_et, AN,
             photo_links,data_soz,data_prod):
    with sq.connect('version_1/lalago.db') as con:
        cur = con.cursor()
    cur.execute(
        f"INSERT INTO base (rooms,series,ploshad,remont,etaj,etaj_iz,rayon,prodaves, price, an, add_link, poslednyi_et, photo_links,data_soz, data_prod) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?,?)",
        ( int(rooms), str(series), int(ploshad), str(remont), int(etaj), int(etaj_iz),str(rayon),str(prodaves), price, str(AN), str(add_link), str(poslednyi_et), str(photo_links) ,str(data_soz), str(data_prod)
        ))
    con.commit()

# Сравнение квартиры с похожимы квартирами
def osenka_kv(series, remont, rooms, rayon, etaj, etaj_iz, poslednyi_et):
    with sq.connect('../base_3.db') as con:
        cur = con.cursor()
    try:

        if int(etaj) == 1:
            info = (cur.execute(
                f"SELECT sum(ploshad), sum(price) FROM base WHERE rooms ='{rooms}' AND series = '{series}' AND remont = '{remont}' AND rayon = '{rayon}' AND price != '1'  GROUP BY rooms,series")).fetchone()
            price_srednyi = (int(info[1]) / int(info[0])) * 0.96

            return price_srednyi
        elif int(etaj) == int(etaj_iz):
            info = (cur.execute(
                f"SELECT sum(ploshad), sum(price) FROM base WHERE rooms ='{rooms}' AND series = '{series}' AND remont = '{remont}' AND rayon = '{rayon}' AND price != '1'  GROUP BY rooms,series")).fetchone()
            price_srednyi = (int(info[1]) / int(info[0])) * 0.96

            return price_srednyi
        else:
            info = (cur.execute(
                f"SELECT sum(ploshad), sum(price) FROM base WHERE rooms ='{rooms}' AND series = '{series}' AND remont = '{remont}' AND rayon = '{rayon}' AND price != '1'  GROUP BY rooms,series")).fetchone()
            price_srednyi = int(info[1]) / int(info[0])

            return price_srednyi

    except:
        pass

def Starter_check_lalafo_kg():
    while True:
        try:
            check_lalafo_kg()
            time.sleep(600)
        except:
            print('exept parser_kvartiry')
            time.sleep(1800)
            continue

def check_lalafo_kg():
    rayon_list = {
        '3 мкр': ['3 мкр', '3мкр', '3мик', '3-ми', '3-мкр', '3 мик'],
        '4 мкр': ['4 мкр', '4мкр', '4мик', '4-ми', '4-мкр', '4 мик'],
        '5 мкр': ['5 мкр', '5мкр', '5мик', '5-ми', '5-мкр', '5 мик'],
        '6 мкр': ['6 мкр', '6мкр', '6мик', '6-ми', '6-мкр', '6 мик'],
        '7 мкр': ['7 мкр', '7мкр', '7мик', '7-ми', '7-мкр', '7 мик'],
        '8 мкр': ['8 мкр', '8мкр', '8мик', '8-ми', '8-мкр', '8 мик'],
        '9 мкр': ['9 мкр', '9мкр', '9мик', '9-ми', '9-мкр', '9 мик'],
        '10 мкр': ['10 мкр', '10мкр', '10мик', '10-ми', '10-мкр', '10 мик'],
        '11 мкр': ['11 мкр', '11мкр', '11мик', '11-ми', '11-мкр', '11 мик'],
        '12 мкр': ['12 мкр', '12мкр', '12мик', '12-ми', '12-мкр', '12 мик'],
        '110 квартал': ['110 квартал', '110-квартал'],
        '1000 мелочей (Карпинка)': ['карпин', '1000 мелоч', '1000-мелоч'],
        'Азамат авторынок': ['Азамат авторынок', 'Азамат авторынок', 'авторынок'],
        'Азия Молл': ['азия мол', 'азия-мол', 'Азия Молл'],
        'Ак-Бата ж/м': ['ак-бата', 'ак бата'],
        'Ак-Босого ж/м': ['ак-босого', 'ак босого'],
        'Ак-Жар ж/м': ['ак-жар', 'ак жар'],
        'Ак-Кеме (старый аэропорт)': ['ак-кеме', 'ак кеме', 'старый аэропорт',
                                      'старого аэропор', 'старый аеропорт',
                                      'старого аеропор'],
        'Ак-Орго ж/м': ['ак-орго', 'ак орго'],
        'Ак-Ордо 1 ж/м': ['ак-ордо 1', 'ак ордо 1', 'акордо 1'],
        'Ак-Ордо 2 ж/м': ['ак-ордо 1', 'ак ордо 2', 'акордо 2'],
        'Ак-Ордо 3 ж/м': ['ак-ордо 3', 'ак ордо 3', 'акордо 3'],
        'Ак-Ордо ж/м': ['ак-ордо', 'ак ордо', 'ак ордо', 'ак-ордо', 'акордо'],
        'Ак-Тилек ж/м': ['ак-тилек', 'актилек', 'ак тилек'],
        'Ала-Арча ТРЦ': ['ала-арча', 'ала арча'],
        'Ала-Арча ж/м': ['ала-арча ж/м', 'ж/м ала-арча', 'ж/м ала арча', 'ала арча ж/м'],
        'Ала-Тоо ж/м': ['Ала-Тоо ж/м', 'ив Ала-Тоо ж/м', 'Ала-Тоо ж/м'],
        'Аламедин-1 мкр': ['Аламедин 1', 'Аламедин-1', 'Аламедин1'],
        'Аламединский рынок / базар': ['Аламединс', 'рынок Аламедин', 'аламедин баз'],
        'Алтын-Казык ж/м': ['Алтын-Казык', 'Алтын Казык', 'Алтын-Казык ж/м',
                            'Алтын Казык ж/м', 'ж/м Алтын-Казык', 'ж/м Алтын Казык'],
        'Алтын-ордо ж/м': ['Алтын-ордо', 'Алтын ордо'],
        'Анар ж/м': ['Анар'],
        'Арча-Бешик ж/м': ['Арча-Бешик ж/м'],
        'Асанбай мкр': ['Асанбай', 'Асанбай мкр'],
        'Аска-Таш ж/м': ['Аска-Таш', 'Аска Таш'],
        'Ата-Журт ж/м': ['Ата-Журт ж/м', 'Ата-Журт ж/м'],
        'Ата-Тюрк парк': ['Ата-Тюрк', 'Ата Тюрк', 'Ата-турк', 'Ата турк', 'Ататюрк', 'Ататурк'],
        'Аэропорт Манас': ['Аэропорт Манас', 'Аэропорта Манас'],
        'БГУ': ['бгу', 'БГУ'],
        'Бакай-Ата ж/м': ['Бакай-Ата ж/м', 'Бакай-Ата'],
        'Баткенский рынок / базар': ['Баткенский рынок', 'рынок Баткен', 'баткен базар'],
        'Баят рынок / базар': ['Баят рынок / базар', 'Баят базар', 'рынок баят'],
        'Бета Сторес': ['Бета Сторес', 'Бета-Сторес'],
        'Бета Сторес 2': ['Бета Сторес 2', 'Бета-Сторес 2', 'Бета Сторес2', 'Бета-Сторес2'],
        'Биримдик-Кут ж/м': ['Биримдик-Кут', 'Биримдик Кут'],
        'Бишкек Парк ТРЦ': ['Бишкек Парк', 'Бишкек-Парк', 'БишкекПарк'],
        'Ботанический сад': [' Ботанический сад', 'Ботсад', 'Бот.сад'],
        'Бугу-Эне-Багыш ж/м': ['Бугу-Эне-Багыш', 'Бугу Эне Багыш'],
        'Бугу-Эне-Сай ж/м': [' Бугу-Эне-Сай', 'Бугу Эне Сай'],
        'Военный городок': [' Военный городок', 'Военный-городок'],
        'Восток-5 мкр': ['Восток-5', 'Восток 5', 'Восток5'],
        'Восточный (старый) автовокзал': [' Восточный автовокзал', ' Восточного автовокзал',
                                          'старый автовокзал', 'старого автовокзал',
                                          'Восточный(с', 'Восточный (с'],
        'Газ Городок': [' Газ Городок', ' Газ-Городок'],
        'Гоин': ['Гоин', 'гоин'],
        'Городок Энергетиков': [' Городок Энергетиков', ' Городок-Энергетиков'],
        'Городок строителей': [' Городок строителей', ' Городок-строителей'],
        'Городская больница №4 (ул. Айни)': ['айни', 'больница №4'],
        'Городское ГАИ': ['Городское ГАИ', 'гаи'],
        'Дворец спорта': ['Дворец спорта', 'Дворец-спорта'],
        'Джал мкр (в т.ч. Верхний, Нижний, Средний)': ['жал'],
        'Джунхай рынок': ['Джунхай рынок', 'Джунхай'],
        'Дордой-1 ж/м': ['Дордой-1 ж/м', 'Дордой-1'],
        'Дордой-2 ж/м': ['Дордой-2 ж/м', 'Дордой-2'],
        'Дордой Моторс рынок': ['Дордой Моторс', 'Дордой Моторс'],
        'Дордой рынок / базар': ['Дордой рынок', 'рынок дордой', 'дордой базар'],
        'Достук мкр': ['Достук', 'Достук мкр'],
        'ЖД вокзал': ['ЖД вокзал', 'ЖД-вокзал'],
        'Западный (новый) автовокзал': ['западного автовокзал', 'Западный автовокзал',
                                        'новый автовокзал'],
        'Интрегельпо': ['Интрегельпо', 'Интрегель'],
        'КНУ': ['кну', 'КНУ'],
        'Калыс-ордо ж/м': ['Калыс-ордо', 'Калыс-ордо ж/м'],
        'Кара-Жыгач ж/м': ['Кара-Жыгач', 'Кара Жыгач'],
        'Караван ТРЦ': ['Караван', 'Караван ТРЦ'],
        'Карагачевая роща': ['Карагачевая роща', 'Карагачевая-роща'],
        'Келечек ж/м': ['Келечек ж/м', 'ж/м Келечек'],
        'Керемет ж/м': ['Керемет ж/м', 'ж/м Керемет'],
        'Киргизия 1': ['Киргизия 1', 'Киргизия-1', 'Киргизия1', 'Киргизия'],
        'Киркомстром': ['Киркомстром', 'Кирком'],
        'Кирпичный Завод': ['Кирпичный Завод', 'Завод Кирпичный'],
        'Кожомкул стадион': ['Кожомкул стадион', 'стадион Кожомкул'],
        'Кок-Жар мкр': ['Кок Жар', 'Кок-Жар', 'кокжар'],
        'Колмо ж/м': ['Колмо ж/м', 'Колмо'],
        'Красный Строитель ж/м': ['Красный Строитель', 'Красный-Строитель'],
        'Кудайберген авторынок': ['Кудайберген', 'Кудайберген авторынок'],
        'Кызыл Аскер': ['Кызыл Аскер', 'Кызыл-Аскер'],
        'Кырман ж/м': ['Кырман ж/м', 'Кырман'],
        'Мадина рынок': ['Мадина', 'мадина'],
        'Мега Комфорт ТЦ': ['Мега Комфорт', 'Мега-Комфорт'],
        'Мед Академия': ['Мед Академия', 'МедАкадемия', 'Мед. Академия', 'Мед-Академия',
                         'Мед.Академия'],
        'Моссовет': ['Моссовет', 'Мосовет'],
        'Мурас-Ордо ж/м': ['Мурас-Ордо', 'Мурас Ордо'],
        'Орок ж/м': ['Орок', 'Орок ж/м'],
        'Ортосайский рынок / базар': ['Ортосай', 'Ортосайский рынок / базар'],
        'Оскон-ордо ж/м': ['Оскон-ордо', 'Оскон-ордо ж/м'],
        'Ошский рынок / базар': ['Ошский', 'ош базар'],
        'Панорама': ['Панорама', 'Панорама'],
        'Пишпек': ['Пишпек', 'Пишпек'],
        'Политех': ['Политех'],
        'Полицейский городок ж/м': ['Полицейский городок ж/м', 'Полицейский городок'],
        'Рабочий Городок': [' Рабочий Городок', ' Рабочий-Городок'],
        'Рухий Мурас ж/м': ['Рухий-Мурас', 'Рухий Мурас'],
        'Рынок Баят': ['Рынок Баят', 'Баят'],
        'Салам-Алик ж/м': ['Салам-Алик', 'Салам-Алик ж/м'],
        'Сары-Озон Дыйкан рынок': ['Сары-Озон', 'дыйкан', 'сары озон'],
        'Сары-Челек ж/м': ['Сары-Челек', 'Сары Челек'],
        'Совмина мкр': ['Совмина', 'Совмина мкр'],
        'Старый толчок рынок / базар': ['Старый толчок', 'Старый толчок рынок / базар'],
        'ТЭЦ': ['район ТЭЦ', 'районе ТЭЦ'],
        'Таатан ТЦ': ['Таатан', 'Таатан ТЦ'],
        'Таш-Добо ж/м': ['Таш-Добо', 'Таш Добо'],
        'Таш-Рабат ТРЦ': ['Таш-Рабат', 'Таш Рабат'],
        'Тендик ж/м': ['Тендик ж/м', 'Тендик'],
        'Токольдош': ['Токольдош', 'Токольдош'],
        'Тунгуч мкр': [' Тунгуч', 'Тунгуч '],
        'Тынчтык ж/м': ['Тынчтык ж/м', 'Тынчтык ж/м'],
        'Улан мкр': [' Улан', ' Улан мкр', 'Улан'],
        'Умут ж/м': [' Умут ж/м', ' Умут'],
        'Учкун мкр': [' Учкун', 'Учкун мкр'],
        'Физприборы': ['физприбор', 'физ.прибор', 'физ прибор'],
        'Филармония': [' Филармони', ' Филармония', 'Филармония '],
        'Центральная мечеть': ['Центральная мечеть', 'Центральный мечеть'],
        'Церковь': ['Церковь', 'Церков'],
        'Цум': ['Цум', 'Цум'],
        'Чекиш-Ата рынок': ['Чекиш-Ата рынок', 'Чекиш-Ата'],
        'Шлагбаум': ['район Шлагбаум', 'Шлагбаума'],
        'Шоро завод': ['Шоро завод', 'Шоро'],
        'Ынтымак ж/м': ['Ынтымак ж/м', ' Ынтымак ж'],
        'Эне-Сай ж/м': ['Эне-Сай', 'Эне-Сай ж/м'],
        'Энесай ж/м': ['Энесай', 'Энесай ж/м'],
        'Юг-2 мкр': ['Юг-2', 'Юг 2', 'Юг2'],
        'парк Фучика': ['Фучика', 'парк Фучика']}
    with sq.connect('version_1/lalago.db') as con:
        cur = con.cursor()
        add_link_list = cur.execute(f"SELECT link FROM link_kv").fetchall()
    for i in add_link_list:
        add_link = (((str(i)).replace("',)","")).replace("('","")).replace(' ','')
        print(add_link)

        cur = con.cursor()
        cur.execute("""CREATE TABLE IF NOT EXISTS base (
                                                                   rooms INTEGER DEFAULT 0,
                                                                   series TEXT,
                                                                   ploshad INTEGER DEFAULT 0,
                                                                   remont TEXT,
                                                                   etaj INTEGER DEFAULT 0,
                                                                   etaj_iz INTEGER DEFAULT 0,
                                                                   rayon TEXT DEFAULT Бишкек,
                                                                   data_soz TEXT,
                                                                   data_prod TEXT,
                                                                   prodaves TEXT,
                                                                   price INTEGER DEFAULT 1,
                                                                   an TEXT,
                                                                   osenka TEXT,
                                                                   add_link TEXT,
                                                                   poslednyi_et TEXT,
                                                                   photo_links TEXT
                                                                   )""")
        # Проверка на уникальность обьявлений
        print('Проверка на уникальность...')
        cur.execute(f"SELECT add_link FROM base where add_link =?", (add_link,))
        try:
            r = requests.get(add_link, headers=headers,proxies=get_proxy(proxy_list), timeout=5)
            soup = Bs(r.text, features="html.parser")
            data_info = soup.find_all('div', class_='about-ad-info__date')
            print(data_info)
        except:
            continue

        if str(data_info) == '[]':
            print('не активно')
        else:
            try:
                if cur.fetchone() is None:

                    print('NEW ADS!!!')

                    # Парсим обявлений

                    price = soup.find('span', class_='heading international-price').text
                    print(price)
                    image = soup.find('div', class_='desktop css-10yjukn')

                    photo_links = []
                    for i in str(image).split('"'):

                        if 'https://img5.lalafo.com/i/posters/api' in i:
                            i = i.replace("/api/", "/original/")

                            if i in photo_links:
                                continue
                            else:
                                photo_links.append(i)

                    try:
                        opisanie = soup.find('div', class_='description__wrap').text
                        link_user = 'https://lalafo.kg' + (soup.find('a', class_='userName')).attrs['href']
                    except:
                        print('ошибка получение ссылки пользователя или Описания!!!')

                    print('точка 1')

                    for i in data_info:
                        i = i.text
                        if 'Создано: ' in i:
                            i = i.replace('Создано: ', '')
                            data_soz = i
                        elif 'Обновлено: ' in i:
                            i = i.replace('Обновлено: ', '')
                            data_prod = i

                        else:
                            data_soz = ' '
                            data_prod = ' '
                            continue

                    print('точка 2')

                    # Стандартизация цены
                    price = price.replace('USD', '').replace(' ', '').replace('KGS', '')
                    parameters = soup.find('ul', class_='details-page__params css-tl517w')
                    remont = ' '
                    series = ' '
                    ploshad = '0'
                    etaj = '0'
                    etaj_iz = '0'
                    rayon = ' '
                    prodaves = ' '
                    rooms = '0'
                    poslednyi_et = ' '

                    print('точка 3')

                    try:
                        print('parametr', parameters)
                        for i in parameters:
                            if 'Площадь (м2):' in i.text:
                                i = i.text
                                i = i.replace('Площадь (м2):', '')
                                ploshad = i

                            elif 'Количество комнат:' in i.text:
                                i = i.text
                                i = i.replace('Количество комнат:', '').replace(' комнаты', '').replace(
                                    ' комната',
                                    '').replace(' комнат', '')

                                rooms = i
                                if rooms == 'Студия':
                                    rooms = 1

                            elif 'Этаж:' in i.text:
                                i = i.text
                                i = i.replace('Этаж:', '')
                                etaj = i

                            elif 'Количество этажей:' in i.text:
                                i = i.text
                                i = i.replace('Количество этажей:', '')
                                etaj_iz = i

                            elif 'Район:' in i.text:
                                i = i.text
                                i = i.replace('Район:', '')
                                rayon = i

                            elif 'Тип предложения:' in i.text:
                                i = i.text
                                i = i.replace('Тип предложения:', '')
                                prodaves = i

                            elif 'Серия:' in i.text:
                                i = i.text
                                i = i.replace('Серия:', '')
                                series = i

                            elif 'Ремонт:' in i.text:
                                i = i.text
                                i = i.replace('Ремонт:', '')
                                remont = i

                            else:
                                continue

                        # Анализ текста для опеределения данных

                        if series == ' ':

                            elitka_list = ['элитка', 'элитный', 'премиум', 'элит']
                            seria_102 = ['102 серия', '102 сер', '102сер', '102']
                            seria_104 = ['104 серия', '104 сери', '104сер', '104']
                            seria_105 = ['105 серия', '105 сери', '105сер', '105']
                            seria_106 = ['106 серия', '106 сери', '106сер', '106']
                            hrishevka = ['хрущёвка', 'хрущевка', 'хруш', "хрущ"]

                            for i in seria_102:
                                if i in opisanie:
                                    series = "102 серия"
                                else:
                                    continue

                            for i in seria_104:
                                if i in opisanie:
                                    series = '104 серия'
                                else:
                                    continue

                            for i in seria_105:
                                if i in opisanie:
                                    series = '105 серия'
                                else:
                                    continue

                            for i in seria_106:
                                if i in opisanie:
                                    series = '106 серия'
                                else:
                                    continue

                            for i in hrishevka:
                                if i in opisanie:
                                    series = 'Хрущёвка'
                                else:
                                    continue

                            for i in elitka_list:
                                if i in opisanie:
                                    series = 'Элитка'
                                else:
                                    continue

                        print('Стандартизация ремонта')
                        if remont == ' ':

                            bez_remont = ['псо', 'без ремонта', 'требуеться рем', 'ПСО', 'самоотделк',
                                          'само отде', 'под само']
                            st_remont = ['старый ремонт', 'ремонт старый', "требуеться косм", 'состояние сре']
                            sv_remont = ['свежий ремонт', 'новый ремонт', 'ремонт', 'обои', 'состояние']
                            ev_remont = ['Евроремонт', 'евро', 'Eвроремонт', 'техника', 'мебел']

                            for i in st_remont:
                                if i in opisanie:
                                    remont = 'Старый ремонт'
                                else:
                                    continue

                            for i in sv_remont:
                                if i in opisanie:
                                    remont = 'Свежий ремонт'
                                else:
                                    continue

                            for i in bez_remont:
                                if i in opisanie:
                                    remont = 'Без ремонта'

                                else:
                                    continue
                            for i in ev_remont:
                                if i in opisanie:
                                    remont = 'Евроремонт'

                                else:
                                    continue

                        print('Стандартизация Района')

                        if rayon == ' ':

                            for k, v in rayon_list.items():
                                for i in v:
                                    if i in opisanie:
                                        rayon = k

                        print('Стандартизация площади')
                        # Стандартизация площади
                        series = str(series)
                        rooms = int(rooms)

                        if series == '102 серия':

                            if rooms == 1:
                                ploshad = 30

                            elif rooms == 2:
                                ploshad = 42

                            elif rooms == 3:
                                ploshad = 56

                        elif series == "104 серия":

                            if rooms == 1:
                                ploshad = 32

                            elif rooms == 2:
                                ploshad = 43

                            elif rooms == 3:
                                ploshad = 58

                        elif series == "104 серия улучшенная":

                            if rooms == 1:
                                ploshad = 30

                            elif rooms == 2:
                                ploshad = 42

                            elif rooms == 3:
                                ploshad = 56
                        elif series == "105 серия":

                            if rooms == 1:
                                ploshad = 34

                            elif rooms == 2:
                                ploshad = 50

                            elif rooms == 3:
                                ploshad = 62

                            elif rooms == 4:
                                ploshad = 74

                            elif rooms == 5:
                                ploshad = 87

                        elif series == "105 серия улучшенная":
                            if rooms == 1:
                                ploshad = 34

                            elif rooms == 2:
                                ploshad = 50

                            elif rooms == 3:
                                ploshad = 62

                            elif rooms == 4:
                                ploshad = 74

                            elif rooms == 5:
                                ploshad = 87

                        elif series == "106 серия":
                            if rooms == 1:
                                ploshad = 35

                            elif rooms == 2:
                                ploshad = 52

                            elif rooms == 3:
                                ploshad = 66

                        elif series == "106 серия улучшенная":
                            if rooms == 1:
                                ploshad = 35

                            elif rooms == 2:
                                ploshad = 52

                            elif rooms == 3:
                                ploshad = 66

                        elif series == "Хрущёвка":
                            if rooms == 1:
                                ploshad = 30

                            elif rooms == 2:
                                ploshad = 42

                            elif rooms == 3:
                                ploshad = 56

                        else:
                            pass

                    except:
                        print('Точка 4')

                    print('Точка 5')

                    AN = str(filter_AN(link_user=link_user))

                    print('точка 7')

                    if price == 'Договорная':
                        continue

                    try:
                        # Проверка на выгодность квартиры

                        if 5 < int(ploshad) < 350:
                            if 280 < int(price) < 1800:
                                price = int(ploshad) * int(price)

                            if int(etaj) == int(etaj_iz):
                                poslednyi_et = 'последний этаж'

                            # Оценка на выгодность

                            sr_price = osenka_kv(series=series, remont=remont, rooms=rooms, rayon=rayon,
                                                 etaj=etaj,
                                                 etaj_iz=etaj_iz, poslednyi_et=poslednyi_et)

                            price_m2 = round(int(price) / int(ploshad))
                            sr_price = int(sr_price)

                            if int(price_m2) < int(sr_price):
                                print('Оценка: TRUE')
                                osenka = 'Выгодный'
                                if rayon == ' ':
                                    print('Район пустой')
                                    continue

                            else:

                                osenka = 'Выше среднего'

                            base_add(rooms=rooms, series=series, ploshad=ploshad, remont=remont, etaj=etaj,
                                     etaj_iz=etaj_iz, rayon=rayon, prodaves=prodaves, price=price, AN=AN,
                                     add_link=add_link, poslednyi_et=poslednyi_et, photo_links=photo_links,
                                     data_soz=data_soz, data_prod=data_prod)

                            print('Добавлено в базу!!!')

                        else:
                            print('Не корректный площад')
                            base_add(rooms=rooms, series=series, ploshad=ploshad, remont=remont, etaj=etaj,
                                     etaj_iz=etaj_iz, rayon=rayon, prodaves=prodaves, price=price, AN=AN,
                                     add_link=add_link, poslednyi_et=poslednyi_et, photo_links=photo_links, data_soz=data_soz,data_prod=data_prod)

                            print('Добавлено в базу!!!')
                            continue

                    except:

                        base_add(rooms=rooms, series=series, ploshad=ploshad, remont=remont, etaj=etaj,
                                 etaj_iz=etaj_iz, rayon=rayon, prodaves=prodaves, price=price, AN=AN,
                                 add_link=add_link, poslednyi_et=poslednyi_et, photo_links=photo_links,
                                 data_soz=data_soz, data_prod=data_prod)

                        print('exept')

                    print('Обновление базы!!!')
                    print(osenka)

                    with sq.connect('version_1/lalago.db') as con:
                        cur = con.cursor()

                        cur.execute("""CREATE TABLE IF NOT EXISTS base (
                                                                                rooms INTEGER DEFAULT 0,
                                                                                series TEXT,
                                                                                ploshad INTEGER DEFAULT 0,
                                                                                remont TEXT,
                                                                                etaj INTEGER DEFAULT 0,
                                                                                etaj_iz INTEGER DEFAULT 0,
                                                                                rayon TEXT DEFAULT Бишкек,
                                                                                data_soz TEXT,
                                                                                data_prod TEXT,
                                                                                prodaves TEXT,
                                                                                price INTEGER DEFAULT 1,
                                                                                an TEXT,
                                                                                osenka TEXT,
                                                                                add_link TEXT,
                                                                                poslednyi_et TEXT,
                                                                                photo_links TEXT
                                                                                )""")

                        cur.execute(
                            f"UPDATE base SET osenka = '{str(osenka)}' WHERE add_link = '{str(add_link)}' ")

                        con.commit()
                    print('ОЦЕНКА ОБНОВЛЕНО!!!')

                else:
                    pass

            except:
                continue

Thread(target=Starter_check_lalafo_kg(), args=()).start()


#!/usr/bin/env python
# coding: utf-8

# Setup
import pandas as pd
from pathlib import Path
from matplotlib import pyplot as plt
import parameters

# Load Tables
games = pd.read_csv(parameters.locations['staging'].joinpath('games.csv'), index_col=0)
views = list(parameters.locations['analysis'].glob('*.csv'))

analysis_views={}
for view in views:
    analysis_views[view.stem] = pd.read_csv(view, index_col=0)
analysis_views['game_scores'].head()

# Reports
## player performance
data_source = analysis_views['game_scores']
focus_player = 'david'

filtered_data = data_source.loc[data_source['players']==focus_player]
overall = filtered_data.groupby('seat').agg({'game_id':'count', 'scores':['mean', 'median']})
overall.columns = ['count', 'mean culture', 'median culture']
place_counts = filtered_data.groupby('seat')['place'].value_counts().unstack().fillna(0)
overall.index = pd.CategoricalIndex(overall.index, ['orange','purple','green','grey'])
overall.sort_index().join(place_counts)

## seating order randomization


# 値のソート
# (動物、最高時速)のリスト(各要素はタプルで作成)
animal_list = [
    ("ライオン", 58),
    ("チーター" , 110),
    ("シマウマ" , 60),
    ("トナカイ" , 80),
]

# 足の速い順に並び替える
faster_list = sorted(
    animal_list,
    key = lambda ani : ani[1], # 1はタプル配列の用を番号0の次の1、速度の値のこと
    reverse = True
)

# 結果を表示
print("\nlist型の並び替え")
for i in faster_list : print(i)

# dict型で動物 : 最高速度を指定
animal_dict = {
    "ライオン" : 58,
    "チーター" : 110,
    "シマウマ" : 60,
    "トナカイ" : 80,
}

# 時速で並び替えて表示
faster_dict = sorted(
    animal_dict.items(), # dict型ではitems()で要素をひとつづつ指定する
    key = lambda ani: ani[1],
    reverse  = True
)

# 結果を表示
print("\ndict型の並び替え")
for name, speed in faster_dict : print(name, speed)

print("\ndict -> tuple")
print("dict : " + str(animal_dict))
print("tuple : " + str(animal_dict.items()))


import random
import tensorflow as tf
import numpy as np
import pickle
import os
from src.utils.movesprep import data_path

from src.utils import moves
from src.utils import movesprep
from src.utils import representations

class Search:

    def __init__(self, left_moves, right_moves):
        self.lm = left_moves
        self.rm = right_moves

    def highest_among(self, values):
        best_state = values[0][0]
        best_score = values[0][1]

        for nxt in values:
            if nxt[1] > best_score:
                best_state = nxt[0]
                best_score = nxt[1]

        return best_state

    def rollout(self, state):
        """
        :param state: game state from which playout happens
        :return: number of steps during the playout
        """
        reachable = moves.possible_moves(state, self.lm, self.rm).values()
        if int(len(reachable)) == 0:
            return 0

        # random strategy
        reachable = list(reachable)
        random.shuffle(reachable)
        return 1 + self.rollout(moves.random_tile(reachable[0]))

    def best_move(self, state, search_iters):
        reachable = moves.possible_moves(state, self.lm, self.rm).values()
        nxts = []
        for r in reachable:
            score = 0
            for i in range(search_iters):
                score += self.rollout(moves.random_tile(r))
            nxts.append((r, score))
        return self.highest_among(nxts)

    def play_game(self, per_move, verbose=False):
        state = [0, 0, 0, 0]
        state = moves.random_tile(state)
        reachable = moves.possible_moves(state, self.lm, self.rm).values()

        lvl = 0
        while len(reachable) > 0:
            if verbose:
                print('move', lvl, state)
            lvl += 1
            n = self.best_move(state, per_move)
            state = moves.random_tile(n)
            reachable = moves.possible_moves(state, self.lm, self.rm).values()

class MCTS(Search):

    def __init__(self, left_moves, right_moves, exploration_coefficient):
        super().__init__(left_moves, right_moves)
        self.scoresum = {}
        self.count = {}
        self.c = exploration_coefficient
        pass

    def save_tree(self):
        with open(os.path.join(data_path(), 'mcts-counts.pickle'), 'wb') as file:
            pickle.dump(self.count, file)
        with open(os.path.join(data_path(), 'mcts-sums.pickle'), 'wb') as file:
            pickle.dump(self.scoresum, file)

    def restore_tree(self):
        with open(os.path.join(data_path(), 'mcts-counts.pickle'), 'rb') as file:
            self.count = pickle.load(file)
        with open(os.path.join(data_path(), 'mcts-sums.pickle'), 'rb') as file:
            self.scoresum = pickle.load(file)

    def avgscore(self, state):
        if tuple(state) not in self.count:
            return 0
        return self.scoresum[tuple(state)] / 10. / self.count[tuple(state)]

    def ucb(self, state, parent):
        if tuple(state) not in self.count:
            return 1000.
        n_parent = self.count[tuple(parent)]
        n_state = self.count[tuple(state)]
        exploration_term = self.c * np.sqrt(np.log(n_parent) / n_state)
        return self.avgscore(state) + exploration_term

    def highest_ucb(self, reachable, parent, verb=False):
        ucbs = []
        for nxt in reachable:
            score = self.ucb(nxt, parent)
            ucbs.append((nxt, score))

        if verb:
            print(parent, ucbs, self.highest_among(ucbs))
        return self.highest_among(ucbs)

    def search(self, state):
        if tuple(state) not in self.count:
            self.count[tuple(state)] = 1
            self.scoresum[tuple(state)] = 0
            res = self.rollout(state)
            self.scoresum[tuple(state)] = res
            return res + 1

        self.count[tuple(state)] += 1
        reachable = moves.possible_moves(state, self.lm, self.rm).values()

        if len(reachable) == 0:
            return 0

        chosen = self.highest_ucb(reachable, state)

        if tuple(chosen) not in self.count:
            self.count[tuple(chosen)] = 0
            self.scoresum[tuple(chosen)] = 0
        self.count[tuple(chosen)] += 1

        nxt = moves.random_tile(chosen)
        res = self.search(nxt)

        self.scoresum[tuple(chosen)] += res - 1
        self.scoresum[tuple(state)] += res
        return 1 + res

    def best_move(self, state, search_iters):
        for i in range(search_iters):
            self.search(state)

        reachable = moves.possible_moves(state, self.lm, self.rm).values()
        state_score = []
        for r in reachable:
            state_score.append((r, self.avgscore(r)))
        return self.highest_among(state_score)

    def play_game(self, search_iters, verbose=False):
        state = [0, 0, 0, 0]
        state = moves.random_tile(state)
        reachable = moves.possible_moves(state, self.lm, self.rm).values()

        lvl = 0
        while len(reachable) > 0:
            if verbose:
                print('move', lvl, state)
            lvl += 1

            chosen = self.best_move(state, search_iters)
            state = moves.random_tile(chosen)
            reachable = moves.possible_moves(state, self.lm, self.rm).values()

    def dataset(self, nn):
        data = []
        batch_size = 10000
        for state, cnt in self.count.items():
            data.append([state, self.scoresum[state] / cnt / 100.])
        x_train = []
        y_train = []
        i = 0
        while i < len(data):
            batch = data[i:i + batch_size]
            x_train.append([list(s[0]) for s in batch])
            y_train.append([[s[1]] for s in batch])
            i += batch_size
        return x_train, y_train

    def policy_dataset(self):
        data = []
        batch_size = 1000
        dct = {'left': 0, 'right': 1, 'up': 2, 'down': 3}
        for state, cnt in self.count.items():
            reachable = moves.possible_moves(state, self.lm, self.rm)  # ri, le, dn, up
            scores = [0.,0.,0.,0.]
            no_data = False
            i = 0
            j = 0
            mx = -1
            for dire, st in reachable.items():
                if tuple(st) not in self.count:
                    no_data = True
                    break
                else:
                    if mx < self.scoresum[tuple(st)] / self.count[tuple(st)]:
                        mx = self.scoresum[tuple(st)] / self.count[tuple(st)]
                        i = j
                j += 1
            if mx != -1 and not no_data:
                scores[i] = 1.
                data.append([list(state), list(scores)])

            if no_data:
                continue

        print(len(data))
        print(data[0])

        x_train = []
        y_train = []

        i = 0
        while i < len(data):
            batch = data[i:i + batch_size]
            x_train.append([representations.chanelled_flat(representations.tiles_2d(s[0])) for s in batch])
            y_train.append([s[1] for s in batch])
            i += batch_size

        return x_train, y_train

class NetworkMCTS(MCTS):

    def __init__(self, left_moves, right_moves, exploration_coefficient, neural_network, discount):
        super().__init__(left_moves, right_moves, exploration_coefficient)
        self.nn = neural_network
        self.batch = []
        self.gamma = discount
        self.data = []
        self.lrs = []

    def rollout(self, state):
        reachable = moves.possible_moves(state, self.lm, self.rm).values()
        if int(len(reachable)) == 0:
            return 0

        tiles = representations.tiles_2d(state)
        #channelled = representations.chanelled_tiles(tiles)
        pred = self.nn.predict(np.array([tiles]))[0]
        return pred

    def sim_pol(self, net, state):
        r = moves.possible_moves(state, self.lm, self.rm)
        if len(r) == 0:
            return 0
        x = np.array([representations.chanelled_tiles(representations.tiles_2d(state))])
        x = np.array([representations.tiles_2d(state)])
        y = net.predict(x)[0]
        #print(y)
        act = 'left'
        mx = -1
        dct = {'left': 0, 'right': 1, 'up': 2, 'down': 3}
        for dire, st in r.items():
            if mx < y[dct[dire]]:
                mx = y[dct[dire]]
                act = dire
        print(act[0], end='')
        state = r[act]
        state = moves.random_tile(state)
        return 1 + self.sim_pol(net, state)

    def best_move_net(self, state, net):
        reachable = moves.possible_moves(state, self.lm, self.rm).values()
        if int(len(reachable)) == 0:
            return None
        x = []
        for r in reachable:
            #x.append(representations.chanelled_tiles(representations.tiles_2d(r)))
            x.append(representations.tiles_2d(r))
        x = np.array(x)
        y = net.predict(x)
        return self.highest_among(list(zip(reachable, y)))

    def play_game_net_policy(self, net):
        state = [0,0,0,0]
        state = moves.random_tile(state)
        return self.sim_pol(net, state)

    def simulate_game_net(self, state, nn):
        reachable = moves.possible_moves(state, self.lm, self.rm).values()
        if int(len(reachable)) == 0:
            return 0
        x = []
        for r in reachable:
            #x.append(representations.chanelled_tiles(representations.tiles_2d(r)))
            x.append(representations.tiles_2d(r))
        x = np.array(x)
        y = nn.predict(x)
        chosen = self.highest_among(list(zip(reachable, y)))
        nxt = moves.random_tile(chosen)
        return self.simulate_game_net(nxt, nn) + 1

    def play_network_only(self, nn):
        state = [0, 0, 0, 0]
        state = moves.random_tile(state)
        return self.simulate_game_net(state, nn)

    def alternative_play(self, search_iters, target=5, verbose=False, make_batch=False):
        self.batch.clear()
        state = [0, 0, 0, 0]
        state = moves.random_tile(state)
        reachable = moves.possible_moves(state, self.lm, self.rm).values()
        batch = []

        lvl = 0
        final_score = -1.

        while len(reachable) > 0:
            if representations.contains(state, target):
                final_score = 1.
                break

            if make_batch:
                batch.append([state, lvl, 0.])

            if verbose:
                print('move', lvl, state)
            lvl += 1

            chosen = self.best_move(state, search_iters)
            if make_batch:
                batch.append([chosen, lvl, 0.])
            state = moves.random_tile(chosen)
            reachable = moves.possible_moves(state, self.lm, self.rm).values()

        print('moves', lvl)
        if make_batch:
            batch.append([state, lvl, final_score])
            for i in range(len(batch) - 2, -1, -1):
                batch[i][2] = final_score
            self.batch += batch

    def play_game(self, search_iters, verbose=False, make_batch=False):
        self.batch.clear()
        state = [0, 0, 0, 0]
        state = moves.random_tile(state)
        reachable = moves.possible_moves(state, self.lm, self.rm).values()
        batch = []

        lvl = 0
        while len(reachable) > 0:
            if make_batch:
                batch.append([state, lvl, 0.])

            if verbose:
                print('move', lvl, state)
            lvl += 1

            chosen = self.best_move(state, search_iters)
            state = moves.random_tile(chosen)
            reachable = moves.possible_moves(state, self.lm, self.rm).values()

        print('moves', lvl)
        if make_batch:
            batch.append([state, lvl, 0.])
            for i in range(len(batch) - 2, -1, -1):
                batch[i][2] = 1 + self.gamma * batch[i + 1][2]
            self.batch += batch

    def fit_batch(self, epochs, lr):
        x_train = []
        y_train = []
        for example in self.batch:
            tiles = representations.tiles_2d(example[0])
            #channelled = representations.chanelled_tiles(tiles)
            x_train.append(tiles)
            y_train.append([example[2]])
        x_train = np.array([x_train])
        y_train = np.array([y_train])
        self.nn.fit(x_train, y_train, epochs, lr, False)

    def battle(self, nn):
        nnscore = 0
        selfscore = 0
        for i in range(10):
            nnscore += self.play_network_only(nn)
            selfscore += self.play_network_only(self.nn)
        print('Scores', round(selfscore / nnscore, 2), selfscore/10, nnscore/10)
        if selfscore / nnscore < 1.05:
            return nn, False
        else:
            print('Updated nets', round(selfscore / nnscore, 2), selfscore/10, nnscore/10)
            return self.nn, True

    def normalize_batch(self):
        for i in range(len(self.batch)):
            self.batch[i][-1] /= 1000.

    def avgnetscores(self, nns):
        sc = []
        for nn in nns:
            sm = 0.
            for i in range(10):
                sm += self.play_network_only(nn)
            sc.append(sm/10)
        return sc

    def train(self, rounds, plays_per_round, batch_lr_discount, initial_lr, batch_ignore_threshold, target):
        self.data = []
        self.lrs = []

        for r in range(rounds):
            print('Round', r + 1, 'begins')
            prevdata = self.data.copy()
            prevlrs = self.lrs.copy()
            for i in range(int(len(self.data))):
                self.lrs[i] *= batch_lr_discount

            while len(self.lrs) > 0 and self.lrs[0] < batch_ignore_threshold:
                self.data = self.data[1:]
                self.lrs = self.lrs[1:]

            for p in range(plays_per_round):
                #self.play_game(100, verbose=False, make_batch=True)
                self.alternative_play(100, target, verbose=False, make_batch=True)

            self.normalize_batch()
            self.data.append(self.batch)
            self.lrs.append(initial_lr)

            nns = []
            setups = [(50, 1., 10), (10, 1., 5), (35, 1., 1), (12, 1., 2), (5, 3., 3)]  # (epochs, lr, datanum)
            for i in range(5):
                nns.append(self.nn.cpy())

            nnself = self.nn.cpy()
            for i, nn in enumerate(nns):
                self.nn = nns[i]
                for j in range(len(self.data) - 1, max(-1, len(self.data) - 1 - setups[i][2]), -1):
                    self.batch = self.data[j]
                    self.fit_batch(setups[i][0], self.lrs[j] * setups[i][1])

            nns.append(nnself)
            avgs = self.avgnetscores(nns)
            print('avg scores', avgs)
            best = 0.
            better = True
            for i, a in enumerate(avgs):
                if a > best:
                    if i == len(avgs) - 1:
                        better = False
                    best = a
                    self.nn = nns[i]

            if not better:
                self.data = prevdata
                self.lrs = prevlrs
            else:
                print('upgraded', best, avgs[-1])

            with open(os.path.join(data_path(), 'mcts-netH1.pickle'), 'wb') as file:
                pickle.dump(self.nn, file)

def play_tf(net, lm, rm):
    state = [0,0,0,0]
    state = moves.random_tile(state)
    return sim_tf(state, net, lm, rm)

def sim_tf(state, net: tf.keras.models.Sequential, lm, rm):
    r = moves.possible_moves(state, lm, rm)
    if len(r) == 0:
        return 0
    x = representations.chanelled_tiles(representations.tiles_2d(state))
    y = net.predict(np.array([x]), batch_size=1)[0]
    act = 'left'
    mx = -1
    dct = {'left': 0, 'right': 1, 'up': 2, 'down': 3}
    for dire, st in r.items():
        if mx < y[dct[dire]]:
            mx = y[dct[dire]]
            act = dire
    print(act[0], end='')
    state = r[act]
    state = moves.random_tile(state)
    return 1 + sim_tf(state, net, lm, rm)

if __name__ == '__main__':

    lm, rm = movesprep.load_moves()
    import src.mllib.models as models
    import src.mllib.layers as layers

    '''
    net = models.Sequential()
    #net.add_layer(layers.Convolutional(4, 4, 16, 16, 2, 'tanh'))
    #net.add_layer(layers.Convolutional(3, 3, 16, 16, 2, 'tanh'))
    net.add_layer(layers.Flatten(4, 4, 1))
    net.add_layer(layers.Dense(16, 32, 'tanh'))
    net.add_layer(layers.Dense(32, 64, 'tanh'))
    net.add_layer(layers.Dense(64, 128, 'tanh'))
    net.add_layer(layers.Dense(128, 64, 'tanh'))
    net.add_layer(layers.Dense(64, 32, 'tanh'))
    net.add_layer(layers.Dense(32, 1, 'tanh'))
    '''

    with open(os.path.join(data_path(), 'mcts-netH0.pickle'), 'rb') as file:
        net = pickle.load(file)

    nmcts = NetworkMCTS(lm, rm, 1., net, 0.997)
    nmcts.train(100, 5, 0.9, 0.001, 0.001, 8)

'''
    tree = MCTS(lm, rm, 10.)
    #tree.restore_tree()
    print(len(tree.count))
    nmcts = NetworkMCTS(lm, rm, 0., net, 0.1)   # only for some functions

    x_train, y_train = tree.dataset(net)

    epochs = 1
    lr = 0.001

    for e in range(epochs):
        print('Epoch', e + 1)
        b = 0
        for X, Y in list(zip(x_train, y_train)):
            b += 1
            X = np.array([representations.chanelled_tiles(representations.tiles_2d(x)) for x in X])
            Y = np.array(Y)
            net.predict(X)
            net.backward(Y, lr)
            nmcts.count = tree.count
            nmcts.scoresum = tree.scoresum
            print('Batch', b, 'loss:', net.summary_loss(np.array([X]), np.array([Y])), 'moves test:', nmcts.play_network_only(net))
    print('Finished')
    with open(os.path.join(data_path(), 'net-free-train.pickle'), 'wb') as file:
        pickle.dump(net, file)

    net.add_layer(layers.Convolutional(4, 4, 1, 8, 2, 'sigmoid'))
    net.add_layer(layers.Convolutional(3, 3, 8, 16, 2, 'sigmoid'))
    net.add_layer(layers.Flatten(2, 2, 16))
    net.add_layer(layers.Dense(64, 128, 'sigmoid'))
    net.add_layer(layers.Dense(128, 64, 'sigmoid'))
    net.add_layer(layers.Dense(64, 4, 'sigmoid'))

    tree = MCTS(lm, rm, 0.)
    tree.restore_tree()
    print(len(tree.count))
    nmcts = NetworkMCTS(lm, rm, 0., net, 0.1)   # only for some functions

    #x_train, y_train = tree.policy_dataset()
    #tp = (x_train, y_train)
    #with open(os.path.join(data_path(), 'policy-dataset-flat.pickle'), 'wb') as file:
    #    pickle.dump(tp, file)

    with open(os.path.join(data_path(), 'policy-dataset.pickle'), 'rb') as file:
        x_train, y_train = pickle.load(file)

    x_train = x_train[:-1]
    y_train = y_train[:-1]
    x_train = np.array(x_train)
    y_train = np.array(y_train)

    nmcts.nn = net
    print(' play', nmcts.play_game_net_policy(net))
    print('loss before', net.summary_loss(x_train, y_train))
    net.fit(x_train, y_train, 1, 0.01, verbose=True)
    print(' play after', nmcts.play_game_net_policy(net))
    net.fit(x_train, y_train, 1, 0.01, verbose=True)
    print(' play after', nmcts.play_game_net_policy(net))
    net.fit(x_train, y_train, 1, 0.01, verbose=True)
    print(' play after', nmcts.play_game_net_policy(net))
    net.fit(x_train, y_train, 1, 0.01, verbose=True)
    print(' play after', nmcts.play_game_net_policy(net))
    net.fit(x_train, y_train, 1, 0.01, verbose=True)
    print(' play after', nmcts.play_game_net_policy(net))

    with open(os.path.join(data_path(), 'net-policy-train.pickle'), 'wb') as file:
        pickle.dump(net, file)

    net = tf.keras.models.Sequential()
    net.add(tf.keras.layers.Conv2D(32, (2, 2), activation='sigmoid'))
    net.add(tf.keras.layers.Conv2D(32, (2, 2), activation='sigmoid'))
    net.add(tf.keras.layers.Flatten())
    net.add(tf.keras.layers.Dense(128, 'sigmoid'))
    net.add(tf.keras.layers.Dense(64, 'sigmoid'))
    net.add(tf.keras.layers.Dense(4, 'sigmoid'))

    opt = tf.keras.optimizers.Adam()
    net.compile(opt, loss=tf.keras.losses.mse)

    x_tr = []
    y_tr = []
    for b in x_train:
        x_tr += list(b)
    for b in y_train:
        y_tr += list(b)

    x_tr = np.array(x_tr)
    y_tr = np.array(y_tr)

    print(play_tf(net, lm, rm))
    net.fit(x_tr, y_tr, epochs=30)

    print(' moves', play_tf(net, lm, rm))
    print(' moves', play_tf(net, lm, rm))
    print(' moves', play_tf(net, lm, rm))
    print(' moves', play_tf(net, lm, rm))
    print(' moves', play_tf(net, lm, rm))

    TRAINING MCTS

    tree = MCTS(lm, rm, 5.)
    tree.restore_tree()
    for i in range(10):
        print('MCTS playing')
        tree.play_game(100, True)
        tree.save_tree()
        print('seen states:', len(tree.count))
        print('Finished')
'''


# -*- coding: utf-8 -*-
# This script goes through USFM files, generating a list of verses that contain properly marked footnotes.
# Reports errors to stderr and issues.txt.
# Set source_dir and usfmVersion to run.

# Global variables
source_dir = r'C:\DCS\Portuguese\pt-br_ulb'
usfmVersion = 2     # if version 3.0 or greater, tolerates unknown tokens and verse fragments

import os
import sys
import parseUsfm
import io
import re
import json
if usfmVersion >= 3.0:
    import usfm_utils

vv_re = re.compile(r'([0-9]+)-([0-9]+)')

class State:
    IDs = []
    ID = ""
    chapter = 0
    verse = 0
    reference = ""
    footnoteRefs = list()

    # Resets state data for a new book
    def addID(self, id):
        State.IDs.append(id)
        State.ID = id
        State.chapter = 0
        State.verse = 0
        State.reference = id

    def getIDs(self):
        return State.IDs

    def addChapter(self, c):
        State.chapter = int(c)
        State.verse = 0
        State.reference = State.ID + " " + c

    def addVerse(self, v):
        State.verse = int(v)
        State.reference = State.ID + " " + str(State.chapter) + ":" + v

    # Adds the current reference to the list of footnote references
    def addFootnote(self):
        if self.reference not in State.footnoteRefs:
            self.footnoteRefs.append(self.reference)

    # Returns list of footnote references as a json string
    def getFootnoteReferences(self):
        return json.dumps(self.footnoteRefs)

    def countFootnotes(self):
        return len(self.footnoteRefs)

def shortname(longpath):
    shortname = longpath
    if source_dir in longpath:
        shortname = longpath[len(source_dir)+1:]
    return shortname

def takeID(id):
    state = State()
    if len(id) < 3:
        reportError("Invalid ID: " + id)
    id = id[0:3].upper()
    if id in state.getIDs():
        reportError("Duplicate ID: " + id)
    state.addID(id)

def takeC(c):
    state = State()
    state.addChapter(c)

# Receives a string containing a verse number or range of verse numbers.
# Reports errors related to the verse number(s), such as missing or duplicated verses.
def takeV(vstr):
    state = State()
    vlist = []
    if vstr.find('-') > 0:
        vv_range = vv_re.search(vstr)
        if vv_range:
            vn = int(vv_range.group(1))
            vnEnd = int(vv_range.group(2))
            while vn <= vnEnd:
                vlist.append(vn)
                vn += 1
        else:
            reportError("Problem in verse range near " + State.reference)
    else:
        vlist.append(int(vstr))

    for vn in vlist:
        v = str(vn)
        state.addVerse(str(vn))

# Writes error message to stderr and to issues.txt.
def reportError(msg):
    try:
        sys.stderr.write(msg + "\n")
    except UnicodeEncodeError as e:
        state = State()
        sys.stderr.write(state.reference + ": (Unicode...)\n")

# Returns true if token is a countable part of a footnote
def isFootnote(token):
    # return token.isF_S() or token.isF_E() or token.isFR() or token.isFR_E() or token.isFT() or token.isFP() or token.isFE_S() or token.isFE_E()
    return token.isF_S() or token.isF_E()

def take(token):
    global usfmVersion

    state = State()
    if isFootnote(token):
        state.addFootnote()
    if token.isID():
        takeID(token.value)
    elif token.isC():
        if not state.ID:
            reportError("Missing book ID: " + state.reference)
            sys.exit(-1)
        takeC(token.value)
    elif token.isV():
        takeV(token.value)

# Corresponding entry point in tx-manager code is verify_contents_quiet()
def processFile(path):
    input = io.open(path, "tr", 1, encoding="utf-8-sig")
    str = input.read(-1)
    input.close()
    if usfmVersion >= 3.0:
        str = usfm_utils.usfm3_to_usfm2(str)

    print("CHECKING " + shortname(path))
    sys.stdout.flush()
    for token in parseUsfm.parseString(str):
        take(token)
    state = State()
    state.addID("")
    sys.stderr.flush()

# Verifies all .usfm files under the specified folder.
def processDir(dirpath):
    for f in os.listdir(dirpath):
        if f[0] != '.':         # ignore hidden files
            path = os.path.join(dirpath, f)
            if os.path.isdir(path):
                # It's a directory, recurse into it
                processDir(path)
            elif os.path.isfile(path) and path[-3:].lower() == 'sfm':
                processFile(path)

# Writes list of footnote references to a file.
def dumpFootnoteReferences():
    state = State()
    refs = state.getFootnoteReferences()
    if refs:
        path = os.path.join(source_dir, "footnotedVerses.json")
        footnoteFile = io.open(path, "tw", encoding='utf-8', newline='\n')
        footnoteFile.write(refs)
        footnoteFile.close()
    sys.stdout.write("Found " + str(state.countFootnotes()) + " footnotes.\n")

if __name__ == "__main__":
    if len(sys.argv) > 1 and sys.argv[1] != 'hard-coded-path':
        source_dir = sys.argv[1]

    if os.path.isdir(source_dir):
        processDir(source_dir)
    elif os.path.isfile(source_dir):
        path = source_dir
        source_dir = os.path.dirname(path)
        processFile(path)
    else:
        reportError("File not found: " + source_dir)

    dumpFootnoteReferences()


import re
import datetime
from .dean import Dean
from bs4 import BeautifulSoup
from freeclass.models import Classroom
from course.models import Course
from main import get_now_week

class FreeClassroom:
    session = Dean().dean_session
    week = get_now_week()
    building_dict = {
        "1": "思源楼",
        "2": "思源西楼",
        "3": "思源东楼",
        "4": "第九教学楼",
        "5": "第八教学楼",
        "6": "第五教学楼",
        "7": "第二教学楼",
        "8": "东区一教",
        "9": "东区二教",
        "10": "东教三楼",
        "11": "逸夫教学楼",
        "12": "机械楼",
        "13": "第十七号教学楼",
        "90": "科技大厦",
        "91": "天佑会堂",
        "92": "工程素质",
        "93": "综合实验楼",
        "94": "机械实验馆",
        "100": "学生活动服务中心"

    }
    status_dict = {
        '#fff': '空闲',
        '#394ed6': '考试占用',
        '#e46868': '排课占用',
        '#77bf6d': '预约',
        '#d8cc56': '个人占用'
    }

    def _get_building_room_info(self, building_no, page=1):
        url = 'https://dean.bjtu.edu.cn/classroom/timeholdresult/room_stat/'
        queries = {
            'zxjxjhh': '2019-2020-1-2',
            'zc': self.week,
            'jxlh': building_no,
            'jash': '',
            'submit': '查询+',
            'has_advance_query': '',
            'page': page,
            'perpage': 200
        }
        building = self.building_dict[str(building_no)]
        content = self.session.get(url, params=queries).content.decode()
        table = BeautifulSoup(content, 'html5lib').find('table', {'class': 'table table-bordered'})
        weekday = datetime.datetime.now().isoweekday()
        classroom_list = table.find_all('tr')[2:]
        for classroom in classroom_list:
            course_list = classroom.find_all('td', {'title': re.compile(r'星期{}'.format(weekday))})
            name = classroom.td.text.split()[0]
            room = Classroom.objects.get_or_create(name=name, building=building)[0]
            for course in course_list:
                no = int(course.get('title')[-2])
                color = course.get('style').replace('background-color: ', '')
                status = self.status_dict[color]
                if status != '空闲':
                    print(self.week, building, name, no, weekday)
                    course = Course.objects.filter(week__regex=r',{week},|^{week},|,{week}$'.format(week=self.week),
                                                   building=building, classroom=name, day_no=no,
                                                   day=weekday)
                    if course:
                        try:
                            teacher = course[0].teacher.name
                        except:
                            teacher = ''
                        status = course[0].name + '-' + teacher
                room.__setattr__('class{}'.format(no), status)
                print(name, no, status)
                room.save()

    def update(self):
        for building_no in self.building_dict.keys():
            self._get_building_room_info(building_no=building_no)

def run():
    FreeClassroom().update()


import os
import re

from traitlets import List, default

from .base import BaseConverter
from ..preprocessors import (
    InstantiateTests,
    ClearOutput,
    CheckCellMetadata
)
from traitlets.config.loader import Config
from typing import Any
from ..coursedir import CourseDirectory

class GenerateSourceWithTests(BaseConverter):

    @default("permissions")
    def _permissions_default(self) -> int:
        return 664 if self.coursedir.groupshared else 644

    @property
    def _input_directory(self) -> str:
        return self.coursedir.source_directory

    @property
    def _output_directory(self) -> str:
        return self.coursedir.source_with_tests_directory

    preprocessors = List([
        InstantiateTests,
        ClearOutput,
        CheckCellMetadata
    ]).tag(config=True)

    def _load_config(self, cfg: Config, **kwargs: Any) -> None:
        super(GenerateSourceWithTests, self)._load_config(cfg, **kwargs)

    def __init__(self, coursedir: CourseDirectory = None, **kwargs: Any) -> None:
        super(GenerateSourceWithTests, self).__init__(coursedir=coursedir, **kwargs)

    def start(self) -> None:
        old_student_id = self.coursedir.student_id
        self.coursedir.student_id = '.'
        try:
            super(GenerateSourceWithTests, self).start()
        finally:
            self.coursedir.student_id = old_student_id


# CS5487 demo script for Programming Assignment 2
#
# The script has been tested with python 2.7.6
#
# It requires the following modules:
#   numpy 1.8.1
#   matplotlib v1.3.1
#   scipy 0.14.0
#   Image (python image library)

import pa2
import numpy as np
import pylab as pl
import scipy.io as sio
from PIL import Image
from Assignment2.Problem1.K_means import *
from Assignment2.Problem1.EM import *
from Assignment2.Problem1.Mean_shift import *
from Assignment2.Problem1.P1_function import *

#import Image
def demo():
    import scipy.cluster.vq as vq

    ## load and show image
    img = Image.open('../images/12003.jpg')
    pl.subplot(1, 3, 1)
    pl.imshow(img)

    ## extract features from image (step size = 7)
    X, L = pa2.getfeatures(img, 7)

    ## Call kmeans function in scipy.  You need to write this yourself!
   # C,Y = vq.kmeans2(vq.whiten(X.T), 2, iter=1000, minit='random')

    d = 4
    K = 4
    N = X.shape[1]
    Mju = RandomMju(150, 155, K, d)
    ##########Kmeans##########
    #Y = Clustering_Kmeans(Mju, X, K, N, d, 1000, 0.000000000001)

     ##########EM#########
    #d = 4
    #K = 2
    #SIGMA = RandomSIGMA(100, 110, K, d)

    #Pi = [0.1, 0.1, 0.05, 0.1, 0.1, 0.1, 0.1, 0.1, 0.05, 0.2]
    #Y = Clustering_EM(Mju, SIGMA, Pi, X, K, d, X.shape[1], 800, 0.00000000000001)
     ###################

    #####Meanshift####
    h = 45

    maxRound = 30
    limitation = 0.00001
    cluster_gap = 8
    X_peak = Peak_MeanShift(X, d, N, h, maxRound, limitation)
    cluster = Clustering_MeanShift(X, X_peak, N, d, cluster_gap)
    Y = cluster[1]
    ##########################

    # Use matlab 1-index labeling

    # make segmentation image from labels
    segm = pa2.labels2seg(Y, L)
    pl.subplot(1, 3, 2)
    pl.imshow(segm)

    # color the segmentation image
    csegm = pa2.colorsegms(segm, img)
    picname ="seastar"
    title = "Menshift, h="+str(h)
    pl.title(title)
    pl.subplot(1, 3, 3)
    pl.imshow(csegm)
    pl.savefig('/Users/gaobrook/Desktop/Clustering/problem2/'+title+'_'+picname+'.png', dpi=200)
    pl.show()

def main():
    demo()
if __name__ == '__main__':
    main()


#A
'''import sys
import math
N,M,a,K=map(int, sys.stdin.readline().split())
print(min(N-1,a-K)+1, math.ceil((a-K)/M)+1)'''

#B
'''import sys
N=int(sys.stdin.readline())
a=[]
answer=0
for _ in range(N):
    aa,bb=map(int, sys.stdin.readline().split())
    a.append(aa)
    answer+=bb
a.sort()
for i in range(1,N+1):
    answer+=a[i-1]*i

print(answer)'''

import sys
N=int(sys.stdin.readline())
alpha=list(sys.stdin.readline().strip())
D={}
K={}
S={}
H={}
for i in range(N):
    if alpha[i]=='D':D[i]=0
    elif alpha[i]=='K':K[i]=0
    elif alpha[i]=='S':S[i]=0
    elif alpha[i]=='H':H[i]=1
for ind in S.keys():
    for ind2 in H.keys():
        if ind<ind2: S[ind]=S[ind]+H[ind2]
for ind in K.keys():
    for ind2 in S.keys():
        if ind<ind2: K[ind]=K[ind]+S[ind2]
for ind in D.keys():
    for ind2 in K.keys():
        if ind<ind2: D[ind]=D[ind]+K[ind2]
print(sum(D.values()))


import json
import re
import logging
from __init__ import app

def extract_dispatch_content(text):
    pattern = r'R9\.redux\.dispatch\((\{.+?\})\)'
    matches = re.findall(pattern, text, flags=re.DOTALL)
    return matches

def add_quotes(json_str):
    # Agregar comillas a las claves de primer nivel
    regex_keys = r'(?<={|,)\s*([a-zA-Z0-9]+)\s*:'
    result = re.sub(regex_keys, r' "\1":', json_str)

    # Reemplazamos las comillas simples por dobles
    pattern = r"(?<!\w)'(?!\w)|(?<=\w)'(?!\w)|(?<!\w)'(?=\w)"
    result = re.sub(pattern, '"', result)
    return result

def get_json_from_kayak_response(response_text):
    if response_text:
        response_json = json.loads(response_text)

        # Accede a los campos del JSON y extrae el contenido de los scripts
        scripts = response_json.get('bufferedScripts')

        # Combina todas las cadenas de los scripts en una sola cadena
        combined_scripts = ' '.join(scripts)
        # Encuentra todas las ocurrencias de 'R9.redux.dispatch(...)'
        matches = extract_dispatch_content(combined_scripts)
        # Busca el contenido en el que el valor de 'type' es 'FlightResultsList'
        flight_results_list = None
        for match in matches:
            formatted_match = add_quotes(match)
            json_data = json.loads(formatted_match)
            logging.info(json_data.get('type'))
            if 'FlightResultsList' in json_data.get('type'):
                flight_results_list = json_data.get('state').get('results')
                return flight_results_list

    app.logger.debug("No se encontró un objeto con 'type' igual a 'FlightResultsList'")


# -*- coding: utf-8 -*-
# +
import argparse
import datetime
import random
import time
import pandas as pd
from pathlib import Path

import torch
import torchvision.transforms as standard_transforms
import numpy as np

from PIL import Image
import cv2
from crowd_datasets import build_dataset
from engine import *
from models import build_model
import os
import warnings
warnings.filterwarnings('ignore')

def get_args_parser():
    parser = argparse.ArgumentParser('Set parameters for P2PNet evaluation', add_help=False)

    # * Backbone
    parser.add_argument('--backbone', default='vgg16_bn', type=str,
                        help="name of the convolutional backbone to use")

    parser.add_argument('--row', default=2, type=int,
                        help="row number of anchor points")
    parser.add_argument('--line', default=2, type=int,
                        help="line number of anchor points")

    parser.add_argument('--output_dir', default='./',
                        help='path where to save')
    parser.add_argument('--weight_path', default='./weights/SHTechA.pth',
                        help='path where the trained weights saved')

    parser.add_argument('--gpu_id', default=0, type=int, help='the gpu used for evaluation')

    return parser

def main(args, debug=False):
    start=time.time()

    os.environ["CUDA_VISIBLE_DEVICES"] = '{}'.format(args.gpu_id)

    print(args)
    device = torch.device('cuda')
    # get the P2PNet
    model = build_model(args)
    # move to GPU
    model.to(device)
    # load trained model
    if args.weight_path is not None:
        checkpoint = torch.load(args.weight_path, map_location='cpu')
        model.load_state_dict(checkpoint['model'])
    # convert to eval mode
    model.eval()
    # create the pre-processing transform
    transform = standard_transforms.Compose([
        standard_transforms.ToTensor(),
        standard_transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])
    url = "https://889962c23ee6.ap-northeast-2.playback.live-video.net/api/video/v1/ap-northeast-2.196202674046.channel.ppLGhcD0q6ZR.m3u8"
    cap=cv2.VideoCapture(url)
    fourcc = cv2.VideoWriter_fourcc(*'FMP4')

    fps = cap.get(cv2.CAP_PROP_FPS)
    w = cap.get(cv2.CAP_PROP_FRAME_WIDTH) #원본 동영상 크기 정보 받아옴
    h = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)

    out = cv2.VideoWriter('output.mp4', fourcc, 30, (int(w//128*128),int(h//128*128)))

    while True:
      hasFrame, img_frame=cap.read() #프레임 읽어옴
      if not hasFrame: #더이상 받아올 프레임 없는 경우
        break

      img_raw = Image.fromarray(img_frame).convert('RGB')
      # round the size
      width, height = img_raw.size
      new_width = width // 128 * 128
      new_height = height // 128 * 128
      img_raw = img_raw.resize((new_width, new_height), Image.ANTIALIAS)
      # pre-proccessing
      img = transform(img_raw)

      samples = torch.Tensor(img).unsqueeze(0)
      samples = samples.to(device)
      # run inference
      outputs = model(samples)
      outputs_scores = torch.nn.functional.softmax(outputs['pred_logits'], -1)[:, :, 1][0]

      outputs_points = outputs['pred_points'][0]

      threshold = 0.5
      # filter the predictions
      points = outputs_points[outputs_scores > threshold].detach().cpu().numpy().tolist()
      predict_cnt = int((outputs_scores > threshold).sum())

      outputs_scores = torch.nn.functional.softmax(outputs['pred_logits'], -1)[:, :, 1][0]

      outputs_points = outputs['pred_points'][0]
      # draw the predictions
      size = 5
      img_to_draw = cv2.cvtColor(np.array(img_raw), cv2.COLOR_RGB2BGR)
      for p in points:
          img_to_draw = cv2.circle(img_to_draw, (int(p[0]), int(p[1])), size, (0, 0, 255), -1)
      img_to_draw=cv2.cvtColor(np.array(img_to_draw), cv2.COLOR_BGR2RGB)
      out.write(img_to_draw)

      if cv2.waitKey(1) & 0xFF == ord('q'):
        break

      if predict_cnt>=5:
        print(predict_cnt)

        outputs_df = pd.DataFrame(points, columns=['width', 'height'])
        outputs_df.to_csv('tmp_points.csv', encoding='utf-8')

    print("original size: ", width, height)
    print("resized size: ", new_width, new_height)

    cap.release()
    out.release()
    print(time.time()-start)

if __name__ == '__main__':
    parser = argparse.ArgumentParser('P2PNet evaluation script', parents=[get_args_parser()])
    args = parser.parse_args()
    main(args)
# -


import datatable as dt
import pytest
import random
import re
from datatable import stype, f
from datatable.internal import frame_integrity_check
from tests import noop, random_string, assert_equals

def test_issue_1912():
    # Check that the expression `A==None` works if A is a string column
    DT = dt.Frame(A=["dfv", None, None, "adfknlkad", None])
    RES = DT[:, f.A == None]
    assert RES.to_list()[0] == [0, 1, 1, 0, 1]

#-------------------------------------------------------------------------------
# split_into_nhot
#-------------------------------------------------------------------------------

def test_split_into_nhot_noarg():
    with pytest.raises(ValueError) as e:
        noop(dt.str.split_into_nhot())
    assert ("Required parameter frame is missing" == str(e.value))

def test_split_into_nhot_none():
    f0 = dt.str.split_into_nhot(None)
    assert f0 is None

def test_split_into_nhot_empty():
    f0 = dt.str.split_into_nhot(dt.Frame(["", None]))
    assert_equals(f0, dt.Frame())

@pytest.mark.parametrize('sort', [False, True])
def test_split_into_nhot0(sort):
    f0 = dt.Frame(["cat, dog, mouse, peacock, frog",
                   "armadillo, fox, hedgehog",
                   None,
                   "dog, fox, mouse, cat, peacock",
                   "horse, raccoon, cat, frog, dog"])
    f1 = dt.str.split_into_nhot(f0, sort = sort)
    frame_integrity_check(f1)
    fr = dt.Frame({"cat":       [1, 0, None, 1, 1],
                   "dog":       [1, 0, None, 1, 1],
                   "mouse":     [1, 0, None, 1, 0],
                   "peacock":   [1, 0, None, 1, 0],
                   "frog":      [1, 0, None, 0, 1],
                   "armadillo": [0, 1, None, 0, 0],
                   "fox":       [0, 1, None, 1, 0],
                   "hedgehog":  [0, 1, None, 0, 0],
                   "horse":     [0, 0, None, 0, 1],
                   "raccoon":   [0, 0, None, 0, 1]})
    assert set(f1.names) == set(fr.names)
    if sort :
        assert list(f1.names) == sorted(fr.names)
    fr = fr[:, f1.names]
    assert f1.names == fr.names
    assert f1.stypes == (dt.stype.bool8, ) * f1.ncols
    assert f1.shape == fr.shape
    assert f1.to_list() == fr.to_list()

def test_split_into_nhot1():
    f0 = dt.Frame(["  meow  \n",
                   None,
                   "[ meow]",
                   "['meow' ,purr]",
                   '(\t"meow", \'purr\')',
                   "{purr}"])
    f1 = dt.str.split_into_nhot(f0)
    frame_integrity_check(f1)
    fr = dt.Frame(meow=[1, None, 1, 1, 1, 0], purr=[0, None, 0, 1, 1, 1])
    assert set(f1.names) == set(fr.names)
    fr = fr[..., f1.names]
    assert f1.shape == fr.shape == (6, 2)
    assert f1.stypes == (dt.stype.bool8, dt.stype.bool8)
    assert f1.to_list() == fr.to_list()

def test_split_into_nhot_sep():
    f0 = dt.Frame(["a|b|c", "b|a", None, "a|c"])
    f1 = dt.str.split_into_nhot(f0, sep="|")
    assert set(f1.names) == {"a", "b", "c"}
    fr = dt.Frame(a=[1, 1, None, 1], b=[1, 1, None, 0], c=[1, 0, None, 1])
    assert set(f1.names) == set(fr.names)
    assert f1.to_list() == fr[:, f1.names].to_list()

def test_split_into_nhot_quotes():
    f0 = dt.str.split_into_nhot(dt.Frame(['foo, "bar, baz"']))
    f1 = dt.str.split_into_nhot(dt.Frame(['foo, "bar, baz']))
    assert set(f0.names) == {"foo", "bar, baz"}
    assert set(f1.names) == {"foo", '"bar', "baz"}

def test_split_into_nhot_bad():
    f0 = dt.Frame([[1.25], ["foo"], ["bar"]])
    with pytest.raises(ValueError) as e:
        dt.str.split_into_nhot(f0)
    assert ("Function split_into_nhot() may only be applied to a single-column "
            "Frame of type string; got frame with 3 columns" == str(e.value))

    with pytest.raises(TypeError) as e:
        dt.str.split_into_nhot(f0[:, 0])
    assert ("Function split_into_nhot() may only be applied to a single-column "
            "Frame of type string; received a column of type float64" ==
            str(e.value))

    with pytest.raises(ValueError) as e:
        dt.str.split_into_nhot(f0[:, 1], sep=",;-")
    assert ("Parameter sep in split_into_nhot() must be a single character"
            in str(e.value))

@pytest.mark.parametrize("seed, st", [(random.getrandbits(32), stype.str32),
                                      (random.getrandbits(32), stype.str64)])
def test_split_into_nhot_long(seed, st):
    random.seed(seed)
    n = int(random.expovariate(0.0001) + 100)
    col1 = [random.getrandbits(1) for _ in range(n)]
    col2 = [random.getrandbits(1) for _ in range(n)]
    col3 = [random.getrandbits(1) for _ in range(n)]
    col4 = [0] * n

    data = [",".join(["liberty"] * col1[i] +
                     ["equality"] * col2[i] +
                     ["justice"] * col3[i]) for i in range(n)]

    # Introduce 1% of None's, making sure we preserve data at freedom_index
    na_indices = random.sample(range(n), n // 100)
    freedom_index = random.randint(0, n - 1)
    for i in na_indices:
        if i == freedom_index: continue
        col1[i] = None
        col2[i] = None
        col3[i] = None
        col4[i] = None
        data[i] = None
    col4[freedom_index] = 1
    data[freedom_index] += ", freedom"

    f0 = dt.Frame(data, stype=st)
    assert f0.stypes == (st,)
    assert f0.shape == (n, 1)
    f1 = dt.str.split_into_nhot(f0)
    assert f1.shape == (n, 4)
    fr = dt.Frame(liberty=col1, equality=col2, justice=col3, freedom=col4)
    assert set(f1.names) == set(fr.names)
    f1 = f1[..., fr.names]
    assert f1.to_list() == fr.to_list()

def test_split_into_nhot_view():
    f0 = dt.Frame(A=["cat,dog,mouse", "mouse", None, "dog, cat"])
    f1 = dt.str.split_into_nhot(f0[::-1, :])
    f2 = dt.str.split_into_nhot(f0[3, :])
    assert set(f1.names) == {"cat", "dog", "mouse"}
    assert f1[:, ["cat", "dog", "mouse"]].to_list() == \
           [[1, None, 0, 1], [1, None, 0, 1], [0, None, 1, 1]]
    assert set(f2.names) == {"cat", "dog"}
    assert f2[:, ["cat", "dog"]].to_list() == [[1], [1]]

def test_split_into_nhot_deprecated():
    DT = dt.Frame(["a, b, c"])
    with pytest.warns(FutureWarning):
        RES = dt.split_into_nhot(DT)
    EXP = dt.Frame([[1], [1], [1]], names=["a", "b", "c"], stype=dt.bool8)
    assert_equals(RES, EXP)


import numpy as np
from scipy import interpolate
class SkeletonASCIIPhraser(object):

    def __init__(self, export_path):

        self.__export_path = export_path

    def save(self, beams):
        file = open(self.__export_path, 'w')
        self.__plot_analytical_skeleton_beams(file, beams)
        file.close()

    def __plot_analytical_skeleton_beams(self, file, beams):
        # Save the ids and the export to the ids in a dictionary
        point_ids = {}
        vector_ids = {}
        spline_ids = {}
        curve_ids = {}
        cross_sections_ids = {}
        beam_ids = {}
        junction_connections = {}
        # ID counters
        point_id = 0
        spline_id = 0
        curve_id = 0
        vector_id = 0
        cross_section_id = 0
        beam_id = 0
        junction_id_point_list = {}
        start_end_cross_section_correspondence = {}

        for beam in beams:
            spline = beam.spline
            # calculate points
            point_id += 1
            p_jun_start_id = point_id
            point_ids[point_id] = [spline.x[0], spline.y[0], spline.z[0]]
            point_id += 1
            # Save point to point id
            point_ids[point_id] = [spline.x[-1], spline.y[-1], spline.z[-1]]
            p_jun_end_id = point_id
            # Calculate vectors
            vector_id += 1
            # Build up vector + id
            n_x1 = spline.degree * (spline.x[0] - spline.x[1])
            n_y1 = spline.degree * (spline.y[0] - spline.y[1])
            n_z1 = spline.degree *  (spline.z[0] - spline.z[1])
            norm = np.sqrt(n_x1 ** 2 + n_y1 ** 2 + n_z1 ** 2)
            norm_vec = 1.0 / norm * np.array([n_x1, n_y1, n_z1])
            vector_ids[vector_id] = [norm_vec[0], norm_vec[1], norm_vec[2]]
            vector_id += 1
            n_x1 = spline.degree * (spline.x[-1] - spline.x[-2])
            n_y1 = spline.degree * (spline.y[-1] - spline.y[-2])
            n_z1 = spline.degree * (spline.z[-1] - spline.z[-2])
            norm = np.sqrt(n_x1 ** 2 + n_y1 ** 2 + n_z1 ** 2)
            norm_vec = 1.0 / norm * np.array([n_x1, n_y1, n_z1])
            vector_ids[vector_id] = [norm_vec[0], norm_vec[1], norm_vec[2]]
            # Calculate spline and curve
            spline_id += 1
            spline_ids[spline_id] = [point_id -1, point_id, vector_id -1, vector_id]
            start_end_cross_section_correspondence[p_jun_start_id] = spline_id
            start_end_cross_section_correspondence[p_jun_end_id] = spline_id
            curve_id += 1
            curve_ids[curve_id] = [spline_id]
            # ---------- Cross section location start
            cut_times = beam.cut_times
            tck = [np.array(spline.t),
                   [np.array(spline.x), np.array(spline.y), np.array(spline.z)],
                   spline.degree]
            [x_coordinates, y_coordinates, z_coordinates] = interpolate.splev(cut_times, tck)
            [x_normals, y_normals, z_normals] = interpolate.splev(cut_times, tck, der=1)
            point_id += 1
            point_ids[point_id] = [x_coordinates[0], y_coordinates[0], z_coordinates[0]]
            # Cross section location end
            point_id += 1
            point_ids[point_id] = [x_coordinates[1], y_coordinates[1], z_coordinates[1]]
            p_c_start = point_id -1
            p_c_end = point_id

            # ----------- Cross section normal vector
            vector_id += 1
            vector_ids[vector_id] = [x_normals[0], y_normals[0], z_normals[0]]
            vector_id += 1
            vector_ids[vector_id] = [x_normals[1], y_normals[1], z_normals[1]]
            v_start = vector_id -1
            v_end = vector_id
            # Calculate beam
            beam_id += 1
            beam_id_list = []
            # Ad spline for beam
            beam_id_list.append(curve_id)
            # Add al valid cross section which are found
            # Calculate start cross sections
            if beam.crossSection1[0] != None:
                [xx, yy, zz] = beam.crossSection1[0]
                splines = []
                for id in range(len(xx)):
                    point_id += 1
                    point_ids[point_id] = [xx[id], yy[id], zz[id]]
                    if id == 0:
                        spline_start_point = point_id
                    if id != 0:
                        spline_id += 1
                        spline_ids[spline_id] = [point_id - 1, point_id]
                        splines.append(spline_id)
                    # Last point and close polygon
                    if id == len(xx) - 1:
                        spline_id += 1
                        spline_ids[spline_id] = [point_id, spline_start_point]
                        splines.append(spline_id)
                curve_id += 1
                curve_ids[curve_id] = splines
                beam_id_list.append(curve_id)
                cross_section_id += 1
                cross_sections_ids[cross_section_id] = [p_c_start, v_start, curve_id]
            # Calculate end cross sections
            if beam.crossSection2[0] != None:
                [xx, yy, zz] = beam.crossSection2[0]
                splines = []
                for id in range(len(xx)):
                    point_id += 1
                    point_ids[point_id] = [xx[id], yy[id], zz[id]]
                    if id == 0:
                        spline_start_point = point_id
                    if id != 0:
                        spline_id += 1
                        spline_ids[spline_id] = [point_id - 1, point_id]
                        splines.append(spline_id)
                    # Last point
                    if id == len(xx) - 1:
                        spline_id += 1
                        spline_ids[spline_id] = [point_id, spline_start_point]
                        splines.append(spline_id)
                curve_id += 1
                curve_ids[curve_id] = splines
                beam_id_list.append(curve_id)
                cross_section_id += 1
                cross_sections_ids[cross_section_id] = [p_c_end, v_end, curve_id]
            beam_ids[beam_id] = beam_id_list
            # Save the junction information
            end_marker = spline.end_connection_id
            start_marker = spline.start_connection_id
            if end_marker != None:
                if tuple(end_marker) in junction_connections:
                    junction_connections[tuple(end_marker)].append(cross_section_id)
                else:
                    junction_connections[tuple(end_marker)] = [cross_section_id]

                # Check if junction point is alreadcy set and then insert them
                if tuple(end_marker) in junction_id_point_list:
                    point_id_jun_end = junction_id_point_list[tuple(end_marker)] = point_id
                else:
                    point_id += 1
                    point_ids[point_id] = [end_marker[0], end_marker[1], end_marker[2]]
                    point_id_jun_end = point_id
                spline_id += 1
                spline_ids[spline_id] = [p_jun_end_id, point_id_jun_end]
                curve_id += 1
                curve_ids[curve_id] = spline_id
                # Create spline between junction and beam

            if start_marker != None:
                if tuple(start_marker) in junction_connections:
                    junction_connections[tuple(start_marker)].append(cross_section_id - 1)
                else:
                    junction_connections[tuple(start_marker)] = [cross_section_id - 1]
                    # Check if junction point is alreadcy set and then insert them

                    if tuple(start_marker) in junction_id_point_list:
                        point_id_jun_start = junction_id_point_list[tuple(start_marker)] = point_id
                    else:
                        point_id += 1
                        point_ids[point_id] = [start_marker[0], start_marker[1], start_marker[2]]
                        point_id_jun_start = point_id
                    spline_id += 1
                    spline_ids[spline_id] = [p_jun_start_id, point_id_jun_start]
                    curve_id += 1
                    curve_ids[curve_id] = spline_id
                    # Create spline between junction and beam

        file.write('POINT \n')
        for point_key in point_ids:
            if point_key in start_end_cross_section_correspondence:
                file.write(
                    f'{point_key}, {point_ids[point_key][0]}, {point_ids[point_key][1]}, {point_ids[point_key][2]},'
                    f' {start_end_cross_section_correspondence[point_key]} \n')
            else:
                file.write(f'{point_key}, {point_ids[point_key][0]}, {point_ids[point_key][1]}, {point_ids[point_key][2]} \n')
        file.write('ENDPOINT \n')

        file.write('VECTOR \n')
        for vector_key in vector_ids:
            file.write(f'{vector_key}, {vector_ids[vector_key][0]}, {vector_ids[vector_key][1]}, {vector_ids[vector_key][2]} \n')
        file.write('ENDVECTOR \n')

        file.write('SPLINE \n')
        for spline_key in spline_ids:
            if len(spline_ids[spline_key]) == 2:
                file.write(f'{spline_key}, {spline_ids[spline_key][0]}, {spline_ids[spline_key][1]} \n')
            else:
                file.write(
                    f'{spline_key}, {spline_ids[spline_key][0]}, {spline_ids[spline_key][1]}, {spline_ids[spline_key][2]}, {spline_ids[spline_key][3]} \n')
        file.write('ENDSPLINE \n')

        file.write('CURVE \n')
        for curve_key in curve_ids:
            spline_ids = curve_ids[curve_key]
            if type(spline_ids) == int:
                file.write(f'{curve_key}, {spline_ids}\n')
                continue
            if len(spline_ids) == 0: continue
            file.write(f'{curve_key}')
            for sp_id in spline_ids:
                file.write(f', {sp_id}')
            file.write("\n")
        file.write('ENDCURVE \n')

        file.write('SECTION \n')
        for cross_key in cross_sections_ids:
            file.write(f'{cross_key}, {cross_sections_ids[cross_key][0]}, {cross_sections_ids[cross_key][1]}, {cross_sections_ids[cross_key][2]}')
            file.write("\n")
        file.write('ENDSECTION \n')

        file.write('BEAM \n')
        for beam_key in beam_ids:
            file.write(f'{beam_key}')
            for beam_id_value in beam_ids[beam_key]:
                file.write(f', {beam_id_value}')
            file.write("\n")
        file.write('ENDBEAM \n')

        file.write('JUNCTION \n')
        junction_id = 0
        for junction_key, beam_ids in junction_connections.items():
            if  len(beam_ids) < 2:
                continue
            junction_id += 1
            file.write(f'{junction_id}')
            for cross_key in cross_sections_ids:
                for beam_id in beam_ids:
                    if beam_id == cross_key:
                        print(cross_key)
                        print(cross_sections_ids[cross_key][2])
                        file.write(f', {cross_sections_ids[cross_key][2]}')
            file.write("\n")
        file.write('ENDJUNCTION \n')
        file.close()


h, w = map(int, input().split())
n_guards = 0
lst = []

for k in range(h):
    lst.append(list(input()))
ww = 0
for i in range(h):
    checka = True
    checkb = True
    for j in range(w):
        if lst[i][j] == 'X':
            checka = False
        if lst[j][i] == 'X':
            checkb = False
    if (checka and checkb) == True:
        n_guards += 1
        lst[i][ww] = 'X'
        i = 0
    ww+=1

ww = 0
for i in range(h):
    checka = True
    checkb = True
    for j in range(w):
        if lst[i][j] == 'X':
            checka = False
        if lst[j][i]:
            checkb = False
    if (checka or checkb) == True:
        n_guards += 1
        i = 0
    ww += 1

print(n_guards)


from simpleimage import SimpleImage

"""
This program highlights fires in an image by identifying pixels
whose red intensity is more than INTENSITY_THRESHOLD times the
average of the red, green, and blue values at a pixel. Those
"sufficiently red" pixels are then highlighted in the image
and other pixels are turned grey, by setting the pixel red,
green, and blue values to be all the same average value.
"""

INTENSITY_THRESHOLD = 1.0
DEFAULT_FILE = 'images/greenland-fire.png'

def find_flames(filename):
    """
    This function should highlight the "sufficiently red" pixels
    in the image and grayscale all other pixels in the image
    in order to highlight areas of wildfires.
    """
    image = SimpleImage(filename)
    # TODO: your code here
    return image

def main():
    # Get file and load image
    filename = get_file()
    image = SimpleImage(filename)

    # Show the original fire
    original_fire = SimpleImage(filename)
    original_fire.show()

    grayscale_flower = grayscale(filename)
    grayscale_flower.show()

    # Show the highlighted fire
    highlighted_fire = find_flames(filename,grayscale_flower)
    highlighted_fire.show()

def get_file():
    # Read image file path from user, or use the default file
    filename = input('Enter image file (or press enter for default): ')
    if filename == '':
        filename = DEFAULT_FILE
    return filename

#-----------------------------------------------
def darker(image):
    """
    Makes image passed in darker by halving red, green, blue values.
    Note: changes in image persist after function ends.
    """
    # Demonstrate looping over all the pixels of an image,
    # changing each pixel to be half its original intensity.
    for pixel in image:
        pixel.red = pixel.red // 2
        pixel.green = pixel.green // 2
        pixel.blue = pixel.blue // 2

def compute_luminosity(red, green, blue):
    """
    Calculates the luminosity of a pixel using NTSC formula
    to weight red, green, and blue values appropriately.
    """
    return (0.299 * red) + (0.587 * green) + (0.114 * blue)

def grayscale(filename):
    """
    Reads image from file specified by filename.
    Change the image to be grayscale using the NTSC
    luminosity formula and return it.
    """
    image = SimpleImage(filename)
    for pixel in image:
        luminosity = compute_luminosity(pixel.red, pixel.green, pixel.blue)
        pixel.red = luminosity
        pixel.green = luminosity
        pixel.blue = luminosity
    return image

def find_flames(main_filename, back_filename):
    """
    Implements the notion of "redscreening".  That is,
    the image in the main_filename has its "sufficiently red"
    pixels replaced with pixel from the corresponding x,y
    location in the image in the file back_filename.
    Returns the resulting "redscreened" image.
    """
    image = SimpleImage(main_filename)
    back = SimpleImage(main_filename)
    for pixel in image:
        average = (pixel.red + pixel.green + pixel.blue) // 3
        # See if this pixel is "sufficiently" red
        if pixel.red >= average * INTENSITY_THRESHOLD:
            # If so, we get the corresponding pixel from the
            # back image and overwrite the pixel in
            # the main image with that from the back image.
            x = pixel.x
            y = pixel.y
            image.set_pixel(x, y, back.get_pixel(x, y))
    return image

if __name__ == '__main__':
    main()