from airflow import DAG
from datetime import timedelta,datetime
from airflow.utils.dates import days_ago
from airflow.providers.google.cloud.operators.dataproc import  (
    ClusterGenerator,
    DataprocCreateClusterOperator,
    DataprocDeleteClusterOperator,
    DataprocSubmitJobOperator
)
from airflow.utils.trigger_rule import TriggerRule
from airflow.operators.python_operator import PythonOperator, BranchPythonOperator
from airflow.providers.google.cloud.operators.bigquery import (
    BigQueryCreateEmptyDatasetOperator,
    BigQueryGetDatasetOperator,
    BigQueryCreateExternalTableOperator,
    BigQueryUpdateTableOperator
)
from airflow.utils.state import State
from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator

CLUSTER_NAME = 'bronze-to-silver-cluster'
REGION='us-east1'
PROJECT_ID='rent-extract-project'
PYSPARK_URI_SILVER='gs://spark-scripts-rent-project/spark_rent_extraction_bronze_to_silver.py'
PYSPARK_URI_GOLD='gs://spark-scripts-rent-project/spark_rent_extraction_silver_to_gold.py'
INIT_BUCKET="gold-init-script"
INIT_FILE="pip-install.sh"
DATASET_NAME="rent_extraction_dataset"
DATA_SAMPLE_GCS_BUCKET_NAME="rent-extraction-us"
DATA_SAMPLE_GCS_OBJECT_NAME='gold/gold_rent_extraction.parquet/*.parquet'

CLUSTER_GENERATOR_CONFIG = ClusterGenerator(
    project_id=PROJECT_ID,
    zone=REGION+"-b",
    master_machine_type="n1-standard-2",
    master_disk_size=32,
    worker_machine_type="n1-standard-2",
    worker_disk_size=32,
    num_workers=2,
    init_actions_uris=[f"gs://{INIT_BUCKET}/{INIT_FILE}"],
    metadata={"PIP_PACKAGES": "googlemaps"}
).make()

PYSPARK_JOB_SILVER = {
    "reference": {"project_id": PROJECT_ID},
    "placement": {"cluster_name": CLUSTER_NAME},
    "pyspark_job": {"main_python_file_uri": PYSPARK_URI_SILVER},
}

PYSPARK_JOB_GOLD = {
    "reference": {"project_id": PROJECT_ID},
    "placement": {"cluster_name": CLUSTER_NAME},
    "pyspark_job": {"main_python_file_uri": PYSPARK_URI_GOLD},
}

def choose_task_to_create_dataset(upstream_task_id, dag_run):
    upstream_task_state = dag_run.get_task_instance(upstream_task_id).state
    if upstream_task_state == State.FAILED:
        return "create_dataset"
    elif upstream_task_state == State.SUCCESS:
        return "create_sao_paulo_rent_analisys"

def choose_task_to_update_table(upstream_task_id, dag_run):
    upstream_task_state = dag_run.get_task_instance(upstream_task_id).state
    if upstream_task_state == State.FAILED:
        return "update_table_sao_paulo_rent_analisys"

with DAG(
    dag_id='bronze_to_silver_gold_rent_dataproc',
    start_date=datetime(2021, 1, 1),
    schedule_interval="0 0 * * *",
    catchup=False
) as dag:

    create_cluster = DataprocCreateClusterOperator(
        task_id="create_cluster",
        project_id=PROJECT_ID,
        cluster_config=CLUSTER_GENERATOR_CONFIG,
        region=REGION,
        cluster_name=CLUSTER_NAME,
        retries=0

    )

    execute_spark_bronze_to_silver_rent = DataprocSubmitJobOperator(
        task_id="execute_spark_bronze_to_silver_rent",
        job=PYSPARK_JOB_SILVER,
        region=REGION,
        project_id=PROJECT_ID,
        retries=0

    )

    execute_spark_silver_to_gold_rent = DataprocSubmitJobOperator(
        task_id="execute_spark_silver_to_gold_rent",
        job=PYSPARK_JOB_GOLD,
        region=REGION,
        project_id=PROJECT_ID,
        retries=0

    )
    delete_cluster = DataprocDeleteClusterOperator(
        task_id="delete_cluster",
        project_id=PROJECT_ID,
        cluster_name=CLUSTER_NAME,
        region=REGION,
        trigger_rule=TriggerRule.ALL_DONE,
        retries=0
    )

    get_dataset = BigQueryGetDatasetOperator(task_id="get-dataset", dataset_id=DATASET_NAME,retries=0)

    branch_task_creation = BranchPythonOperator(
        task_id='branch_task_creation',
        python_callable=choose_task_to_create_dataset,
        op_args=[get_dataset.task_id],
        trigger_rule="all_done",
        provide_context=True,
        dag=dag,
    )

    create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET_NAME,retries=0)

    create_sao_paulo_rent_analisys = GCSToBigQueryOperator(
        task_id="create_sao_paulo_rent_analisys",
        bucket=DATA_SAMPLE_GCS_BUCKET_NAME,
        source_objects=[DATA_SAMPLE_GCS_OBJECT_NAME],
        source_format='PARQUET',
        destination_project_dataset_table=f"{DATASET_NAME}.sao_paulo_rent_analisys",
        schema_fields=[
        {"name": "price", "type": "INTEGER", "mode": "REQUIRED"},
        {"name": "total_price", "type": "INTEGER", "mode": "REQUIRED"},
        {"name": "address", "type": "STRING", "mode": "REQUIRED"},
        {"name": "floor_size", "type": "INTEGER", "mode": "REQUIRED"},
        {"name": "number_of_rooms", "type": "INTEGER", "mode": "REQUIRED"},
        {"name": "number_of_bathrooms", "type": "INTEGER", "mode": "REQUIRED"},
        {"name": "latitude", "type": "FLOAT64", "mode": "REQUIRED"},
        {"name": "longitude", "type": "FLOAT64", "mode": "REQUIRED"},
        {"name": "neighborhood", "type": "STRING", "mode": "REQUIRED"}
    ],
    write_disposition="WRITE_TRUNCATE",
    retries=0,
    trigger_rule="one_success"
)

    create_cluster >> execute_spark_bronze_to_silver_rent
    execute_spark_bronze_to_silver_rent >> execute_spark_silver_to_gold_rent
    execute_spark_silver_to_gold_rent >> delete_cluster
    delete_cluster >> get_dataset
    get_dataset>>branch_task_creation
    branch_task_creation >> [create_dataset,create_sao_paulo_rent_analisys]
    create_dataset >> create_sao_paulo_rent_analisys
    create_sao_paulo_rent_analisys


import web,math,os,xlrd,conv,model,simplejson

from web import form

urls = (
    '/','index',
    '/upload', 'upload',
    '/marks','marks',
    '/markedit','markedit',
    '/reports', 'reports',
    '/delmarks', 'delmarks',
    '/stu', 'stu',
    '/stuview','stuview',
    )

app=web.application(urls,globals())

uploadform = form.Form (
    form.File('datafile'),
    # form.Dropdown('asmt',[('FA1','FA1'),('FA2','FA2'),('SA1','SA1'),('FA3','FA3'), ('FA4','FA4'), ('SA2','SA2')]),
    # form.Dropdown('year',[('2010-11','2010-11'),('2011-12','2011-12')]),
    # form.Dropdown('std',[('9','9'),('10','10')]),
    # form.Dropdown('section',[('A','A'),('B','B'),('C','C')]),
    # form.Dropdown('subject',[('Kannada','Kannada'),('Maths','Maths')]),
    # form.Textbox('per'),
    form.Button('Upload')
    )

marksform = form.Form(
    form.Dropdown('asmt',[('FA1','FA1'),('FA2','FA2'),('SA1','SA1'),('FA3','FA3'), ('FA4','FA4'), ('SA2','SA2'),('All','All')]),
    form.Dropdown('year',[('2010-11','2010-11'),('2011-12','2011-12')]),
    form.Dropdown('std',[('9','9'),('10','10')]),
    form.Dropdown('section',[('A','A'),('B','B'),('C','C')]),
    form.Dropdown('subject',[('Kannada','Kannada'),('Maths','Maths')]),
    form.Button('Submit')
    )

stuform = form.Form(
    form.Dropdown('year',[('2010-11','2010-11'),('2011-12','2011-12')]),
    form.Dropdown('std',[('9','9'),('10','10')]),
    form.Dropdown('section',[('A','A'),('B','B'),('C','C')]),
    form.Button('Submit')
    )

session=web.session.Session(app,web.session.DiskStore('sessions'),initializer={'logged':False})

render = web.template.render('templates/',
                             base="layout",
                             globals={'context':session},
                             )

class index:
    def __init__(self):
        pass
    def GET(self):
        return render.index()
    def POST(self):
        pass

class upload:
    def __init__(self): pass

    def GET(self):
        upform = uploadform()
        message = ''
        reports = model.getReports()
        return render.upload(message,upform,reports)
    def POST(self):
        disdir = os.path.join(os.getcwd(),'files')
        x= web.input(datafile={})

        # asmt = x.asmt.lower()
        # year = x.year
        # std = x.std
        # section = x.section
        # sub = x.subject
        # max = int(x.max)
        a = ('year','section','assessment','percent')

        if 'datafile' in x:
            filename = x.datafile.filename
            f = open(os.path.join(disdir,filename),'w')
            f.write(x.datafile.file.read())
            f.close()
            book = xlrd.open_workbook(os.path.join(disdir,filename))
            sheet = book.sheet_by_index(0)
            # print sheet
            for rownum in range(sheet.nrows):
                # if rownum == 0: continue
                row = sheet.row_values(rownum)
                if row[0].lower() == 'year':
                    year = row[0].lower()
                    continue
                if row[0].lower() == 'standard':
                    std = row[0].lower()
                    continue
                if row[0].lower() == 'section':
                    section = row[0].lower()
                    continue
                if row[0].lower() == 'assessment':
                    asmt = row[0].lower()
                    continue
                if row[0].lower() == 'percent':
                    per = row[0].lower()
                    continue
                if row[0].lower() == 'marks': continue
                if row[0].lower() == 'sl': continue
                name = row[1]
                adnum = row[2]
                english = float(row[3])
                seclang = float(row[4])
                maths = float(row[5])
                science = float(row[6])
                social = float(row[7])
                studentid = model.getStudentId(year,std,section,adnum,name)

                fields =  model.asmts[asmt]['fields']
                forten = conv.toten(mark,max)
                grade = conv.grade(conv.tocent(mark,max))
                per = conv.getContri(asmt,forten)
                data = [mark,max,grade,per]
                markid = model.getMarkId(studentid,sub,year,std,section)

                model.updateMark(markid,fields,data)

        raise web.seeother('/upload')

class marks:
    def __init__(self):
        pass

    def GET(self):
        mform= marksform()
        message = ''
        return render.marks(message,mform)

    def POST(self):
        form = marksform()
        q = web.input()
        render = web.template.render('templates/', base=None,globals={'context':session})
        if model.isMarkThere(q.subject,q.year,q.std,q.section) == False:
            return render.marksajax(None,None,None,None,None)

        marks = model.getMarks(q.subject,q.year,q.std,q.section,q.asmt.lower())
        field_names = model.asmts[q.asmt.lower()]['names']
        fields = model.asmts[q.asmt.lower()]['fields']
        asmt = q.asmt.lower()
        return render.marksajax(fields,field_names,marks,asmt)

class markedit:
    def __init__(self):
        pass
    def GET(self):
        q = web.input()
        render = web.template.render('templates/', base=None,globals={'context':session})
        mark = model.getAsmtMarksById(int(q.id))
        mark = mark[0]
        active= model.asmts[q.a]['fields'][0]
        return render.markedit(mark,active)

    def POST(self):
        q = web.input()
        markid = int(q.markid)
        marks = model.getMarksByMarkId(markid)
        ret = {}
        ret = {'fields':{},'id':0, 'status': False}
        render = web.template.render('templates/', base=None,globals={'context':session})

        if 'fa1_m' in q:
            fa1_m = float(q.fa1_m)
            forten = conv.toten(fa1_m,marks['fa1_o'])
            grade = conv.grade(conv.tocent(fa1_m,marks['fa1_o']))
            per = conv.getContri('fa1',forten)
            fields = model.asmts['fa1']['fields']
            data= [fa1_m,marks['fa1_o'],grade,per]
            model.updateMark(markid,fields,data)
            ret['fields'] = {'fa1_m':fa1_m,'fa1_g':grade,'fa1_p': per}
            ret['id'] = markid
            ret['status'] = True

        if 'fa2_m' in q:
            fa2_m = float(q.fa2_m)
            forten = conv.toten(fa2_m,marks['fa2_o'])
            grade = conv.grade(forten)
            per = conv.getContri('fa2',forten)
            fields = model.asmts['fa2']['fields']
            data = [fa2_m,marks['fa2_o'],grade,per]
            model.updateMark(markid,fields,data)
            ret['fields'] = {'fa2_m':fa2_m,'fa2_g':grade,'fa2_p': per}
            ret['id'] = markid
            ret['status'] = True

        if 'sa1_m' in q:
            sa1_m = float(q.sa1_m)
            forten = conv.toten(sa1_m,marks['sa1_o'])
            grade = conv.grade(forten)
            per = conv.getContri('sa1_m',forten)
            fields = model.asmts['sa1']['fields']
            data= [sa1_m,marks['sa1_o'],grade,per]
            model.updateMark(markid,fields,data)
            ret['fields'] = {'sa1_m':sa1_m,'sa1_g':grade,'sa1_p': per}
            ret['id'] = markid
            ret['status'] = True

        if 'fa3_m' in q:
            fa3_m = float(q.fa3_m)
            forten = conv.toten(fa3_m,marks['fa3_o'])
            grade = conv.grade(forten)
            per = conv.getContri('fa3_m',forten)
            fields = model.asmts['fa3']['fields']
            data= [fa3_m,marks['fa3_o'],grade,per]
            model.updateMark(markid,fields,data)
            ret['fields'] = {'fa3_m':fa3_m,'fa3_g':grade,'fa3_p': per}
            ret['id'] = markid
            ret['status'] = True

        if 'fa4_m' in q:
            fa4_m = float(q.fa4_m)
            forten = conv.toten(fa4_m,marks['fa4_o'])
            grade = conv.grade(forten)
            per = conv.getContri('fa4_m',forten)
            fields = model.asmts['fa4']['fields']
            data=[fa4_m,marks['fa4_o'],grade,per]
            model.updateMark(markid,fields,data)
            ret['fields'] = {'fa4_m':fa4_m,'fa4_g':grade,'fa3_p': per}
            ret['id'] = markid
            ret['status'] = True

        if 'sa2_m' in q:
            sa2_m = float(q.sa2_m)
            forten = conv.toten(sa2_m,marks['sa2_o'])
            grade = conv.grade(forten)
            per = conv.getContri('sa2_m',forten)
            fields = model.asmts['sa2']['fields']
            data= [sa2_m,marks['sa2_o'],grade,per]
            model.updateMark(markid,fields,data)
            ret['fields'] = {'sa2_m':fa4_m,'sa2_g':grade,'sa23_p': per}
            ret['id'] = markid
            ret['status'] = True

        web.header("Content-Type","text/json;charset=utf-8")
        return render.savemark(simplejson.dumps(ret))

class reports:
    def __init__(self):
        pass
    def GET(self):
        reports = model.getAllReports()
        return render.reports(reports)

class delmarks:
    def __init__(self):
        pass
    def GET(self):
        q = web.input()
        ret = {}
        ret['done'] = False
        count = model.delMarks(q.year,q.sec,q.sub)
        if count > 0:
            ret['done'] = True
            ret['count'] = count

        render = web.template.render('templates/', base=None,globals={'context':session})
        web.header("Content-Type","text/json;charset=utf-8")
        return render.delmarks(simplejson.dumps(ret))

class stu:
    def __init__(self):
        pass
    def GET(self):
        sform = stuform()
        return render.stu(sform)
    def POST(self):
        q = web.input()
        students = model.getStudents(q.year,q.std,q.section)
        render = web.template.render('templates/', base=None,globals={'context':session})
        return render.stuajax(students)

class stuview:
    def __init__(self):
        pass
    def GET(self):
        q = web.input()
        marks = model.getStudentMarks(q.id,q.sub)
        return render.stuviewajax(marks[0])
    def POST(self):
        pass

if __name__=='__main__': app.run()


from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as exp
from selenium.webdriver.common.by import By
from selenium  import webdriver
from selenium import webdriver

class Base:

    def __init__(self,driver:webdriver.Chrome):
        #driver如果不加：webdriver.Firefox就是传的形参，加了就是映射了webdriver这个类，其他方法可以直接调用
        self.driver = driver
        self.timeout = 10
        self.t = 0.5

    # 重写元素定位方法 新加2020-1
    def find_element(self, *loc):
        #        return self.driver.find_element(*loc)
        try:
            # 确保元素是可见的。
            # 注意：以下入参为元组的元素，需要加*。Python存在这种特性，就是将入参放在元组里。
            #            WebDriverWait(self.driver,10).until(lambda driver: driver.find_element(*loc).is_displayed())
            # 注意：以下入参本身是元组，不需要加*
            WebDriverWait(self.driver, 10).until(exp.visibility_of_element_located(loc))
            return self.driver.find_element(*loc)
        except:
            print(u"%s 页面中未能找到 %s 元素" % (self, loc))

    def findelement_new(self, locator, timeout = 10, t = 0.5,):
        #定位到元素返回元素对象，没有定位到元素，返回timeout异常
        element = WebDriverWait(self.driver,timeout,t).until(exp.presence_of_element_located(locator))
        return element

    def findelement(self, locator, timeout = 10, t = 0.5,):
        # 抓取单个元素
        element = WebDriverWait(self.driver,timeout,t).until(lambda x: x.find_element(*locator))
        return element

    def findelements(self, loctor, timeout = 10, t = 0.5,):
        # 抓取一组元素
        try:
            elements = WebDriverWait(self.driver,timeout,t).until(lambda x: x.find_elements(*loctor))
            return elements
        except:
             return []

    def sendkeys(self,loctor,text,is_clear=False):
        # 找到元素并输入
        if is_clear:
            ele = self.findelement(loctor)
            ele.clear()
            ele.send_keys(text)
        # 转True的时候执行上面：
        else:
            ele = self.findelement(loctor)
            ele.send_keys(text)

    def click(self,loctor):
        # 找到元素并点击
        ele = self.findelement(loctor)
        ele.click()

    def clear(self,loctor):
        # 找到元素并清除
        ele = self.findelement(loctor)
        ele.clear()

    def isselected(self,loctor):
        # 找到元素判断是否为选中状态，包括多选框、radio、select下拉框等，返回bool值：true or false
        ele = self.findelement(loctor)
        r = ele.is_selected()
        return r

    def isdisplayed(self,loctor):
        # 判断元素是否存在，返回bool值：true or false
        ele = self.findelement(loctor)
        r = ele.is_displayed()
        return r

    def result(self,loctor):
        # 判断是否被全选中
        try:
            r = []
            all = self.findelements(loctor)
            for i in all:
                if not i.is_selected():
                    i.click()
                    r.append(i.is_selected())
                else:
                    r.append(i.is_selected())
            return r
        except:
            return False

    def is_title(self,_title,timeout = 10, t = 0.5):
        '''title中预期结果与实际传入driver是否相等'''
        try:
            result = WebDriverWait(self.driver, timeout, t).until(exp.title_is(_title))
            return result
        except:
            return False

    def is_title_contains(self,_title,timeout = 10, t = 0.5):
        #title中预期结果是否包含driver实际结果的数据
        try:
            result = WebDriverWait(self.driver, timeout, t).until(exp.title_contains(_title))
            return result
        except:
            return False

    def text_in_to_element(self,locator,_text,timeout = 10, t = 0.5):
        try:
        #判断某个元素中的text是否包含了预期的字符串
            result = WebDriverWait(self.driver, timeout, t).until(exp.text_to_be_present_in_element(locator,_text))
            return result
        except:
            return False

    def text_to_present_in_element_value(self,locator,_value,timeout = 10, t = 0.5):
        try:
        #判断某个元素中的value属性是否 包含了预期的字符串
        #返回bool值，values为空时，返回False
            result = WebDriverWait(self.driver, timeout, t).until(exp.text_to_be_present_in_element_value(locator,_value))
            return result
        except:
            return False

# if __name__== "__mian__":
#     driver = webdriver.Firefox()
#     driver.maximize_window()
#     driver.get("http://66.16.17.180:8001/login.html")
#     findele = Base()
#     findele.sendkeys(("id","loginUsername"),"admin",True)


# -*- coding: utf-8 -*-
import openai
import datetime
import time
import traceback
import json
import os
import pandas as pd

from pathlib import Path as path
from tqdm import tqdm
from collections import OrderedDict

from data_utils import load_data, dump_data
from openai_api import get_response_chatcompletion
from openai_apikey import api_key
from data_manager import DataManager
from settings import (KEYWORD_FOLD,
                      KEYWORD_QUERY_FILE_JSONL,
                      KEYWORD_FILTER_FILE_JSONL)

openai.api_key = api_key

sentence_sep = '。？！.?!．'

keyword_query_prompt = '''
请从文章中抽取出所有的航空航天领域的科学技术术语，以列表形式给出。
输出格式：
- xxx
- xxx
样例：
- 涡轮喷气式发动机
- 超低空飞行
- 神舟十二号载人飞船
- 近地卫星导航系统
文章：
'''.strip()

keyword_filter_prompt = '''请判断下列词语是否属于航天航空领域。输出两行，以逗号分割。
输出格式：
是: xxx,xxx,xxx
否: xxx,xxx,xxx
词语：
'''.strip()

class KeywordQueryer:
    def __init__(self,
                 prompt,
                 engine='gpt-3.5-turbo',
                 max_query_len=1000,
                 max_ans_token=2048,
                 retry_time=3,
                 save_keyword=True,
                 ) -> None:
        print(f'prompt\n{prompt}')
        self._prompt = prompt
        self._engine = engine
        self._max_query_len = max_query_len
        self._max_ans_token = max_ans_token
        self._retry_time = retry_time
        self._save_keyword = save_keyword

    def _get_response(self, content):
        message_list = [
            {'role': 'user', 'content': f'{self._prompt}\n{content}'},
        ]

        output_keyword = get_response_chatcompletion(
            messages=message_list,
            engine=self._engine,
            max_tokens=self._max_ans_token,
            retry_time=self._retry_time,
        )
        output_keyword = output_keyword.split('\n')
        output_keyword = filter(lambda x:len(x)>2 and x[0]=='-', output_keyword)
        output_keyword = map(lambda x:x[1:].strip(), output_keyword)
        return list(output_keyword)

    def _clip_content(self, content):
        def clip_sentence(sentence):
            return [sentence[p:p+self._max_query_len]
                    for p in range(0, len(sentence), self._max_query_len)]

        sentences = []
        cur_sentence = ''
        for d in content:
            cur_sentence += d
            if d in sentence_sep:
                sentences.extend(clip_sentence(cur_sentence))
                cur_sentence = ''
        if cur_sentence:
            sentences.extend(clip_sentence(cur_sentence))

        cur_content = ''
        for sen in sentences:
            if len(cur_content+sen) > self._max_query_len:
                yield cur_content
                cur_content = sen
            else:
                cur_content += sen
        yield cur_content

    def get_new_keywords(self, keys, contents):
        record = load_data(KEYWORD_QUERY_FILE_JSONL, default={}, merge_jsonl=True)

        todo_lst = []
        for key, content in zip(keys, contents):
            content = content.strip()
            if content and key not in record:
                todo_lst.append([key, content])
        todo_lst.sort()

        print('\n=== openai processing ===\n')
        for key, content in tqdm(todo_lst, desc='query keywords'):
            new_keywords = []
            for cliped_content in self._clip_content(content):
                cur_keywords = self._get_response(cliped_content)
                new_keywords.extend(cur_keywords)

            # record[key] = new_keywords
            if self._save_keyword:
                # dump_data(KEYWORD_QUERY_FILE_JSON, record, 'w', indent=4)

                if new_keywords:
                    dump_data(KEYWORD_QUERY_FILE_JSONL, {key:new_keywords})

class KeywordFilter:
    def __init__(self,
                 prompt,
                 engine='gpt-3.5-turbo',
                 max_filter_cnt=100,
                 max_query_len=None,
                 max_ans_token=2048,
                 retry_time=3,
                 save_keyword=True,
                 ) -> None:
        print(f'prompt\n{prompt}')
        self._prompt = prompt
        self._engine = engine
        self._max_filter_cnt = max_filter_cnt
        self._max_ans_token = max_ans_token
        self._retry_time = retry_time
        self._save_keyword = save_keyword

        self._max_query_len = max_query_len if max_query_len else 1000

    def _get_response(self, content):
        message_list = [
            {'role': 'user', 'content': f'{self._prompt}\n{content}'},
        ]

        output_filter = get_response_chatcompletion(
            messages=message_list,
            engine=self._engine,
            max_tokens=self._max_ans_token,
            retry_time=self._retry_time,
        )
        output_filter = output_filter.split('\n')
        output_res = {}

        def split_line(line):
            words = line[2:].split(',')
            words = map(lambda x:x.strip(), words)
            words = filter(lambda x:x, words)
            return list(words)

        for line in output_filter:
            line = line.strip()
            if not line:
                continue
            if line[0] == '是':
                output_yes = split_line(line)
                for w in output_yes:
                    output_res[w] = True
            if line[0] == '否':
                output_no = split_line(line)
                for w in output_no:
                    output_res[w] = False
        return output_res

    def _clip_content(self, keywords):
        content = '\n'.join(keywords)
        if len(content) <= self._max_query_len:
            yield content
            return
        cur_content = ''
        for k in keywords:
            if len(cur_content+'\n'+k) > self._max_query_len:
                yield cur_content
                cur_content = k
            else:
                cur_content += '\n'+k
        yield cur_content

    def filter_keywords(self, todo_keywords, deduplicate=False):
        # record_filter = load_data(KEYWORD_FILTER_FILE_JSONL, default={}, merge_jsonl=True)
        if deduplicate:
            todo_keywords = sorted(set(todo_keywords))

        print('\n=== openai processing ===\n')
        for p in tqdm(list(range(0, len(todo_keywords), self._max_filter_cnt)), desc='filter keywords'):
            cur_keywords = todo_keywords[p:p+self._max_filter_cnt]
            for content in self._clip_content(cur_keywords):
                filter_res = self._get_response(content)
                # record_filter.update(filter_res)
                if self._save_keyword:
                    # dump_data(KEYWORD_FILTER_FILE_JSON, record_filter, 'w', indent=4)
                    for k, v in filter_res.items():
                        dump_data(KEYWORD_FILTER_FILE_JSONL, {k:v})

class KeywordManager:

    @staticmethod
    def keyword_excel2txt():
        keyword_tot = 0

        for file in os.listdir(KEYWORD_FOLD):
            file = KEYWORD_FOLD/file
            if file.suffix not in ['.xls', '.xlsx']:
                continue
            file_excel = file
            file_txt = file.with_suffix('.txt')
            if file_txt.exists():
                continue

            keyword_yes = []

            for sheet_name in pd.ExcelFile(file_excel).sheet_names:
                df_keyword = pd.read_excel(file_excel, header=None, sheet_name=sheet_name)
                # print(file_excel, sheet_name)
                keyword_tot += df_keyword.shape[0]
                for sym in ['是', '1', 1]:
                    df_keyword__ = df_keyword.loc[df_keyword.iloc[:, 1]==sym]
                    df_keyword__ = df_keyword__.iloc[:, 0]
                    keyword_yes.extend(df_keyword__.tolist())

            # for sym in ['是', '1', 1]:
            #     df_keyword = pd.read_excel(file_excel, header=None)
            #     df_keyword = df_keyword.loc[df_keyword.iloc[:, 1]==sym]
            #     df_keyword = df_keyword.iloc[:, 0]
            #     keyword_yes.extend(df_keyword.tolist())

            str_keyword = '\n'.join(map(str, keyword_yes))
            dump_data(file_txt, str_keyword, 'w')
            print(f'{keyword_tot} keyword\n'
                  f'{len(keyword_yes)} yes\n'
                  f'from {file_excel} to {file_txt}\n')

        return keyword_tot

    @staticmethod
    def get_final_keywords():
        return DataManager.keyword_final()

def main_query_new_keywords():
    queried_keywords = load_data(KEYWORD_QUERY_FILE_JSONL, default={}, merge_jsonl=True)
    keyword_queryer = KeywordQueryer(prompt=keyword_query_prompt)
    # for k in list(queried_keywords.keys()):
    #     if not queried_keywords[k]:
    #         del queried_keywords[k]
    # dump_data(KEYWORD_QUERY_FILE_JSON, queried_keywords, mode='w', indent=4)

    baike_urls = []
    baike_passage = []
    with open('./dataspace/baike.all_crawled_info.jsonl', 'r', encoding='utf-8')as f:
        for line in f.readlines():
            piece = json.loads(line)
            if piece['url'] not in queried_keywords:
                baike_urls.append(piece['url'])
                baike_passage.append(piece['content'])
    keyword_queryer.get_new_keywords(keys=baike_urls, contents=baike_passage)

    zhidao_urls = []
    zhidao_passage = []
    with open('./dataspace/zhidao.all_crawled_info.jsonl', 'r', encoding='utf-8')as f:
        for line in f.readlines():
            piece = json.loads(line)
            if piece['url']+'@@1' not in queried_keywords:
                zhidao_urls.append(piece['url']+'@@1')
                zhidao_passage.append(piece['content1'])
            if piece['url']+'@@2' not in queried_keywords:
                zhidao_urls.append(piece['url']+'@@2')
                zhidao_passage.append(piece['content2'])
    keyword_queryer.get_new_keywords(keys=zhidao_urls, contents=zhidao_passage)

def main_filter_new_keywords():
    filter_todo = DataManager.keyword_filter_k_todo()
    filter_ = KeywordFilter(
        prompt=keyword_filter_prompt,
    )
    filter_.filter_keywords(filter_todo)

def main_excel2txt_manual_todo(part_size=(), sheet_names=()):
    keyword_excel2txt_tot = KeywordManager.keyword_excel2txt()

    assert part_size
    if keyword_excel2txt_tot != sum(part_size):
        print('\n>>> check the number of keywords from excel to notebook <<<\n'
              '>>> which differs from the number of todo keywords <<<\n')
    print('excel to txt done\n')

    if not sheet_names:
        sheet_names = range(1, len(part_size)+1)
    assert len(part_size) == len(sheet_names)

    manual_todo = DataManager.keyword_manual_k_todo()[:sum(part_size)]
    if len(manual_todo) != sum(part_size):
        print('\n>>> not enough todo keywords <<<\n')

    excel_file = path(f'./dataspace/data_keyword_{str(datetime.date.today())}.xlsx')
    if excel_file.exists():
        os.remove(excel_file)

    needle = 0
    for p, (size, sheet_name) in enumerate(zip(part_size, sheet_names)):
        df = pd.DataFrame(manual_todo[needle:needle+size])
        needle += size
        with pd.ExcelWriter(excel_file, mode=('a' if p else 'w'))as f:
            df.to_excel(f, sheet_name=str(sheet_name), index=False, header=False)
    print(f'{len(manual_todo)} manual todo keywords prepared')

if __name__ == '__main__':
    main_excel2txt_manual_todo([1000]*2)

    # sentence = '''看我的收藏0有用+1已投票0轨道器播报编辑锁定讨论上传视频特型编辑太空拖船轨道器是指往来于航天站与空间基地之间的载人或无人飞船。它的主要用途是更换、修理航天站上的仪器设备。补给消耗品，从航天站取回资料和空间加工的产品等。由于它专门来往于各个空间站，又被称为“太空拖船”。中文名轨道器别名太空拖船往返航天站与空间基地用途更换、修理航天站上的仪器设备相关视频查看全部轨道飞行器分为两种。一种是活动范围较小的，叫做轨道机动飞行器；另一种是在大范围内实行轨道转移的，成为轨道转移飞行器。可以搭载七名宇航员在太空至少逗留10天。机舱内有三层甲板：飞行甲板、中甲板和设有生命支持系统的底层甲板。轨道飞行器长37.24米，高17.27米，翼展29.79米，载荷舱尺寸18.3*4.6米，轨道速度每小时28800公里，可容忍温度1500摄氏度，轨道高度185公里至1000公里，持续时间10至16天。词条图册更多图册'''
    # kq = KeywordQueryer(save_keyword=False)
    # print(kq.get_new_keywords(['123'], [sentence]))

    # kf = KeywordFilter(save_keyword=False)
    # kf.filter_keywords

    # KeywordManager.keyword_excel2txt()

    # all_keywords = KeywordManager.get_all_keywords()
    # print(len(all_keywords))

    pass


import numpy as np

data = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], [[11, 12, 13], [14, 15, 16], [17, 18, 19]], [[21, 22, 23], [24, 25, 26], [27, 28, 29]]], np.int32)

with open('test1.dat', 'w') as outfile:
    # I'm writing a header here just for the sake of readability
    # Any line starting with "#" will be ignored by numpy.loadtxt
    outfile.write('# Array shape: {0}\n'.format(data.shape))

    # Iterating through a ndimensional array produces slices along
    # the last axis. This is equivalent to data[i,:,:] in this case
    for data_slice in data:
        print(data_slice)
        print('\n')
        # The formatting string indicates that I'm writing out
        # the values in left-justified columns 7 characters in width
        # with 2 decimal places.
        np.savetxt(outfile, data_slice, fmt='%d')

        # Writing out a break to indicate different slices...
        outfile.write('# New slice\n')

with open('test2.dat', 'w') as outfile:
    outfile.write('# Array shape: {0}\n'.format(data.shape))
    iSlice_Z = 0
    for data_slice in data:
        iSlice_Y = 0
        for data_slice_sub in data_slice:
            iData = 0
            for data in data_slice_sub:
                data_to_write = str(iData) + ' ' + str(iSlice_Y) + ' ' + str(iSlice_Z) + ' ' + str(data) + '\n'
                outfile.write(data_to_write)
                iData = iData + 1
            iSlice_Y = iSlice_Y + 1
        iSlice_Z = iSlice_Z + 1

#np.save('test.npy', arrayX, allow_pickle=False, fix_imports=True)
#Y=np.load('test.npy')
#print('Y:',Y)
#print('Y[0,:]:',Y[0][1][1])


# https://docs.micropython.org/en/latest/esp8266/tutorial/network_tcp.html#http-get-request

def http_get(url):
    _, _, host, path = url.split('/', 3)
    addr = socket.getaddrinfo(host, 80)[0][-1]
    s = socket.socket()
    s.connect(addr)
    s.send(bytes('GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n' % (path, host), 'utf8'))
    while True:
        data = s.recv(100)
        if data:
            print(str(data, 'utf8'), end='')
        else:
            break
    s.close()


""" Script for spy Gitlab.com """
import sys
import requests
from bs4 import BeautifulSoup

URL_GITLAB = 'https://about.gitlab.com/features/'
TITLE_OF_FREE_PRODUCTS = "Available in GitLab SaaS Free"
TITLE_OF_ENTERPRISE_PRODUCTS = "Not available in SaaS Free"

def process_for_gitlab(url_gitlab):
    """ Return number of products on Gitlab.com """
    gitlab_html = requests.get(url_gitlab)
    gitlab_html = gitlab_html.text
    gitlab_soup = BeautifulSoup(gitlab_html, features='html.parser')

    links_on_free_products = gitlab_soup.find_all(
        "a", attrs={"title": TITLE_OF_FREE_PRODUCTS}
    )
    links_on_enterprise_products = gitlab_soup.find_all(
        "a", attrs={"title": TITLE_OF_ENTERPRISE_PRODUCTS}
    )

    num_free_products = len(links_on_free_products)
    num_enterprise_products = len(links_on_enterprise_products)

    return num_free_products, num_enterprise_products

def print_result_gitlab():
    """ Print result of process_for_gitlab """
    num_free_products, num_enterprise_products = process_for_gitlab(URL_GITLAB)
    print(f'free products: {num_free_products}')
    print(f'enterprise products: {num_enterprise_products}')

if __name__ == "__main__":
    if 'gitlab' in sys.argv and len(sys.argv) == 2:
        print_result_gitlab()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Aug  1 15:35:38 2022

@author: diegorodriguez
"""
from csv import reader
from random import seed
from random import randrange

# Load a CSV file
def load_csv(filename):
	dataset = list()
	with open(filename, 'r') as file:
		csv_reader = reader(file)
		for row in csv_reader:
			if not row:
				continue
			dataset.append(row)
	return dataset

# Convert string column to float
def str_column_to_float(dataset, column):
	for row in dataset:
		row[column] = float(row[column].strip())

# Convert string column to integer
def str_column_to_int(dataset, column):
	class_values = [row[column] for row in dataset]
	unique = set(class_values)
	lookup = dict()
	for i, value in enumerate(unique):
		lookup[value] = i
	for row in dataset:
		row[column] = lookup[row[column]]
	return lookup

# Split a dataset into a train and test set
def train_test_split(dataset, split=0.60):
	train = list()
	train_size = split * len(dataset)
	dataset_copy = list(dataset)
	while len(train) < train_size:
		index = randrange(len(dataset_copy))
		train.append(dataset_copy.pop(index))
	return train, dataset_copy

#make predictions
def predict_pct(row, weights):
    activation = weights[0]
    for i in range(len(row) - 1):
        activation+= weights[i + 1] * row[i]
    return 1.0 if activation >=0.0 else 0.0

#estimate coefficients using gradient descent
def train_coef(train, l_rate, n_epoch):
    weights = [0.0 for i in range(len(train[0]))]
    #iterating over n_epochs
    for epoch in range(n_epoch):
        sum_error = 0.0
        #iterating over rows
        for row in train:
            yhat = predict_pct(row, weights)
            error = row[-1] - yhat
            sum_error += error**2
            weights[0] = weights[0] + l_rate * error
            #iterating over weights
            for i in range(len(row) - 1):
                weights[i + 1] = weights[i + 1] + l_rate * error * row[i]
                print('>epoch=%d, lrate=%.3f, error=%.3f' % (epoch, l_rate, sum_error))
    return weights

# Estimate Perceptron weights using stochastic gradient descent
def train_weights(train, l_rate, n_epoch):
	weights = [0.0 for i in range(len(train[0]))]
	for _ in range(n_epoch):
		for row in train:
			prediction = predict_pct(row, weights)
			error = row[-1] - prediction
			weights[0] = weights[0] + l_rate * error
			for i in range(len(row)-1):
				weights[i + 1] = weights[i + 1] + l_rate * error * row[i]
	return weights

seed(1)
# load and prepare data
filename = 'sonar.all-data.csv'
dataset = load_csv(filename)
for i in range(len(dataset[0])-1):
	str_column_to_float(dataset, i)
# convert string class to integers
str_column_to_int(dataset, len(dataset[0])-1)
train, test = train_test_split(dataset)
# evaluate algorithm
l_rate = 0.01
n_epoch = 5
weights = train_coef(test, l_rate, n_epoch)
print(weights)

for row in test:
  prediction = predict_pct(row, weights)
  print("Expected=%d, Predicted=%d" % (row[-1], prediction))


import sys
from collections import deque

N = int(sys.stdin.readline().rstrip())
stack = list(map(int, sys.stdin.readline().split()))

maximum = -1
big_group = deque()
result = deque()

while stack:
    current = stack.pop()
    if current > maximum:
        result.appendleft(-1)
        maximum = current
    else:
        has_b = False
        for b in big_group:
            if current < b:
                result.appendleft(b)
                has_b = True
                break
        if not has_b:
            result.appendleft(maximum)

    big_group.appendleft(current)

print(*result)


from PIL import Image, ImageDraw, ImageFont

image = Image.open("image.png")

draw = ImageDraw.Draw(image)

font = ImageFont.truetype("arial.ttf", size = 30)

name = "Матвеев Кирилл"

x = 150
y = 275

draw.text((x,y), name, font = font, fill = (0, 0, 0))

image.save("image_with_name.png")

www/resources/python/script for image before school/main.py


#!/usr/bin/env python3

import fileinput

from rich import print
from rich.traceback import install

install(show_locals=True)

def process_part1(lines: list[str]) -> str:
    total = 0
    for line in lines:
        first, second = line.split(",")
        first_start, first_end = first.split("-")
        second_start, second_end = second.split("-")
        first_range = set(list(range(int(first_start), int(first_end) + 1)))
        second_range = set(list(range(int(second_start), int(second_end) + 1)))
        total += first_range.issuperset(second_range) or second_range.issuperset(first_range)
    return str(total)

def process_part2(lines: list[str]) -> str:
    total = 0
    for line in lines:
        first, second = line.split(",")
        first_start, first_end = first.split("-")
        second_start, second_end = second.split("-")
        first_range = set(list(range(int(first_start), int(first_end) + 1)))
        second_range = set(list(range(int(second_start), int(second_end) + 1)))
        if first_range & second_range:
            total += 1
    return str(total)

def test_part1():
    lines = [line.rstrip("\n") for line in fileinput.input("input-testing.txt")]
    solution = process_part1(lines)
    assert solution == "2"
    print("testing part 1: ✓")

def test_part2():
    lines = [line.rstrip("\n") for line in fileinput.input("input-testing.txt")]
    solution = process_part2(lines)
    assert solution == "4"
    print("testing part 2: ✓")

def solve_part1():
    lines = [line.rstrip("\n") for line in fileinput.input("input-solving.txt")]
    solution = process_part1(lines)
    print(f"solving part 1: {solution}")

def solve_part2():
    lines = [line.rstrip("\n") for line in fileinput.input("input-solving.txt")]
    solution = process_part2(lines)
    print(f"solving part 2: {solution}")

def main():
    test_part1()
    solve_part1()
    test_part2()
    solve_part2()

if __name__ == "__main__":
    main()


'''
Created on 24.03.2011

@author: michi
'''
from PyQt4.QtGui import QItemDelegate

from sqlalchemy import Table
from sqlalchemy.sql import Alias,Select

from ems import qt4

class ColumnSectionMapper(object):
    def __init__(self,alchemySelect=None, parent=None):
        self.__columnConfigs = []
        self.__columnConfigIdByName = {}
        self.__alchemySelect = alchemySelect
        self.__delegate = MapperDelegate(self,parent)
        pass

    def addColumn(self,columnName,translatedName=None, delegate=None):
        if self.__columnConfigIdByName.has_key(columnName):
            raise KeyError("Column %s already assigned" % columnName)
        index = len(self.__columnConfigs)
        self.__columnConfigs.append({'name':columnName,
                                     'translatedName':translatedName,
                                     'delegate':delegate})
        self.__columnConfigIdByName[columnName] = index

    @property
    def translatedColumnNames(self):
        names = {}
        for config in self.__columnConfigs:
            names[config['name']] = config['translatedName']

        return names

    def __extractTablesFormSelect(self,alchemySelect):
        tableDict = {}
        for fromCond in alchemySelect.locate_all_froms():
            if isinstance(fromCond, Table):
                tableDict[str(fromCond.name)] = fromCond
            elif isinstance(fromCond,Alias):
                if isinstance(fromCond.original,Table):
                    tableDict[str(fromCond.name)] = fromCond
        return tableDict

    def getDelegate(self):
        return self.__delegate

    def getColConfig(self, column):
        if isinstance(column, int):
            index = column
        else:
            index = self.__columnConfigIdByName[unicode(column)]
        return self.__columnConfigs[index]

    def getSelectColumns(self, alchemySelect=None):
        if alchemySelect is None:
            alchemySelect = self.__alchemySelect
        if not isinstance(alchemySelect, Select):
            raise TypeError("alchemySelect has to be instanceof sqlalchemy.select")
        tableDict = self.__extractTablesFormSelect(alchemySelect)
        columnList = []
        for config in self.__columnConfigs:
            tableName,colName = config['name'].split('.')
            if tableDict.has_key(tableName):
                columnList.append(tableDict[tableName].c[colName])
        return columnList

class MapperDelegate(QItemDelegate):

    def __init__(self, mapper, parent=None):
        super(MapperDelegate, self).__init__(parent)
        self.__mapper = mapper

    def getDelegate(self, index):
        colName = index.data(qt4.ColumnNameRole).toString()
        delegate = self.__mapper.getColConfig(colName)['delegate']
        return delegate

    def paint(self, painter, option, index):
        delegate = self.getDelegate(index)
        if delegate is not None:
            delegate.paint(painter, option, index)
        else:
            QItemDelegate.paint(self, painter, option, index)

    def createEditor(self, parent, option, index):
        delegate = self.getDelegate(index)
        if delegate is not None:
            return delegate.createEditor(parent, option, index)
        else:
            return QItemDelegate.createEditor(self, parent, option,
                                              index)

    def setEditorData(self, editor, index):
        delegate = self.getDelegate(index)
        if delegate is not None:
            delegate.setEditorData(editor, index)
        else:
            QItemDelegate.setEditorData(self, editor, index)

    def setModelData(self, editor, model, index):
        delegate = self.getDelegate(index)
        if delegate is not None:
            delegate.setModelData(editor, model, index)
        else:
            QItemDelegate.setModelData(self, editor, model, index)


import os

class Image:
    def __init__(self, search_tag, img_id, tags, sample_url, sample_width, sample_height, jpeg_url, jpeg_width, jpeg_height):
        self.search_tag = search_tag
        self.img_id = img_id
        self.tags = tags
        self.sample_url = sample_url
        self.sample_width = int(sample_width)
        self.sample_height = int(sample_height)
        self.jpeg_url = jpeg_url
        self.jpeg_width = int(jpeg_width)
        self.jpeg_height = int(jpeg_height)

    def is_hd(self):
        if self.sample_width == self.jpeg_height:
            return False
        else:
            return True

    def get_local_path(self):
        if self.search_tag == '':
            self.search_tag = 'All'
        if os.path.exists('/home/pi/spider/%s/' % self.search_tag) is False:
            os.makedirs('/home/pi/spider/%s/' % self.search_tag)
        return '/home/pi/spider/%s/%s.%s' % (self.search_tag, self.img_id, 'jpg' if self.is_hd() else 'png')

    def get_img_url(self):
        return self.jpeg_url if self.is_hd() else self.sample_url


# example from https://github.com/pytorch/examples/blob/master/vae/main.py
# commented and type annotated by Charl Botha <cpbotha@vxlabs.com>
import os
import torch
import torch.utils.data
import numpy as np
from torch import nn, optim
from torch.autograd import Variable
from torch.nn import functional as F
from torchvision.utils import save_image
from custom_dataset import CustomDatasetFromImages

# changed configuration to this instead of argparse for easier interaction
CUDA = True
SEED = 1
BATCH_SIZE = 64
LOG_INTERVAL = 10
EPOCHS = 100

torch.manual_seed(SEED)
if CUDA:
    torch.cuda.manual_seed(SEED)

# DataLoader instances will load tensors directly into GPU memory
kwargs = {'num_workers': 1, 'pin_memory': True} if CUDA else {}

# Download or load downloaded MNIST dataset
# shuffle data at every epoch

train_loader = torch.utils.data.DataLoader(
    CustomDatasetFromImages('lfw'),
    batch_size=BATCH_SIZE, shuffle=True, **kwargs)

# Same for test data
test_loader = torch.utils.data.DataLoader(
    CustomDatasetFromImages('lfw'),
    batch_size=BATCH_SIZE, shuffle=True, **kwargs)

class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()
        self.conv1 = nn.Conv2d(3, 8, 7)
        self.conv2 = nn.Conv2d(8, 16, 9)
        self.conv3 = nn.Conv2d(16, 32, 9)
        self.pool = nn.MaxPool2d(2)
        self.relu = nn.ReLU()
        self.norm1 = nn.BatchNorm2d(8)
        self.norm2 = nn.BatchNorm2d(16)
        self.norm3 = nn.BatchNorm2d(32)

        self.fc1 = nn.Linear(32 * 24 * 24, 250)
        self.fc2 = nn.Linear(250, 80)
        self.fc3 = nn.Linear(80, 1)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x = self.conv1(x)
        x = self.norm1(x)
        x = self.relu(x)
        x = self.pool(x)
        x = self.conv2(x)
        x = self.norm2(x)
        x = self.relu(x)
        x = self.pool(x)
        x = self.conv3(x)
        x = self.norm3(x)
        x = self.relu(x)
        x = self.pool(x)

       # print(x.shape)
        x = x.view(-1, 32 * 24 * 24)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.sigmoid(self.fc3(x))

        return x

class VAE(nn.Module):

    def __init__(self):
        super(VAE, self).__init__()

        self.conv1 = nn.Conv2d(3, 8, 7, padding=1)
        self.conv2 = nn.Conv2d(8, 8, 7)
        self.conv3 = nn.Conv2d(8, 16, 9)
        self.conv4 = nn.Conv2d(16, 16, 9)
        self.conv5 = nn.Conv2d(16, 32, 9)
        self.conv6 = nn.Conv2d(32, 32, 9)
        self.pool = nn.MaxPool2d(2)
        self.norm1 = nn.BatchNorm2d(8)
        self.norm2 = nn.BatchNorm2d(16)
        self.norm3 = nn.BatchNorm2d(32)

        self.fc11 = nn.Linear(32 * 18 * 18, 2000)
        self.fc12 = nn.Linear(32 * 18 * 18, 2000)
        self.fc2 = nn.Linear(2000, 32 * 18 * 18)

        self.unconv6 = nn.ConvTranspose2d(32, 32, 9)
        self.unconv5 = nn.ConvTranspose2d(32, 16, 9)
        self.unconv4 = nn.ConvTranspose2d(16, 16, 9)
        self.unconv3 = nn.ConvTranspose2d(16, 8, 9)
        self.unconv2 = nn.ConvTranspose2d(8, 8, 7)
        self.unconv1 = nn.ConvTranspose2d(8, 3, 7, padding=1)
        self.interpolate = F.interpolate
        self.sigmoid = nn.Sigmoid()
        self.norm4 = nn.BatchNorm2d(16)
        self.norm5 = nn.BatchNorm2d(8)
        self.norm6 = nn.BatchNorm2d(3)
        self.relu = nn.ReLU()

    def encode(self, x: Variable) -> (Variable, Variable):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.norm1(x)
        x = self.relu(x)
        x = self.pool(x)
        assert x.shape[2]%2 == 0, "The width and height of the image are not even"
        x = self.conv3(x)
        x = self.conv4(x)
        x = self.norm2(x)
        x = self.relu(x)
        x = self.pool(x)
        assert x.shape[2]%2 == 0, "The width and height of the image are not even"
        x = self.conv5(x)
        x = self.conv6(x)
        x = self.norm3(x)
        x = self.relu(x)
        x = self.pool(x)
        x = x.view(-1, 32 * 18 * 18)
        mu, logvar = self.fc11(x), self.fc12(x)
        return mu, logvar

    def decode(self, x: Variable) -> Variable:
        x = self.fc2(x)
        x = x.view(-1, 32, 18, 18)
        x = self.interpolate(x,scale_factor=2)
        x = self.unconv6(x)
        x = self.unconv5(x)
        x = self.norm4(x)
        x = self.relu(x)
        x = self.interpolate(x,scale_factor=2)
        x = self.unconv4(x)
        x = self.unconv3(x)
        x = self.norm5(x)
        x = self.relu(x)
        x = self.interpolate(x,scale_factor=2)
        x = self.unconv2(x)
        x = self.unconv1(x)
        x = self.norm6(x)
        x = self.sigmoid(x)
        assert x.shape[1:4] == torch.Size([3, 250, 250])
        return x

    def reparameterize(self, mu: Variable, logvar: Variable) -> Variable:
        """THE REPARAMETERIZATION IDEA:
        For each training sample (we get 128 batched at a time)
        - take the current learned mu, stddev for each of the ZDIMS
          dimensions and draw a random sample from that distribution
        - the whole network is trained so that these randomly drawn
          samples decode to output that looks like the input
        - which will mean that the std, mu will be learned
          *distributions* that correctly encode the inputs
        - due to the additional KLD term (see generator_loss_function() below)
          the distribution will tend to unit Gaussians
        Parameters
        ----------
        mu : [128, ZDIMS] mean matrix
        logvar : [128, ZDIMS] variance matrix
        Returns
        -------
        During training random sample from the learned ZDIMS-dimensional
        normal distribution; during inference its mean.
        """

        if self.training:
            # multiply log variance with 0.5, then in-place exponent
            # yielding the standard deviation
            std = logvar.mul(0.5).exp_()  # type: Variable
            # - std.data is the [128,ZDIMS] tensor that is wrapped by std
            # - so eps is [128,ZDIMS] with all elements drawn from a mean 0
            #   and stddev 1 normal distribution that is 128 samples
            #   of random ZDIMS-float vectors
            eps = Variable(std.data.new(std.size()).normal_())
            # - sample from a normal distribution with standard
            #   deviation = std and mean = mu by multiplying mean 0
            #   stddev 1 sample with desired std and mu, see
            #   https://stats.stackexchange.com/a/12338
            # - so we have 128 sets (the batch) of random ZDIMS-float
            #   vectors sampled from normal distribution with learned
            #   std and mu for the current input
            return eps.mul(std).add_(mu)

        else:
            # During inference, we simply spit out the mean of the
            # learned distribution for the current input.  We could
            # use a random sample from the distribution, but mu of
            # course has the highest probability.
            return mu

    def forward(self, x: Variable) -> (Variable, Variable, Variable):
        mu, logvar = self.encode(x)
        z = self.reparameterize(mu, logvar)
        return self.decode(z), mu, logvar

generator = VAE()
discriminator = Discriminator()
if CUDA:
    generator.cuda()
    discriminator.cuda()

def generator_loss_function(recon_x, x, mu, logvar) -> Variable:
    # how well do input x and output recon_x agree?
    BCE = F.binary_cross_entropy(recon_x, x)

    # KLD is Kullback–Leibler divergence -- how much does one learned
    # distribution deviate from another, in this specific case the
    # learned distribution from the unit Gaussian

    # see Appendix B from VAE paper:
    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
    # https://arxiv.org/abs/1312.6114
    # - D_{KL} = 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
    # note the negative D_{KL} in appendix B of the paper
    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
    # Normalise by same number of elements as in reconstruction
    KLD /= BATCH_SIZE * (250*250*3)

    # BCE tries to make our reconstruction as accurate as possible
    # KLD tries to push the distributions as close as possible to unit Gaussian
    return BCE.mul(0.5) + KLD.mul(0.5)

adversarial_loss = torch.nn.BCELoss()

optimizer_G = torch.optim.Adam(generator.parameters(), lr = 0.002)
optimizer_D = torch.optim.Adam(discriminator.parameters(), lr = 0.002)

Tensor = torch.cuda.FloatTensor if CUDA else torch.FloatTensor

train_losses = []
def train(epoch):
    generator.train()
    discriminator.train()
    running_loss = 0.0
    for i, (inputs, _) in enumerate(train_loader, 0):

        # Adversarial ground truths
        valid = Variable(Tensor(inputs.shape[0],1).fill_(1.0), requires_grad=False)
        fake = Variable(Tensor(inputs.shape[0],1).fill_(0.0), requires_grad=False)

        # get the inputs
        inputs = Variable(inputs)

        if CUDA:
            inputs = inputs.cuda()

        # -----------------
        #  Train Generator
        # -----------------

        optimizer_G.zero_grad()

        recon_img, mu, logvar = generator(inputs)

        g_loss =    0.001 * adversarial_loss(discriminator(recon_img), valid) + \
                    0.999 * generator_loss_function(recon_img, inputs, mu, logvar)

        g_loss.backward()
        optimizer_G.step()

        running_loss += g_loss.item()

        ave_loss = running_loss / (i + 1.0)
        if i%80==0:
            print('average loss is: ', str(ave_loss))

        # ---------------------
        #  Train Discriminator
        # ---------------------

        optimizer_D.zero_grad()

        # Measure discriminator's ability to classify real from generated samples
        real_loss = adversarial_loss(discriminator(inputs), valid)
        fake_loss = adversarial_loss(discriminator(recon_img.detach()), fake)
        d_loss = 0.5 * (real_loss + fake_loss)

        d_loss.backward()
        optimizer_D.step()

    train_losses.append(running_loss / len(train_loader))
    np.savetxt('train_losses.txt', np.array(train_losses))

def test(epoch):
    if not os.path.isdir("results"):
        os.mkdir("results")

    generator.eval()
    discriminator.eval()
    test_loss = 0

    for i, (inputs, _) in enumerate(test_loader):

        inputs = Variable(inputs)

        if CUDA:
            inputs = inputs.cuda()

        recon_img, mu, logvar = generator(inputs)

        test_loss += generator_loss_function(recon_img, inputs, mu, logvar).item()
        if i == 0:
          n = min(inputs.size(0), 8)
          # for the first 128 batch of the epoch, show the first 8 input digits
          # with right below them the reconstructed output digits
          comparison = torch.cat([inputs[:n],
                                  recon_img.view(BATCH_SIZE, 3, 250, 250)[:n]])
          save_image(comparison.data.cpu(),
                     'results/reconstruction_' + str(epoch) + '.png', nrow=n)

    test_loss /= len(test_loader.dataset)
    print('====> Test set loss: {:.4f}'.format(test_loss))

for epoch in range(1, EPOCHS + 1):
    print("Epoch:",epoch)
    train(epoch)
    test(epoch)


from tkinter import *

root = Tk()
count = 0

def onClickButtoncount():
    global count
    count += 1
    label.config(text=count)

label = Label(root, text='Zero')
label.pack()

button1 = Button(root, text='ADD', command=onClickButtoncount, bg='grey')
button2 = Button(root, text='remove', command=onClickButtoncount, bg='red')

button1.pack()
button2.pack()

root.mainloop()


from typing import List, Union

from yattag import Doc

from ..exceptions import WQXException
from .Organization_Delete import OrganizationDelete

class WQXDelete:
    """
    Main Schema used to delete a portion of water monitoring results from EPA Office of
    Water system.
    """

    __organizationDelete: List[OrganizationDelete]

    def __init__(
        self, o: dict = None, *, organizationDelete: List[OrganizationDelete] = None
    ):
        if isinstance(o, WQXDelete):
            # Assign attributes from object without typechecking
            self.__organizationDelete = o.organizationDelete
        elif isinstance(o, dict):
            # Assign attributes from dictionary with typechecking
            self.organizationDelete = o.get("organizationDelete")
        else:
            # Assign attributes from named keywords with typechecking
            self.organizationDelete = organizationDelete

    @property
    def organizationDelete(self) -> List[OrganizationDelete]:
        return self.__organizationDelete

    @organizationDelete.setter
    def organizationDelete(
        self, val: Union[OrganizationDelete, List[OrganizationDelete]]
    ) -> None:
        if val is None:
            self.__organizationDelete = []
        elif isinstance(val, list):
            r: List[OrganizationDelete] = []
            for x in val:
                r.append(OrganizationDelete(x))
            self.__organizationDelete = r
        else:
            self.__organizationDelete = [OrganizationDelete(val)]

    def generateXML(self, name="WQXDelete"):
        doc = Doc()
        asis = doc.asis
        tag = doc.tag

        with tag(
            name,
            ("xmlns", "http://www.exchangenetwork.net/schema/wqx/3"),
            ("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"),
            (
                "xsi:schemaLocation",
                "http://www.exchangenetwork.net/schema/wqx/3 "
                "http://www.exchangenetwork.net/schema/wqx/3/index.xsd",
            ),
        ):
            if self.__organizationDelete is None:
                raise WQXException("Attribute 'organizationDelete' is required.")
            for x in self.__organizationDelete:
                asis(x.generateXML("OrganizationDelete"))

        return doc.getvalue()


from django.shortcuts import render, get_object_or_404
from .models import Agent

# Create your views here.
def index(request):
    agents = Agent.objects.order_by('-join_date')

    context= {
        'agents': agents
    }
    return render(request, 'agents/agents.html', context)

def agent(request, agent_id):

    agent = get_object_or_404(Agent, pk=agent_id)

    context = {
        'agent' : agent
    }

    return render(request, 'agents/agent.html', context)


from config import (
    part_table_names,
    build_table_name,
    build_full_join_map,
    ignored_build_filter_fields,
)
from actions.Action import Action
from database.pages import get_current_and_total_pages
from database.filters import get_resource_filters
from database.resources import get_resources
from database.utils import load_tables

class GetBuilds(Action):
    def __init__(self):
        self._name = "GetBuilds"
        self._descripption = "Returns build data"

    def perform(self, db, offset, filters, by_popularity):
        tables, parent_tables = load_tables(db, [build_table_name], part_table_names)
        build_table = tables.get(build_table_name)

        filter_data = get_resource_filters(
            db,
            build_table,
            parent_tables,
            build_full_join_map,
            ignored_build_filter_fields,
        )
        current_page, total_pages, offset = get_current_and_total_pages(
            db, build_table, filters, current_page=offset
        )
        builds = get_resources(
            db,
            build_table,
            parent_tables,
            offset,
            filters,
            build_full_join_map,
            by_popularity,
        )
        return {
            "filters": filter_data,
            "currentPage": current_page,
            "totalPages": total_pages,
            "builds": builds,
        }


import random

who_start_turn = random.randint(0, 1)
c = 0
# create test bots
# card 1 = 5/7
# card 2 = 3/7

first_my_attack = 5
first_my_hp = 7
first_my_team = 0
first_enemy_attack = 3
first_enemy_hp = 7
first_enemy_team = 1
first_my_card = [first_my_attack, first_my_hp, first_my_team]
first_enemy_card = [first_enemy_attack, first_enemy_hp, first_enemy_team]
all_my_cards = []
all_enemy_cards = []
ab = []

def count_cards(check_card):
    if check_card[1] > 0:
        if check_card[2] == 0:
            all_my_cards.append(check_card)
        elif check_card[2] == 1:
            all_enemy_cards.append(check_card)
    else:
        pass

count_cards(first_my_card)
count_cards(first_enemy_card)
print(all_my_cards)
print(all_enemy_cards)

def attack(all_my_cards, all_enemy_cards):
    a = random.randint(0, len(all_my_cards) - 1)
    b = random.randint(0, len(all_enemy_cards) - 1)
    all_my_cards[a][1] = all_my_cards[a][1] - all_enemy_cards[b][0]
    all_enemy_cards[b][1] = all_enemy_cards[b][1] - all_my_cards[a][0]
    ab = all_my_cards[a], all_enemy_cards[b]
    return ab

zxc = attack(all_my_cards, all_enemy_cards)
all_my_cards = zxc[0]
all_enemy_cards = zxc[1]
print(all_my_cards)
print(all_enemy_cards)


import os, sys, cv2, pickle, argparse
import numpy as np
from matplotlib import pyplot as plt
from tqdm import tqdm
from scipy.signal import savgol_filter
from block_matching import estimate_flow
from flow_utils import plot_flow

sys.path.append("../W1")
from utils import save_gif

def parse_args(args=sys.argv[1:]):
    parser = argparse.ArgumentParser(description='Video stabilization using optical flow (block matching)')

    parser.add_argument('--video_path', type=str, default='../data/video_stabilization/1.mp4')
    parser.add_argument('--pkl_path', type=str, default='./pkls')

    return parser.parse_args()

if __name__ == '__main__':
    args = parse_args()

    cap = cv2.VideoCapture(args.video_path)
    num_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

    compute_acc_motions = False  # caution: this is slow (max 5-6 sec video!!!)
    visualize_flow = False
    stabilize_video = True
    smooth_trajectories = False
    visualize_stabilization = False
    plot_corrections = False
    create_gifs = True

    if compute_acc_motions:
        os.makedirs(args.pkl_path, exist_ok=True)

        acc_mean_x_motion = [0]
        acc_mean_y_motion = [0]

        acc_median_x_motion = [0]
        acc_median_y_motion = [0]

        for frame_id in range(num_frames-1):
            cap.set(cv2.CAP_PROP_POS_FRAMES, frame_id)
            _, frame_prev = cap.read()
            _, frame_next = cap.read()

            flow = estimate_flow(motion_type='forward', N=16, P=32, distance_metric='ncc',
                                 img_reference=frame_prev, img_current=frame_next)

            if visualize_flow:
                plot_flow(frame_prev, flow)
                # sleep or something

            acc_mean_x_motion.append(acc_mean_x_motion[frame_id] + np.mean(flow[:, :, 0]))
            acc_mean_y_motion.append(acc_mean_y_motion[frame_id] + np.mean(flow[:, :, 1]))

            acc_median_x_motion.append(acc_median_x_motion[frame_id] + np.median(flow[:, :, 0]))
            acc_median_y_motion.append(acc_median_y_motion[frame_id] + np.median(flow[:, :, 1]))

            print(frame_id)

        with open(os.path.join(args.pkl_path, 'acc_mean_y_motion.pkl'), 'wb') as f:
            pickle.dump(acc_mean_y_motion, f)

        with open(os.path.join(args.pkl_path, 'acc_mean_x_motion.pkl'), 'wb') as f:
            pickle.dump(acc_mean_x_motion, f)

        with open(os.path.join(args.pkl_path, 'acc_median_y_motion.pkl'), 'wb') as f:
            pickle.dump(acc_median_y_motion, f)

        with open(os.path.join(args.pkl_path, 'acc_median_x_motion.pkl'), 'wb') as f:
            pickle.dump(acc_median_x_motion, f)

    if stabilize_video:
        trajectory_type = 'median'  # median or mean
        with open(os.path.join(args.pkl_path, 'acc_'+trajectory_type+'_y_motion.pkl'), 'rb') as f:
            acc_y_motions = pickle.load(f)

        with open(os.path.join(args.pkl_path, 'acc_'+trajectory_type+'_x_motion.pkl'), 'rb') as f:
            acc_x_motions = pickle.load(f)

        if smooth_trajectories:
            acc_y_motions = savgol_filter(acc_y_motions, 51, 3)
            acc_x_motions = savgol_filter(acc_x_motions, 51, 3)

        if plot_corrections:
            plt.figure(figsize=(20,8))
            plt.title('Smoothed (SavGol filter) horizontal and vertical corrections due to accumulated ' + trajectory_type + ' motions')
            plt.plot(-np.array(acc_x_motions), label='horizontal motion')
            plt.plot(-np.array(acc_y_motions), label='vertical motion')
            plt.ylabel('Correction (pxs)')
            plt.xlabel('Frame')
            plt.legend()
            plt.show()

        start_flag = True

        first = True

        for frame_id in tqdm(range(num_frames), desc='Stabilizing video frames'):
            _, frame = cap.read()

            transl_matrix = np.array([[1, 0, -acc_x_motions[frame_id]],
                                      [0, 1, -acc_y_motions[frame_id]]],
                                     dtype=np.float32)

            frame_stab = cv2.warpAffine(frame, transl_matrix, (frame.shape[1], frame.shape[0]))

            # s = frame_stab.shape
            # T = cv2.getRotationMatrix2D((s[1]/2, s[0]/2), 0, 1.3)
            # frame = cv2.warpAffine(frame, T, (s[1], s[0]))
            # frame_stab = cv2.warpAffine(frame_stab, T, (s[1], s[0]))

            # mask = np.where(frame_stab==[0,0,0], 1, 0).astype(np.uint8)
            #
            # if first:
            #     frame_stab = frame_stab + frame*mask
            #     first = False
            # else:
            #     frame_stab = frame_stab + prev_frame*mask
            #
            # prev_frame = frame_stab

            if visualize_stabilization:
                cv2.namedWindow('prev', cv2.WINDOW_NORMAL)
                cv2.resizeWindow('prev', 1000, 800)
                cv2.imshow('prev', frame)

                cv2.namedWindow('stabilized', cv2.WINDOW_NORMAL)
                cv2.resizeWindow('stabilized', 1000, 800)
                cv2.imshow('stabilized', frame_stab)

                if start_flag:
                    cv2.waitKey()
                    start_flag = False
                else:
                    cv2.waitKey(1)

            if create_gifs:
                scale = 0.3
                w = int(frame.shape[1] * scale)
                h = int(frame.shape[0] * scale)
                dim = (w, h)

                cv2.imwrite('./gif_original/'+str(frame_id+100)+'.png', cv2.resize(frame, dim, interpolation=cv2.INTER_AREA))
                cv2.imwrite('./gif_stab/' + str(frame_id + 100) + '.png', cv2.resize(frame_stab, dim, interpolation=cv2.INTER_AREA))

        if create_gifs:
            print('Creating GIFs')
            save_gif('./gif_original/', './gif_original.gif', fps=30)
            save_gif('./gif_stab/', './gif_stab.gif', fps=30)


# -*- coding: utf-8 -*-
"""Validator plugin for Houdini Camera ROP settings."""
import pyblish.api
from openpype.pipeline import PublishValidationError

class ValidateCameraROP(pyblish.api.InstancePlugin):
    """Validate Camera ROP settings."""

    order = pyblish.api.ValidatorOrder
    families = ["camera"]
    hosts = ["houdini"]
    label = "Camera ROP"

    def process(self, instance):

        import hou

        node = hou.node(instance.data.get("instance_node"))
        if node.parm("use_sop_path").eval():
            raise PublishValidationError(
                ("Alembic ROP for Camera export should not be "
                 "set to 'Use Sop Path'. Please disable."),
                title=self.label
            )

        # Get the root and objects parameter of the Alembic ROP node
        root = node.parm("root").eval()
        objects = node.parm("objects").eval()
        errors = []
        if not root:
            errors.append("Root parameter must be set on Alembic ROP")
        if not root.startswith("/"):
            errors.append("Root parameter must start with slash /")
        if not objects:
            errors.append("Objects parameter must be set on Alembic ROP")
        if len(objects.split(" ")) != 1:
            errors.append("Must have only a single object.")

        if errors:
            for error in errors:
                self.log.error(error)
            raise PublishValidationError(
                "Some checks failed, see validator log.",
                title=self.label)

        # Check if the object exists and is a camera
        path = root + "/" + objects
        camera = hou.node(path)

        if not camera:
            raise PublishValidationError(
                "Camera path does not exist: %s" % path,
                title=self.label)

        if camera.type().name() != "cam":
            raise PublishValidationError(
                ("Object set in Alembic ROP is not a camera: "
                 "{} (type: {})").format(camera, camera.type().name()),
                title=self.label)


import numpy as np
import random
from math import log, floor
from multiprocessing.dummy import Pool as ThreadPool

def mutli_run_wrapper(args):
    return buildTree(*args)

def getRandomForest(X,Y,K,classCount,M):

    """
        X is the training parameters (n x m matrix)
        Y is the training labels (n x 1 matrix)
        K is the number of parameters considered at each node
        classCount is the number of unique classes in Y
        M is the number of subsets we divide our data into
    """
    a,b = np.shape(X)

    # Get random permutation
    randPerm = np.random.permutation(range(a))

    # Get size of each bag
    bagSize = int(a/M)

    pool = ThreadPool(M)

    # Divide data randomly into M bags and create random tree for each
    args = []
    for m in range(M):
        bag = randPerm[m*bagSize:min((m+1)*bagSize,a-1)]
        args.append((X[bag,:],Y[bag],classCount,K))

    randomForest = pool.map(buildTree,args)

    return randomForest

def treeClassify(A,N):
    # If not a leaf, test data against node's function
    if(N.Label == None):
        t = N.test(A)
        leftSplit = A[t == 0,:]
        rightSplit = A[t == 1,:]
        Yhat = np.zeros(A.shape[0])
        Yhat[t == 0] = treeClassify(leftSplit,N.leftChild)
        Yhat[t == 1] = treeClassify(rightSplit,N.rightChild)
        return Yhat

    # Else return the class of the leaf
    else:
        return N.Label

def buildTree(X,Y,classCount,k):
    # If Y contains a single class create a leaf node
    for i in range(0,classCount):
        if(sum(abs(Y-i)) == 0):
            return Node(Label=i)
    # Otherwise create a node and two children for the split data
    if(len(Y) < 100):
        classDist = []
        for i in range(0,classCount):
            classDist.append(sum(Y == i))
        return Node(Label=np.argmax(classDist))
    n = Node(X,Y,k,classCount)
    div = n.test(X)
    n.leftChild = buildTree(X[div],Y[div],classCount,k)
    sortRight = np.logical_not(div)
    n.rightChild = buildTree(X[sortRight],Y[sortRight],classCount,k)
    n.leftChild.parent = n
    n.rightChild.parent = n
    return n

class Node(object):
    """
        X is the training parameters (n x m matrix)
        Y is the training labels (n x 1 matrix)
        K is the number of parameters considered at each node
        If Y is one class, Label is this class
        Intializes with the test function with highest information gain
    """
    def __init__(self, X = None, Y = None, k = None, classCount = None, Label = None):
        self.leftChild = None
        self.rightChild = None
        self.parent = None
        self.Label = Label
        if(Label == None):
            self.Thresh, self.feature = findBestF(X,Y,k,classCount)

    def test(self,X):
        sortLabels = X[:,self.feature] < self.Thresh
        return sortLabels

def findBestF(X,Y,k,classCount):

    samples,parameters = np.shape(X)
    # J = random.sample(range(parameters),k)
    J = range(parameters)
    A = np.sort(X[:,J],0)

    Thresholds = (A[:-1,:] + A[1:,:])/2

    numOfThresh = 100
    s = np.linspace(0,samples-2,numOfThresh,dtype=int)
    Thresholds = Thresholds[s,:]

    minH = float("infinity")
    bestF = []
    bestDiv = []

    for i in range(numOfThresh):
        for j in range(k):
            sortLabels = np.array(X[:,J[j]] < Thresholds[i,j]).astype('int')

            HA = 0
            HB = 0

            nA = sum(sortLabels)
            nB = samples - nA

            # Calculate entropy
            for y in range(0,classCount):
                classTruth = np.array(Y == y).astype('int')
                totalInClass = sum(classTruth)
                pA = np.dot(classTruth,sortLabels)
                pB = totalInClass - pA
                if(pA == 0 or nA == 0):
                    HA += 0
                else:
                    HA += -pA/nA*log(float(pA)/nA)/log(2)
                if(pB == 0 or nB == 0):
                    HB += 0
                else:
                    HB += -pB/nB*log(float(pB)/nB)/log(2)

            H = (HA*nA + HB*nB)/samples

            if(H < minH):
                bestT = Thresholds[i,j]
                bestF = J[j]
                minH = H

    return bestT, bestF

def DFS(N,X):
    """ Counts numer of nodes """
    if(N.leftChild == None):
        if(N.rightChild == None):
            return 1
        else:
            n1 = DFS(N.rightChild,X)
    elif(N.rightChild == None):
        if(N.leftChild == None):
            return 1
        else:
            n2 = DFS(N.rightChild)
    elif(N.leftChild != None and N.rightChild != None):
        print('Left: ' + str(sum(N.test(X))) + '. Right: ' + str(X.shape()[0]-sum(N.test(X))))
        n2 = DFS(N.leftChild,X)
        n1 = DFS(N.rightChild,X)
    return n1 + n2 + 1


# Blob Detection Example
#
# 这个例子展示了如何使用find_blobs函数来查找图像中的颜色色块。这个例子特别寻找深绿色的物体。

import sensor, image, time
from pyb import Pin
out_1 = Pin('P0',Pin.OUT_PP)
out_2 = Pin('P1',Pin.OUT_PP)
out_3 = Pin('P2',Pin.OUT_PP)

# 为了使色彩追踪效果真的很好，你应该在一个非常受控制的照明环境中。
green_threshold   = (81, 100, -31, 127, 22, 127)
#设置绿色的阈值，括号里面的数值分别是L A B 的最大值和最小值（minL, maxL, minA,
# maxA, minB, maxB），LAB的值在图像左侧三个坐标图中选取。如果是灰度图，则只需
#设置（min, max）两个数字即可。

# You may need to tweak the above settings for tracking green things...
# Select an area in the Framebuffer to copy the color settings.

sensor.reset() # Initialize the camera sensor.
sensor.set_pixformat(sensor.RGB565) # use RGB565.
sensor.set_framesize(sensor.QQVGA) # use QQVGA for speed.
sensor.skip_frames(10) # Let new settings take affect.
sensor.set_auto_whitebal(False) # turn this off.
#关闭白平衡。白平衡是默认开启的，在颜色识别中，需要关闭白平衡。
clock = time.clock() # Tracks FPS.

while(True):
    clock.tick() # Track elapsed milliseconds between snapshots().
    img = sensor.snapshot() # Take a picture and return the image.

    blobs = img.find_blobs([green_threshold])
    #find_blobs(thresholds, invert=False, roi=Auto),thresholds为颜色阈值，
    #是一个元组，需要用括号［ ］括起来。invert=1,反转颜色阈值，invert=False默认
    #不反转。roi设置颜色识别的视野区域，roi是一个元组， roi = (x, y, w, h)，代表
    #从左上顶点(x,y)开始的宽为w高为h的矩形区域，roi不设置的话默认为整个图像视野。
    #这个函数返回一个列表，[0]代表识别到的目标颜色区域左上顶点的x坐标，［1］代表
    #左上顶点y坐标，［2］代表目标区域的宽，［3］代表目标区域的高，［4］代表目标
    #区域像素点的个数，［5］代表目标区域的中心点x坐标，［6］代表目标区域中心点y坐标，
    #［7］代表目标颜色区域的旋转角度（是弧度值，浮点型，列表其他元素是整型），
    #［8］代表与此目标区域交叉的目标个数，［9］代表颜色的编号（它可以用来分辨这个
    #区域是用哪个颜色阈值threshold识别出来的）。
    y=0
    if blobs:
            pixs = []
            for blob in blobs:
                pixs.append(blob[4])
            mx = max(pixs)
            inde = pixs.index(mx)
            y = blobs[inde][6]
            print(y)
            img.draw_cross(blobs[inde][5],blobs[inde][6],color = (255, 0, 0)) # cx, cy
            if(y>90):
                out_1.high()
            else:
                out_1.low()
    else:
        out_1.low()
    print(clock.fps()) # Note: Your OpenMV Cam runs about half as fast while
    # connected to your computer. The FPS should increase once disconnected.