'''
mapclean class is responsible for reading the OSM map xml, auditing, cleaning, generating JSON and finally to insert the Modeled Data into DB.
Created on May 30, 2015

@author: Feby Thomas
'''
import xml.etree.cElementTree as ET
from collections import defaultdict
import re
import pprint
import codecs
import json
from OSM.dbutil import get_db

#The source XML file from OSM
OSMFILE = "milwaukee_wisconsin.osm"

#Regular expression to find the ending of street names
street_type_re = re.compile(r'\b\S+\.?$', re.IGNORECASE)

#Array with expected and correct values for street name ending
expected = ["Street", "Avenue", "Boulevard", "Drive", "Court", "Place", "Square", "Lane", "Road",
            "Trail", "Parkway", "Commons"]

#Regular expressions to check the various tag 'k' sub elements
lower = re.compile(r'^([a-z]|_)*$')
lower_colon = re.compile(r'^([a-z]|_)*:([a-z]|_)*$')
problemchars = re.compile(r'[=\+/&<>;\'"\?%#$@\,\. \t\r\n]')

#Array to check and group the child for Created tag
CREATED = [ "version", "changeset", "timestamp", "user", "uid"]

#Mapping for correcting the street names
mapping = { "St": "Street",
            "St.": "Street",
            "Ave": "Avenue",
            "Ave.": "Avenue",
            "Rd.": "Road",
            "Blvd": "Boulevard",
            "Cir": "Circle",
            "Ct": "Court",
            "Dr": "Drive",
            "Dr.": "Drive",
            "Ln": "Lane",
            "PL": "Place",
            "Pl.": "Place",
            "Pkwy": "Parkway",
            "Rd": "Road",
            "W": "West"
            }

def update_name(name, mapping):
    """
    Function to correct the street name using regular expression and the mapping array above.
    """
    m = street_type_re.search(name)
    if m:
        street_type = m.group()
        for key in mapping:
            if key == street_type:
                name = street_type_re.sub(mapping[key],name)

    return name

def isright_street_type(street_types, street_name):
    """
    Function to check the street type are correct and if not it adds to a set a set dictionary and returns True or False
    """
    m = street_type_re.search(street_name)
    if m:
        street_type = m.group()
        if street_type not in expected:
            street_types[street_type].add(street_name)
            return False
        else:
            return True

def isright_state(state_set,state_name):
    """
    Function to check if the state name is consistent and correct, returns boolean
    """
    if len(state_name) > 2 and state_name != 'WI':
        state_set.add(state_name)
        return False
    else:
        return True

def audit_city(city_set,city_name):
        city_set.add(city_name)

def audit_postcode(postcode):
    """
    Function to check if the post code is consistent and correct, was used to audit only
    """
    if len(postcode) != 5 and len(postcode) != 10 or not postcode.startswith('53'):
        print(postcode)

def is_street_name(elem):
    """
    Function to if XML element attribute is for street
    """
    return (elem.attrib['k'] == "addr:street")

def is_postcode(elem):
    """
    Function to if XML element attribute is for postcode
    """
    return (elem.attrib['k'] == "addr:postcode")

def is_state(elem):
    """
    Function to if XML element attribute is for state
    """
    return (elem.attrib['k'] == "addr:state")

def is_city(elem):
    """
    Function to if XML element attribute is for city
    """
    return (elem.attrib['k'] == "addr:city")

def shape_element(element):
    """
    Function to shape a XML element into data structure as expected for final JSON.
    """
    node = {}
    if element.tag == "node" or element.tag == "way" :

        pos = []
        created = {}
        node['type'] = element.tag
        #Looping for all the attributes to be child of 'created' node.
        for attr in element.attrib:
            if attr in CREATED:
                created[attr] = element.attrib[attr]
                node['created'] = created
            elif attr == 'lat':
                pos.insert(0,float(element.attrib[attr]))
            elif attr == 'lon':
                pos.insert(1,float(element.attrib[attr]))
            else:
                node[attr] = element.attrib[attr]
        node['pos'] = pos
        address = {}
        nd = []
        #Looping for sub elements and attributes of 'tag' elements
        for tag in element.iter("tag"):
            tagk = tag.attrib['k']
            if problemchars.search(tagk):
                pass
            m = lower_colon.match(tagk)
            if m:
                if tagk.startswith("addr:"):
                    address[tagk.split(':')[1]] = tag.attrib['v']
                elif ':' not in tagk:
                    #print(tagk)
                    node[tagk] = tag.attrib['v']
            else:
                node[tagk] = tag.attrib['v']
        if len(address) > 0:
            node['address'] = address
        #print node
        #handling for Node References in the XML
        for tag in element.iter("nd"):
            nd.append(tag.attrib['ref'])
        if len(nd) > 0:
            node['node_refs'] = nd
        return node
    else:
        return None

def auditncleannjson(osmfile,pretty = False):
    """
    Function to audit, clean and structure the XML to the JSON like data dictionary.
    """
    street_types = defaultdict(set)
    cntry = set()
    city = set()
    data = []
    #opening file for JSON output
    file_out = "{0}.json".format(osmfile)
    with codecs.open(file_out, "w") as fo:
        # Looping through the XML Element Tree
        for event, elem in ET.iterparse(osmfile, events=("start",)):
            #Initial audit related checks happen here
            if elem.tag == "node" or elem.tag == "way":
                for tag in elem.iter("tag"):
                    if is_street_name(tag):
                        if not isright_street_type(street_types, tag.attrib['v']):
                            tag.set('v',update_name(tag.attrib['v'], mapping))
                            #print(tag.attrib['v'])
                    if is_state(tag):
                        if not isright_state(cntry, tag.attrib['v']):
                            tag.set('v','WI')
                    #Optional postcode check was used only to audit
                    #if is_postcode(tag):
                    #    audit_postcode(tag.attrib['v'])
                    if is_city(tag):
                        audit_city(city,tag.attrib['v'])
            #Handling within a  single overall loop to shape the element into the  required dictionary data structure.
            el = shape_element(elem)
            if el:
                data.append(el)
                #Writing to the JSON file
                if pretty:
                    fo.write(json.dumps(el, indent=2)+"\n")
                else:
                    fo.write(json.dumps(el) + "\n")

    print('City Audit List: ')
    pprint.pprint(city)
    print('Street Audit List: ')
    pprint.pprint(dict(street_types))
    #print(data[0])
    #returning the full XML converted to Data Dictionary
    return data

def process_map(file_in, pretty = False):
    """
    Starts the audit and cleanup steps.
    """
    return auditncleannjson(file_in, pretty)

def insert_db(osmdata):
    """
    Inserts the data dictionary into the Mongo DB
    """
    db = get_db()
    db.osmdatamke.insert(osmdata)

def process(file_in):
    """
    Overall processing function, which first calls the cleanup and XML to JSON conversion and alter inserts teh data into the database.
    """
    osmdata = process_map(OSMFILE,True)
    print('Process Completed Before DB insert')
    insert_db(osmdata)
    print('DB Insert completed')

if __name__ == '__main__':
    process(OSMFILE)


"""Plugwise Binary Sensor component for Home Assistant."""

import logging

import voluptuous as vol

from homeassistant.components.binary_sensor import BinarySensorEntity, DOMAIN as BINARY_SENSOR_DOMAIN
from homeassistant.const import ATTR_DEVICE_CLASS, ATTR_ICON, ATTR_NAME, ATTR_STATE

from homeassistant.core import callback
from homeassistant.helpers import config_validation as cv, entity_platform

from .const import (
    API,
    ATTR_SCAN_DAYLIGHT_MODE,
    ATTR_SCAN_SENSITIVITY_MODE,
    ATTR_SCAN_RESET_TIMER,
    ATTR_SED_STAY_ACTIVE,
    ATTR_SED_SLEEP_FOR,
    ATTR_SED_MAINTENANCE_INTERVAL,
    ATTR_SED_CLOCK_SYNC,
    ATTR_SED_CLOCK_INTERVAL,
    CB_NEW_NODE,
    COORDINATOR,
    DOMAIN,
    FLAME_ICON,
    FLOW_OFF_ICON,
    FLOW_ON_ICON,
    GW_BINARY_SENSORS,
    IDLE_ICON,
    NO_NOTIFICATION_ICON,
    NOTIFICATION_ICON,
    PW_CLASS,
    PW_MODEL,
    PW_TYPE,
    SCAN_SENSITIVITY_MODES,
    SERVICE_CONFIGURE_BATTERY,
    SERVICE_CONFIGURE_SCAN,
    SEVERITIES,
    STICK,
    STICK_API,
    USB,
    USB_AVAILABLE_ID,
    USB_MOTION_ID,
)

from .gateway import SmileGateway
from .usb import NodeEntity

PARALLEL_UPDATES = 0

_LOGGER = logging.getLogger(__name__)

async def async_setup_entry(hass, config_entry, async_add_entities):
    """Set up the Smile switches from a config entry."""
    if hass.data[DOMAIN][config_entry.entry_id][PW_TYPE] == USB:
        return await async_setup_entry_usb(hass, config_entry, async_add_entities)
    # Considered default and for earlier setups without usb/network config_flow
    return await async_setup_entry_gateway(hass, config_entry, async_add_entities)

async def async_setup_entry_usb(hass, config_entry, async_add_entities):
    """Set up Plugwise binary sensor based on config_entry."""
    api_stick = hass.data[DOMAIN][config_entry.entry_id][STICK]
    platform = entity_platform.current_platform.get()

    async def async_add_binary_sensor(mac):
        """Add plugwise binary sensor."""
        if USB_MOTION_ID in api_stick.devices[mac].features:
            _LOGGER.debug("Add binary_sensors for %s", mac)
            async_add_entities([USBBinarySensor(api_stick.devices[mac])])

            # Register services
            platform.async_register_entity_service(
                SERVICE_CONFIGURE_SCAN,
                {
                    vol.Required(ATTR_SCAN_SENSITIVITY_MODE): vol.In(
                        SCAN_SENSITIVITY_MODES
                    ),
                    vol.Required(ATTR_SCAN_RESET_TIMER): vol.All(
                        vol.Coerce(int), vol.Range(min=1, max=240)
                    ),
                    vol.Required(ATTR_SCAN_DAYLIGHT_MODE): cv.boolean,
                },
                "_service_configure_scan",
            )
            platform.async_register_entity_service(
                SERVICE_CONFIGURE_BATTERY,
                {
                    vol.Required(ATTR_SED_STAY_ACTIVE): vol.All(
                        vol.Coerce(int), vol.Range(min=1, max=120)
                    ),
                    vol.Required(ATTR_SED_SLEEP_FOR): vol.All(
                        vol.Coerce(int), vol.Range(min=10, max=60)
                    ),
                    vol.Required(ATTR_SED_MAINTENANCE_INTERVAL): vol.All(
                        vol.Coerce(int), vol.Range(min=5, max=1440)
                    ),
                    vol.Required(ATTR_SED_CLOCK_SYNC): cv.boolean,
                    vol.Required(ATTR_SED_CLOCK_INTERVAL): vol.All(
                        vol.Coerce(int), vol.Range(min=60, max=10080)
                    ),
                },
                "_service_configure_battery_savings",
            )

    for mac in hass.data[DOMAIN][config_entry.entry_id][BINARY_SENSOR_DOMAIN]:
        hass.async_create_task(async_add_binary_sensor(mac))

    def discoved_binary_sensor(mac):
        """Add newly discovered binary sensor."""
        hass.async_create_task(async_add_binary_sensor(mac))

    # Listen for discovered nodes
    api_stick.subscribe_stick_callback(discoved_binary_sensor, CB_NEW_NODE)

async def async_setup_entry_gateway(hass, config_entry, async_add_entities):
    """Set up the Smile binary_sensors from a config entry."""
    api = hass.data[DOMAIN][config_entry.entry_id][API]
    coordinator = hass.data[DOMAIN][config_entry.entry_id][COORDINATOR]

    entities = []
    is_thermostat = api.single_master_thermostat()

    devices = api.get_all_devices()
    for dev_id in devices:
        if devices[dev_id][PW_CLASS] == "heater_central":
            _LOGGER.debug("Plugwise device_class %s found", devices[dev_id][PW_CLASS])
            data = api.get_device_data(dev_id)
            for binary_sensor in GW_BINARY_SENSORS:
                _LOGGER.debug("Binary_sensor: %s", binary_sensor)
                if binary_sensor not in data:
                    continue

                _LOGGER.debug(
                    "Plugwise binary_sensor Dev %s", devices[dev_id][ATTR_NAME]
                )
                entities.append(
                    GwBinarySensor(
                        api,
                        coordinator,
                        devices[dev_id][ATTR_NAME],
                        dev_id,
                        True,
                        binary_sensor,
                        GW_BINARY_SENSORS[binary_sensor],
                        devices[dev_id][PW_MODEL],
                    )
                )
                _LOGGER.info(
                    "Added binary_sensor.%s",
                    f"{devices[dev_id][ATTR_NAME]}_{binary_sensor}",
                )
        if devices[dev_id][PW_CLASS] == "gateway" and is_thermostat is not None:
            _LOGGER.debug("Plugwise device_class %s found", devices[dev_id][PW_CLASS])
            entities.append(
                GwNotifySensor(
                    api,
                    coordinator,
                    devices[dev_id][ATTR_NAME],
                    dev_id,
                    "plugwise_notification",
                    devices[dev_id][PW_MODEL],
                )
            )
            _LOGGER.info(
                "Added binary_sensor.%s",
                f"{devices[dev_id][ATTR_NAME]}_{'plugwise_notification'}",
            )

    async_add_entities(entities, True)

class SmileBinarySensor(SmileGateway):
    """Represent Smile Binary Sensors."""

    def __init__(
        self, api, coordinator, name, dev_id, enabled_default, binary_sensor, key, model
    ):
        """Initialise the binary_sensor."""
        super().__init__(api, coordinator, name, dev_id)

        self._binary_sensor = binary_sensor
        self._enabled_default = enabled_default
        self._model = model

        self._icon = None
        self._is_on = False

        self._name = key[ATTR_NAME] if key else None

        if dev_id == self._api.heater_id:
            self._entity_name = "Auxiliary"

        if not self._name:
            sensorname = binary_sensor.replace("_", " ").title()
            self._name = f"{self._entity_name} {sensorname}"

        if dev_id == self._api.gateway_id:
            self._entity_name = f"Smile {self._entity_name}"

        self._unique_id = f"{dev_id}-{binary_sensor}"

    @property
    def entity_registry_enabled_default(self) -> bool:
        """Return if the entity should be enabled when first added to the entity registry."""
        return self._enabled_default

    @property
    def icon(self):
        """Return the icon of this entity."""
        return self._icon

    @property
    def is_on(self):
        """Return true if the binary sensor is on."""
        return self._is_on

    @callback
    def _async_process_data(self):
        """Update the entity."""
        raise NotImplementedError

class GwBinarySensor(SmileBinarySensor, BinarySensorEntity):
    """Representation of a Gateway binary_sensor."""

    @callback
    def _async_process_data(self):
        """Update the entity."""
        _LOGGER.debug("Update binary_sensor called")
        data = self._api.get_device_data(self._dev_id)

        if self._binary_sensor not in data:
            self.async_write_ha_state()
            return

        self._is_on = data[self._binary_sensor]

        if self._binary_sensor == "dhw_state":
            self._icon = FLOW_ON_ICON if self._is_on else FLOW_OFF_ICON
        if self._binary_sensor == "flame_state":
            self._icon = FLAME_ICON if self._is_on else IDLE_ICON
        if self._binary_sensor == "slave_boiler_state":
            self._icon = FLAME_ICON if self._is_on else IDLE_ICON

        self.async_write_ha_state()

class GwNotifySensor(SmileBinarySensor, BinarySensorEntity):
    """Representation of a Plugwise Notification binary_sensor."""

    def __init__(self, api, coordinator, name, dev_id, binary_sensor, model):
        """Initialise the notification binary_sensor."""
        super().__init__(
            api, coordinator, name, dev_id, False, binary_sensor, None, model
        )

        self._attributes = {}

    @property
    def device_state_attributes(self):
        """Return the state attributes."""
        return self._attributes

    @callback
    def _async_process_data(self):
        """Update the entity."""
        for severity in SEVERITIES:
            self._attributes[f"{severity.upper()}_msg"] = []

        notify = self._api.notifications

        self._is_on = False
        self._icon = NO_NOTIFICATION_ICON

        if notify != {}:
            self._is_on = True
            self._icon = NOTIFICATION_ICON

            for notify_id, details in notify.items():
                for msg_type, msg in details.items():
                    if msg_type not in SEVERITIES:
                        msg_type = "other"

                    self._attributes[f"{msg_type.upper()}_msg"].append(msg)
                    self.hass.components.persistent_notification.async_create(
                        f"{msg_type.title()}: {msg}",
                        "Plugwise Notification:",
                        f"{DOMAIN}.{notify_id}",
                    )

        self.async_write_ha_state()

class USBBinarySensor(NodeEntity, BinarySensorEntity):
    """Representation of a Stick Node Binary Sensor."""

    def __init__(self, node):
        """Initialize a Node entity."""
        super().__init__(node, USB_MOTION_ID)
        self.node_callbacks = (USB_AVAILABLE_ID, USB_MOTION_ID)

    @property
    def is_on(self):
        """Return true if the binary_sensor is on."""
        return getattr(self._node, STICK_API[USB_MOTION_ID][ATTR_STATE])

    @property
    def unique_id(self):
        """Get unique ID."""
        return f"{self._node.mac}-{USB_MOTION_ID}"

    def _service_configure_scan(self, **kwargs):
        """Service call to configure motion sensor of Scan device."""
        sensitivity_mode = kwargs.get(ATTR_SCAN_SENSITIVITY_MODE)
        reset_timer = kwargs.get(ATTR_SCAN_RESET_TIMER)
        daylight_mode = kwargs.get(ATTR_SCAN_DAYLIGHT_MODE)
        _LOGGER.debug(
            "Configure Scan device '%s': sensitivity='%s', reset timer='%s', daylight mode='%s'",
            self.name,
            sensitivity_mode,
            str(reset_timer),
            str(daylight_mode),
        )
        self._node.Configure_scan(reset_timer, sensitivity_mode, daylight_mode)

    def _service_configure_battery_savings(self, **kwargs):
        """Configure battery powered (sed) device service call."""
        stay_active = kwargs.get(ATTR_SED_STAY_ACTIVE)
        sleep_for = kwargs.get(ATTR_SED_SLEEP_FOR)
        maintenance_interval = kwargs.get(ATTR_SED_MAINTENANCE_INTERVAL)
        clock_sync = kwargs.get(ATTR_SED_CLOCK_SYNC)
        clock_interval = kwargs.get(ATTR_SED_CLOCK_INTERVAL)
        _LOGGER.debug(
            "Configure SED device '%s': stay active='%s', sleep for='%s', maintenance interval='%s', clock sync='%s', clock interval='%s'",
            self.name,
            str(stay_active),
            str(sleep_for),
            str(maintenance_interval),
            str(clock_sync),
            str(clock_interval),
        )
        self._node.Configure_SED(
            stay_active,
            maintenance_interval,
            sleep_for,
            clock_sync,
            clock_interval,
        )


import cv2
import numpy
import math
from PIL import Image
#from matplotlib import pyplot

with open("final_image.txt", "r") as text_file:

	iksevi = text_file.readlines() #iksevi = redovi
	kolone = iksevi[1].split(' ')
	print('iksevi : ',len(iksevi))
	print('kolone : ',len(kolone))
	imgArray = numpy.empty([len(iksevi),len(kolone)], dtype=numpy.int16)
	xerino = 0

	for x in iksevi:
		brojevi = x.split(' ')
		y = 0
		for broj in brojevi:
			if (broj != '\n' and broj != ''):
				imgArray[xerino][y] = int(broj)
				y+=1
		xerino+=1

	img = Image.fromarray(imgArray)
	img.show()


from typing import List, Optional, Dict
import os
import pickle

from elasticsearch_dsl import DocType, Text, Keyword, Search, Index
from elasticsearch_dsl.connections import connections
import elasticsearch
import progressbar
import numpy as np

from qanta.datasets.abstract import QuestionText
from qanta.datasets.quiz_bowl import QuizBowlDataset
from qanta.guesser.abstract import AbstractGuesser
from qanta.spark import create_spark_context
from qanta.wikipedia.cached_wikipedia import CachedWikipedia
from qanta.config import conf
from qanta.preprocess import preprocess_dataset, tokenize_question
from qanta.guesser import nn
from qanta import logging

log = logging.get(__name__)
connections.create_connection(hosts=['localhost'])
index_name = 'ir_dan_guesser'

IR_DAN_WE_TMP = '/tmp/qanta/deep/ir_dan_we.pickle'
IR_DAN_WE = 'ir_dan_we.pickle'

load_embeddings = nn.create_load_embeddings_function(IR_DAN_WE_TMP, IR_DAN_WE, log)

class Answer(DocType):
    page = Text(fields={'raw': Keyword()})
    wiki_content = Text()

    class Meta:
        index = index_name

class ElasticSearchIndex:
    @staticmethod
    def delete():
        try:
            Index(index_name).delete()
        except elasticsearch.exceptions.NotFoundError:
            log.info('Count not delete non-existent index: {}'.format(index_name))

    @staticmethod
    def exists():
        return Index(index_name).exists()

    @staticmethod
    def build(pages, rebuild_index=False):
        if rebuild_index or os.getenv('QB_REBUILD_INDEX', False):
            ElasticSearchIndex.delete()

        if ElasticSearchIndex.exists():
            log.info('Found index, skipping building index')
        else:
            Answer.init()
            cw = CachedWikipedia()
            log.info('Indexing questions and corresponding wikipedia pages...')
            bar = progressbar.ProgressBar()
            for page in bar(pages):
                wiki_content = cw[page].content
                answer = Answer(page=page, wiki_content=wiki_content)
                answer.save()

    @staticmethod
    def search(text: str, max_n_guesses: int):
        s = Search(index=index_name)[0:max_n_guesses].query(
            'multi_match', query=text, fields=['wiki_content'])
        results = s.execute()
        return [(r.page, r.meta.score) for r in results]

IRDAN_PARAMS = 'irdan_params.pickle'
es_index = ElasticSearchIndex()

def preprocess_wikipedia(all_top_guesses, cached_wikipedia, vocab):
    rows = []
    for q_guesses in all_top_guesses:
        q_rows = []
        for page, _ in q_guesses:
            content = tokenize_question(cached_wikipedia[page])[:150]
            q_rows.append(content)
            for t in content:
                vocab.append(t)
        rows.append(q_rows)
    return rows

class IrDanGuesser(AbstractGuesser):
    def __init__(self):
        super().__init__()
        guesser_conf = conf['guessers']['ElasticSearch']
        self.min_appearances = guesser_conf['min_appearances']
        self.n_cores = guesser_conf['n_cores']
        self.use_wiki = guesser_conf['use_wiki']
        self.use_qb = guesser_conf['use_qb']
        self.embeddings = None
        self.embedding_lookup = None
        self.n_classes = None
        self.max_len = None

    def qb_dataset(self):
        return QuizBowlDataset(self.min_appearances, guesser_train=True)

    def parameters(self):
        return {
            'min_appearances': self.min_appearances,
            'n_cores': self.n_cores,
            'use_wiki': self.use_wiki,
            'use_qb': self.use_qb
        }

    def build_model(self):
        pass

    def train(self, training_data):
        log.info('Building elastic search index...')
        ElasticSearchIndex.build(set(training_data[1]))

        def fetch_top_guesses(query):
            return ElasticSearchIndex.search(query, 10)

        sentences = []
        for q in training_data[0]:
            for s in q:
                sentences.append(s)

        sc = create_spark_context('QB: IR NN Re-ranker')
        all_top_guesses = sc.parallelize(sentences).map(fetch_top_guesses).collect()
        sc.stop()

        log.info('Processing training data...')
        x_train, y_train, x_test, y_test, vocab, class_to_i, i_to_class = preprocess_dataset(
            training_data
        )

        cw = CachedWikipedia()
        all_top_pages = preprocess_wikipedia(all_top_guesses, cw, vocab)

        log.info('Creating embeddings...')
        embeddings, embedding_lookup = load_embeddings(vocab=vocab, expand_glove=True, mask_zero=True)
        self.embeddings = embeddings
        self.embedding_lookup = embedding_lookup

        x_train = [nn.convert_text_to_embeddings_indices(q, embedding_lookup) for q in x_train]
        x_test = [nn.convert_text_to_embeddings_indices(q, embedding_lookup) for q in x_test]
        self.n_classes = nn.compute_n_classes(training_data[1])
        self.max_len = nn.compute_max_len(training_data)
        x_train = np.array(nn.tf_format(x_train, self.max_len, 0))
        x_test = np.array(nn.tf_format(x_test, self.max_len, 0))

    def guess(self, questions: List[QuestionText], max_n_guesses: Optional[int]):
        sc = create_spark_context(configs=[('spark.executor.cores', self.n_cores), ('spark.executor.memory', '4g')])

        def es_search(query):
            return es_index.search(query, max_n_guesses)

        return sc.parallelize(questions, 16 * self.n_cores).map(es_search).collect()

    @classmethod
    def targets(cls):
        return []

    @classmethod
    def load(cls, directory: str):
        with open(os.path.join(directory, IRDAN_PARAMS), 'rb') as f:
            params = pickle.load(f)
        guesser = IrDanGuesser()
        guesser.min_appearances = params['min_appearances']
        guesser.n_cores = params['n_cores']
        return guesser

    def save(self, directory: str):
        with open(os.path.join(directory, IRDAN_PARAMS), 'wb') as f:
            pickle.dump({
                'min_appearances': self.min_appearances,
                'n_cores': self.n_cores
            }, f)


import os
import time
from collections import deque
import pickle

import numpy as np
import tensorflow as tf
from mpi4py import MPI

from baselines.ddpg.ddpg import DDPG
import baselines.common.tf_util as tf_util
from baselines import logger

def train(env, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic,
          normalize_returns, normalize_observations, critic_l2_reg, actor_lr, critic_lr, action_noise,
          popart, gamma, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_steps, batch_size, memory,
          tau=0.01, eval_env=None, param_noise_adaption_interval=50):
    """
    Runs the training of the Deep Deterministic Policy Gradien (DDPG) model

    DDPG: https://arxiv.org/pdf/1509.02971.pdf

    :param env: (Gym Environment) the environment
    :param nb_epochs: (int) the number of training epochs
    :param nb_epoch_cycles: (int) the number cycles within each epoch
    :param render_eval: (bool) enable rendering of the evalution environment
    :param reward_scale: (float) the value the reward should be scaled by
    :param render: (bool) enable rendering of the environment
    :param param_noise: (AdaptiveParamNoiseSpec) the parameter noise type (can be None)
    :param actor: (TensorFlow Tensor) the actor model
    :param critic: (TensorFlow Tensor) the critic model
    :param normalize_returns: (bool) should the critic output be normalized
    :param normalize_observations: (bool) should the observation be normalized
    :param critic_l2_reg: (float) l2 regularizer coefficient
    :param actor_lr: (float) the actor learning rate
    :param critic_lr: (float) the critic learning rate
    :param action_noise: (ActionNoise) the action noise type (can be None)
    :param popart: (bool) enable pop-art normalization of the critic output
        (https://arxiv.org/pdf/1602.07714.pdf)
    :param gamma: (float) the discount rate
    :param clip_norm: (float) clip the gradients (disabled if None)
    :param nb_train_steps: (int) the number of training steps
    :param nb_rollout_steps: (int) the number of rollout steps
    :param nb_eval_steps: (int) the number of evalutation steps
    :param batch_size: (int) the size of the batch for learning the policy
    :param memory: (Memory) the replay buffer
    :param tau: (float) the soft update coefficient (keep old values, between 0 and 1)
    :param eval_env: (Gym Environment) the evaluation environment (can be None)
    :param param_noise_adaption_interval: (int) apply param noise every N steps
    """
    rank = MPI.COMM_WORLD.Get_rank()

    assert (np.abs(env.action_space.low) == env.action_space.high).all()  # we assume symmetric actions.
    max_action = env.action_space.high
    logger.info('scaling actions by {} before executing in env'.format(max_action))
    agent = DDPG(actor, critic, memory, env.observation_space.shape, env.action_space.shape, param_noise=param_noise,
                 action_noise=action_noise, gamma=gamma, tau=tau, normalize_returns=normalize_returns,
                 enable_popart=popart, normalize_observations=normalize_observations, batch_size=batch_size,
                 critic_l2_reg=critic_l2_reg, actor_lr=actor_lr, critic_lr=critic_lr, clip_norm=clip_norm,
                 reward_scale=reward_scale)
    logger.info('Using agent with the following configuration:')
    logger.info(str(agent.__dict__.items()))

    # Set up logging stuff only for a single worker.
    if rank == 0:
        tf.train.Saver()

    eval_episode_rewards_history = deque(maxlen=100)
    episode_rewards_history = deque(maxlen=100)
    with tf_util.single_threaded_session() as sess:
        # Prepare everything.
        agent.initialize(sess)
        sess.graph.finalize()

        agent.reset()
        obs = env.reset()
        if eval_env is not None:
            eval_obs = eval_env.reset()
        episode_reward = 0.
        episode_step = 0
        episodes = 0
        step = 0

        start_time = time.time()

        epoch_episode_rewards = []
        epoch_episode_steps = []
        epoch_actions = []
        epoch_qs = []
        epoch_episodes = 0
        for epoch in range(nb_epochs):
            for _ in range(nb_epoch_cycles):
                # Perform rollouts.
                for _ in range(nb_rollout_steps):
                    # Predict next action.
                    action, q_value = agent.policy(obs, apply_noise=True, compute_q=True)
                    assert action.shape == env.action_space.shape

                    # Execute next action.
                    if rank == 0 and render:
                        env.render()
                    assert max_action.shape == action.shape
                    # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
                    new_obs, reward, done, _ = env.step(max_action * action)
                    step += 1
                    if rank == 0 and render:
                        env.render()
                    episode_reward += reward
                    episode_step += 1

                    # Book-keeping.
                    epoch_actions.append(action)
                    epoch_qs.append(q_value)
                    agent.store_transition(obs, action, reward, new_obs, done)
                    obs = new_obs

                    if done:
                        # Episode done.
                        epoch_episode_rewards.append(episode_reward)
                        episode_rewards_history.append(episode_reward)
                        epoch_episode_steps.append(episode_step)
                        episode_reward = 0.
                        episode_step = 0
                        epoch_episodes += 1
                        episodes += 1

                        agent.reset()
                        obs = env.reset()

                # Train.
                epoch_actor_losses = []
                epoch_critic_losses = []
                epoch_adaptive_distances = []
                for t_train in range(nb_train_steps):
                    # Adapt param noise, if necessary.
                    if memory.nb_entries >= batch_size and t_train % param_noise_adaption_interval == 0:
                        distance = agent.adapt_param_noise()
                        epoch_adaptive_distances.append(distance)

                    critic_loss, actor_loss = agent.train()
                    epoch_critic_losses.append(critic_loss)
                    epoch_actor_losses.append(actor_loss)
                    agent.update_target_net()

                # Evaluate.
                eval_episode_rewards = []
                eval_qs = []
                if eval_env is not None:
                    eval_episode_reward = 0.
                    for _ in range(nb_eval_steps):
                        eval_action, eval_q = agent.policy(eval_obs, apply_noise=False, compute_q=True)
                        # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
                        eval_obs, eval_r, eval_done, _ = eval_env.step(max_action * eval_action)
                        if render_eval:
                            eval_env.render()
                        eval_episode_reward += eval_r

                        eval_qs.append(eval_q)
                        if eval_done:
                            eval_obs = eval_env.reset()
                            eval_episode_rewards.append(eval_episode_reward)
                            eval_episode_rewards_history.append(eval_episode_reward)
                            eval_episode_reward = 0.

            mpi_size = MPI.COMM_WORLD.Get_size()
            # Log stats.
            # XXX shouldn't call np.mean on variable length lists
            duration = time.time() - start_time
            stats = agent.get_stats()
            combined_stats = stats.copy()
            combined_stats['rollout/return'] = np.mean(epoch_episode_rewards)
            combined_stats['rollout/return_history'] = np.mean(episode_rewards_history)
            combined_stats['rollout/episode_steps'] = np.mean(epoch_episode_steps)
            combined_stats['rollout/actions_mean'] = np.mean(epoch_actions)
            combined_stats['rollout/Q_mean'] = np.mean(epoch_qs)
            combined_stats['train/loss_actor'] = np.mean(epoch_actor_losses)
            combined_stats['train/loss_critic'] = np.mean(epoch_critic_losses)
            combined_stats['train/param_noise_distance'] = np.mean(epoch_adaptive_distances)
            combined_stats['total/duration'] = duration
            combined_stats['total/steps_per_second'] = float(step) / float(duration)
            combined_stats['total/episodes'] = episodes
            combined_stats['rollout/episodes'] = epoch_episodes
            combined_stats['rollout/actions_std'] = np.std(epoch_actions)
            # Evaluation statistics.
            if eval_env is not None:
                combined_stats['eval/return'] = eval_episode_rewards
                combined_stats['eval/return_history'] = np.mean(eval_episode_rewards_history)
                combined_stats['eval/Q'] = eval_qs
                combined_stats['eval/episodes'] = len(eval_episode_rewards)

            def as_scalar(scalar):
                """
                check and return the input if it is a scalar, otherwise raise ValueError

                :param scalar: (Any) the object to check
                :return: (Number) the scalar if x is a scalar
                """
                if isinstance(scalar, np.ndarray):
                    assert scalar.size == 1
                    return scalar[0]
                elif np.isscalar(scalar):
                    return scalar
                else:
                    raise ValueError('expected scalar, got %s' % scalar)
            combined_stats_sums = MPI.COMM_WORLD.allreduce(np.array([as_scalar(x) for x in combined_stats.values()]))
            combined_stats = {k: v / mpi_size for (k, v) in zip(combined_stats.keys(), combined_stats_sums)}

            # Total statistics.
            combined_stats['total/epochs'] = epoch + 1
            combined_stats['total/steps'] = step

            for key in sorted(combined_stats.keys()):
                logger.record_tabular(key, combined_stats[key])
            logger.dump_tabular()
            logger.info('')
            logdir = logger.get_dir()
            if rank == 0 and logdir:
                if hasattr(env, 'get_state'):
                    with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as file_handler:
                        pickle.dump(env.get_state(), file_handler)
                if eval_env and hasattr(eval_env, 'get_state'):
                    with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as file_handler:
                        pickle.dump(eval_env.get_state(), file_handler)


"""
This file may not be shared/redistributed without permission. Please read copyright notice in the git repo. If this file contains other copyright notices disregard this text.
"""
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm
from irlc import savepdf
from irlc.ex01.agent import train
from irlc.ex12.semi_grad_nstep_sarsa import LinearSemiGradSarsaN
import gym
from irlc import main_plot
from irlc.ex10.mc_evaluate_blackjack import plot_surface_2
from irlc.ex12.semi_grad_sarsa_lambda import LinearSemiGradSarsa

def plot_mountaincar_value_function(env, value_function, ax):
    """
    3d plot
    """
    grid_size = 40
    low = env.unwrapped.observation_space.low
    high = env.unwrapped.observation_space.high
    X,Y = np.meshgrid( np.linspace(low[0], high[0], grid_size), np.linspace(low[1], high[1], grid_size)  )
    Z = X*0
    for i, (x,y) in enumerate(zip(X.flat, Y.flat)):
        Z.flat[i] = value_function([x,y])

    plot_surface_2(X,Y,Z,ax=ax)
    ax.set_xlabel('Position')
    ax.set_ylabel('Velocity')
    ax.set_zlabel('Cost to go')

def figure_10_1():
    episodes = 9000
    plot_episodes = [1, 99, episodes - 1]
    scale = 8
    fig = plt.figure(figsize=(4*scale, scale))
    axes = [fig.add_subplot(1, len(plot_episodes), i+1, projection='3d') for i in range(len(plot_episodes))]
    num_of_tilings = 8
    alpha = 0.3

    env = gym.make("MountainCar-v0")
    agent = LinearSemiGradSarsa(env, gamma=1, alpha=alpha/num_of_tilings, epsilon=0)
    for ep in tqdm(range(episodes)):
        train(env, agent, num_episodes=1, max_steps=np.inf, verbose=False)
        if ep in plot_episodes:
            v = lambda s: -agent.v(s)
            ax = axes[plot_episodes.index(ep)]
            plot_mountaincar_value_function(env, v, ax=ax)
            ax.set_title('Episode %d' % (ep + 1))

    from irlc import savepdf
    savepdf("semigrad_sarsa_10-1")
    plt.show()

def figure_10_2():
    episodes = 500
    num_of_tilings = 8
    alphas = [0.1, 0.2, 0.5]
    env = gym.make("MountainCar500-v0")

    experiments = []
    for alpha in alphas:
        agent = LinearSemiGradSarsa(env, gamma=1, alpha=alpha / num_of_tilings, epsilon=0)
        experiment = f"experiments/mountaincar_10-2_{agent}_{episodes}"
        train(env, agent, experiment_name=experiment, num_episodes=episodes,max_runs=10)
        experiments.append(experiment)

    main_plot(experiments=experiments, y_key="Length")
    plt.xlabel('Episode')
    plt.ylabel('Steps per episode')
    plt.title(env.spec._env_name + " - Semigrad Sarsa - Figure 10.2")
    savepdf("mountaincar_10-2")
    plt.show()

def figure_10_3():
    from irlc.ex12.semi_grad_sarsa_lambda import LinearSemiGradSarsaLambda
    from irlc.ex11.semi_grad_q import LinearSemiGradQAgent

    max_runs = 10
    episodes = 500
    num_of_tilings = 8
    alphas = [0.5, 0.3]
    n_steps = [1, 8]

    env = gym.make("MountainCar500-v0")
    experiments = []

    """ Plot results of experiments here. """
    # TODO: 16 lines missing.
    raise NotImplementedError("")

    main_plot(experiments=experiments, y_key="Length")
    plt.xlabel('Episode')
    plt.ylabel('Steps per episode')
    plt.title(env.spec._env_name + " - Semigrad N-step Sarsa - Figure 10.3")
    savepdf("mountaincar_10-3")
    plt.show()

def figure_10_4():
    from irlc import log_time_series
    alphas = np.arange(0.25, 1.75, 0.25)
    n_steps = np.power(2, np.arange(0, 5))
    episodes = 50
    env = gym.make("MountainCar500-v0")
    experiments = []
    num_of_tilings = 8
    max_asteps = 500
    run = True
    for n_step_index, n_step in enumerate(n_steps):
        aexp = []
        did_run = False
        for alpha_index, alpha in enumerate(alphas):
            if not run:
                continue
            if (n_step == 8 and alpha > 1) or (n_step == 16 and alpha > 0.75):
                # In these cases it won't converge, so ignore them
                asteps = max_asteps #max_steps * episodes
            else:
                n = n_step
                agent = LinearSemiGradSarsaN(env, gamma=1, alpha=alpha / num_of_tilings, epsilon=0, n=n)
                _, stats, _ = train(env, agent, num_episodes=episodes)
                asteps = np.mean( [s['Length'] for s in stats] )
                did_run = did_run or stats is not None

            aexp.append({'alpha': alpha, 'average_steps': asteps})

        experiment = f"experiments/mc_10-4_lsgn_{n_step}"
        experiments.append(experiment)
        if did_run:
            log_time_series(experiment, aexp)

    main_plot(experiments, x_key="alpha", y_key="average_steps", ci=None)
    plt.xlabel('alpha')
    plt.ylabel('Steps per episode')
    plt.title("Figure 10.4: Semigrad n-step Sarsa on mountain car")
    plt.ylim([150, 300])
    savepdf("mountaincar_10-4")
    plt.show()

if __name__ == '__main__':
    # figure_10_1()
    # figure_10_2()
    figure_10_3()
    # figure_10_4()


import ConfigParser
import logging
import os

_VARIABLE_NAMES = 'envvars.ini'
_CONFIG_FILE = 'config.ini'

def get_configuration(section_name, config_file=_CONFIG_FILE,
                      variables=_VARIABLE_NAMES):
    var_config = ConfigParser.SafeConfigParser()
    var_config.read(variables)
    try:
        items = var_config.items(section=section_name)
    except ConfigParser.NoSectionError:
        logging.debug('Var config file %s not found', variables)
        return None

    config = ConfigParser.SafeConfigParser()
    config.read(config_file)
    properties = {}
    for name, variable in items:
        if variable in os.environ:
            properties[name] = os.getenv(variable)
        else:
            try:
                properties[name] = config.get(section_name, name.lower())
            except:
                logging.error('Unable to initialize %s for %s', name,
                              section_name)
                return None
    return properties


#!/usr/bin/python3
from flask import Flask, render_template
from models import storage
from os import getenv
from models.state import State
from models.city import City
app = Flask(__name__)

def show_state():
    my_list = storage.all("State").values()
    return render_template('7-states_list.html', my_list=my_list)

@app.route('/cities_by_states')
def cities_by_states_html():
    my_list = storage.all("State").values()
    return render_template('8-cities_by_states.html', my_list=my_list)

@app.teardown_appcontext
def teardown(self):
    storage.close()

if __name__ == "__main__":
    app.run(host="0.0.0.0", port="5000")


from typing import Optional, Tuple
from uuid import uuid4

from anytree import search

from tree.node import Node

class Tree:
    def __init__(self):
        self.tree = Node(1, 'root')

    def reset(self):
        self.__init__()

    def delete_node(self, node_id: int):
        node = self.get_node_by_id(node_id)
        node.mark_as_deleted()
        for child in node.children:
            self.delete_node(child.id)

    def create_node(
        self,
        parent_id: Optional[int],
        value: str,
        node_id: int = None,
        children: Tuple[Node] = tuple(),
        is_deleted: bool = False,
    ):
        node_id = node_id if node_id else uuid4().int
        parent_node = self.get_node_by_id(parent_id)
        parent_node = parent_node if parent_node else self.tree
        new_node = Node(
            node_id,
            value=value,
            parent_id=parent_id,
            parent=parent_node,
            children=children,
            is_deleted=parent_node.is_deleted,
        )
        return new_node

    def get_node_by_id(self, node_id: int) -> Node:
        return search.find(self.tree, lambda node: node.id == node_id)


'''
Created on Dec 15, 2016

@author: njand
'''
import socket, json
#ip = "169.254.26.174"
ip = "127.0.0.1"
port = 63819
message = json.JSONEncoder().encode({"1":{"wp":[[0, 0, 1],[0,1,1]]}})

sock = socket.socket(socket.AF_INET,
                      socket.SOCK_DGRAM)
sock.sendto(message, (ip, port))


from math import pow, sqrt
import Body
from wx.lib.pubsub import Publisher as pub
import random

SIZE = 250

class GravitySimulator():
    def __init__(self, numberOfBodies, maxVelocity, minMass, maxMass):
        self.maxVelocity = maxVelocity
        self.minMass = minMass
        self.maxMass = maxMass
        self.bodies = {}
        self.creatBodies(numberOfBodies, self.bodies)

    @classmethod
    def initFromDictionary(cls, bodies):
        newSimulator = GravitySimulator(0,0,0,0) #python is fucking stupid
        newSimulator.bodies = bodies
        return newSimulator

    def creatBodies(self, numberOfBodies, bodies):
        """generate random bodies and add them to bodies dictionary"""
        random.seed()
        for _ in range(numberOfBodies):
            tempPosition = Body.position(random.randint(-SIZE,SIZE), random.randint(-SIZE,SIZE), random.randint(-SIZE,SIZE))

            tempVelocity = Body.velocity(random.randint(-self.maxVelocity,self.maxVelocity),random.randint(-self.maxVelocity,self.maxVelocity),random.randint(-self.maxVelocity,self.maxVelocity))

            tempBody = Body.Body(random.uniform(self.minMass,self.maxMass), tempPosition, tempVelocity)
            self.bodies.update({tempPosition:tempBody})

    def moveBodies(self):

        #gravity
        for Body.position1, body1 in self.bodies.iteritems():
                for Body.position2, body2 in self.bodies.iteritems():
                    if(body1!=body2):
                        if (body1.position.x < body2.position.x):
                            body2.velocity.x + Newtonian.gravity(body1, body2)
                        else:
                            body2.velocity.x - Newtonian.gravity(body1, body2)

                        if (body1.position.y < body2.position.y):
                            body2.velocity.y + Newtonian.gravity(body1, body2)
                        else:
                            body2.velocity.y - Newtonian.gravity(body1, body2)

                        if (body1.position.z < body2.position.z):
                            body2.velocity.z + Newtonian.gravity(body1, body2)
                        else:
                            body2.velocity.z - Newtonian.gravity(body1, body2)

        #move the bodies
        newBodies = {}
        for Body.position,body in self.bodies.iteritems():
            body.move()
            newBodies.update({body.position:body})

        self.Bodies = newBodies

        pub.sendMessage("BODIES CHANGED", self.bodies)

class Newtonian:
    Gravity = .0000000000667259 #m^2/kg^2

    @staticmethod
    def gravity(mass1, mass2):
        distance = Newtonian.distance(mass1, mass2)
        return Newtonian.Gravity*((mass1.mass*mass2.mass)/pow(distance,2))

    @staticmethod
    def distance(mass1, mass2):
        p1 = mass1.position.x
        p2 = mass1.position.y
        p3 = mass1.position.z

        q1 = mass2.position.x
        q2 = mass2.position.y
        q3 = mass2.position.z
        return sqrt( pow(p1-q1,2) + pow(p2-q2,2) + pow(p3-q3,2) )


'''
279. Perfect Squares

Question: Given a positive integer n, find the least number of perfect square numbers (for example, 1, 4, 9, 16, ...) which sum to n.
Input: n = 13
Output: 2
Explanation: 13 = 4 + 9.

Solution: Use Breadth First Search
    - Optimizations: Maintain a set of sum of squares to prevent including multiple copies of the same sum

Created on May 12, 2019

@author: smaiya
'''

class Solution:
    def numSquares(self, n):
        squares = [i**2 for i in range(1, int(n**0.5 + 1))]

        count, queue, next_queue = 0, set(), set()
        queue.add(0)
        while queue:
            for curr_sum in queue:
                for sq in squares:
                    if curr_sum+sq==n:
                        return count+1
                    elif curr_sum+sq<n:
                        next_queue.add(curr_sum+sq)
                    else:
                        break
            count, queue, next_queue = count+1, next_queue, set()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2017/11/28 上午9:22
# @Author : Matrix
# @Github : https://github.com/blackmatrix7/
# @Blog : http://www.cnblogs.com/blackmatrix/
# @File : __init__.py.py
# @Software: PyCharm
import json
import logging
from .space import *
from .contacts import *
from .callback import *
from .retry import retry
from functools import wraps
from datetime import datetime
from operator import methodcaller
from .foundation import get_timestamp
from .exceptions import DingTalkExceptions
from .workflow import create_bpms_instance, get_bpms_instance_list
from .customers import get_corp_ext_list, add_corp_ext, get_label_groups
from .auth import get_access_token, get_jsapi_ticket, generate_jsapi_signature
from .messages import async_send_msg, get_msg_send_result, get_msg_send_progress

__author__ = 'blackmatrix'

no_value = object()

methods = {}

def dingtalk(method_name):
    def wrapper(func):
        methods.update({method_name: func.__name__})

        @wraps(func)
        def _wrapper(*args, **kwargs):
            return func(*args, **kwargs)
        return _wrapper
    return wrapper

class DingTalkApp:

    def __init__(self, name, cache, corp_id, corp_secret, agent_id=None,
                 noncestr=None, domain=None, callback_url=None, aes_key=None,
                 token=None):
        """

        :param name:
        :param cache:
        :param corp_id:
        :param corp_secret:
        :param agent_id:
        :param noncestr:
        :param domain: 域名
        :param callback_url: 回调函数地址，回调函数必须符合check_url方法的要求
        :param aes_key: 用于加解密的aes_key，必须是43为字符串，由大小写字母和数字组成，不能有标点符号
        :param token: 用于回调时生成签名的token，非access_token，传入随机字符串
        """
        self.name = name
        self.cache = cache
        self.corp_id = corp_id
        self.corp_secret = corp_secret
        self.agent_id = agent_id
        self.callback_url = callback_url
        # AES_KEY
        self.aes_key = aes_key or 'gbDjdBRfcxrwQA7nSFELj9c0HoWUpcfg8YURx7G84YI'
        self.token = token or 'LnPxMAp7doy'
        self.domain = domain or 'A2UOM1pZOxQ'
        self.noncestr = noncestr or 'VCFGKFqgRA3xtYEhvVubdRY1DAvzKQD0AliCViy'

    @property
    def methods(self):
        return methods

    def get_access_token(self):
        """
        在缓存中设置access token 7000秒过期，每次过期会自动重新获取 access token
        :return:
        """
        key_name = '{}_access_token'.format(self.name)
        if self.cache.get(key_name) is not None:
            access_token = self.cache.get(key_name)
            logging.info('在缓存中找到access token，直接返回缓存数据：{}'.format(access_token))
        else:
            time_out = 7000
            resp = get_access_token(self.corp_id, self.corp_secret)
            access_token = resp['access_token']
            logging.warning('没有在缓存中找到access token，重新向钉钉请求access token：{}'.format(access_token))
            self.cache.set(key_name, access_token, time_out)
            logging.info('重新将access token: {0} 写入缓存，过期时间{1}秒'.format(access_token, time_out))
        return access_token

    def refresh_access_token(self):
        """
        刷新access_token
        :return:
        """
        key_name = '{}_access_token'.format(self.name)
        resp = get_access_token(self.corp_id, self.corp_secret)
        access_token = resp['access_token']
        self.cache.set(key_name, access_token, 7000)
        return access_token

    @property
    def access_token(self):
        return self.get_access_token()

    def get_jsapi_ticket(self):
        jsapi_ticket_key = '{}_jsapi_ticket'.format(self.name)
        access_token_key = '{}_access_token'.format(self.name)

        def _callback(err):
            logging.error(err)
            try:
                self.cache.delete(access_token_key)
                self.cache.delete(jsapi_ticket_key)
            except Exception as ex:
                logging.error(ex)

        @retry(max_retries=5, step=5, callback=_callback)
        def _get_jsapi_ticket():
            if self.cache.get(jsapi_ticket_key) is not None:
                ticket = self.cache.get(jsapi_ticket_key)
                logging.info('在缓存中找到jsapi ticket，直接返回缓存数据：{}'.format(ticket))
            else:
                time_out = 3000
                resp = get_jsapi_ticket(self.access_token)
                ticket = resp['ticket']
                logging.warning('没有在缓存中找到jsapi ticket，重新向钉钉请求jsapi ticket：{}'.format(ticket))
                self.cache.set(jsapi_ticket_key, ticket, time_out)
                logging.info('重新将jsapi ticket {0}写入缓存，过期时间{1}秒'.format(ticket, time_out))
            return ticket

        jsapi_ticket = _get_jsapi_ticket()
        return jsapi_ticket

    @property
    def jsapi_ticket(self):
        return self.get_jsapi_ticket()

    @staticmethod
    def jsapi_signature(jsapi_ticket, noncestr, timestamp, url):
        """
        计算签名信息
        :param jsapi_ticket:
        :param noncestr:
        :param timestamp:
        :param url:
        :return:
        """
        logging.info('jsapi_ticket:{}'.format(jsapi_ticket))
        logging.info('noncestr:{}'.format(noncestr))
        logging.info('timestamp:{}'.format(timestamp))
        logging.info('url:{}'.format(url))
        sign = generate_jsapi_signature(jsapi_ticket=jsapi_ticket, noncestr=noncestr, timestamp=timestamp, url=url)
        logging.info('sign:{}'.format(sign))
        return sign

    @property
    def timestamp(self):
        return get_timestamp()

    def run(self, method_name, *args, **kwargs):
        """
        传入方法名和参数，直接调用指定的钉钉接口，参数只需要传入钉钉的请求参数部分；
        不需要传入钉钉的公共参数部分，公共参数会自动补完。
        例如，需要调用"获取外部联系人标签"的接口，伪代码：
        # 实例化一个DingTalk App
        app = DingTalkApp(.....)
        # 传入需要调用的接口方法名及参数，返回接口调用结果
        data = app.run('dingtalk.corp.ext.listlabelgroups', size=20, offset=0)
        :param method_name:
        :param args:
        :param kwargs:
        :return:
        """
        func_name = methods.get(method_name)
        if func_name is None:
            raise AttributeError('没有找到对应的方法，可能是方法名有误，或SDK暂未实现此方法。')
        f = methodcaller(func_name, *args, **kwargs)
        return f(self)

    def get_request_url(self, method, format_='json', v='2.0', simplify='false', partner_id=None):
        url = 'https://eco.taobao.com/router/rest?method={0}&session={1}&timestamp={2}&format={3}&v={4}'.format(
            method, self.access_token, self.timestamp, format_, v)
        if format_ == 'json':
            url = '{0}&simplify={1}'.format(url, simplify)
        if partner_id:
            url = '{0}&partner_id={1}'.format(url, partner_id)
        return url

    def get_user_list(self, department_id=None):
        data = get_user_list(self.access_token, department_id)
        user_list = data['userlist']
        return user_list

    def get_user_info(self, user_id):
        user_info = get_user(self.access_token, user_id)
        return user_info

    def create_user(self, **user_info):
        """
        创建用户
        :param user_info:
        :return:
        """
        result = create_user(self.access_token, **user_info)
        return result

    def update_user(self, **user_info):
        """
        更新用户
        :param user_info:
        :return:
        """
        result = update_user(self.access_token, **user_info)
        return result

    def delete_user(self, userid):
        """
        删除用户
        :param userid:
        :return:
        """
        result = delete_user(self.access_token, userid)
        return result

    def get_user_by_code(self, code: str):
        data = get_user_by_code(self.access_token, code)
        return data

    def get_department_list(self, id_=None):
        data = get_department_list(self.access_token, id_)
        depart_list = data['department']
        return depart_list

    def get_department(self, id_):
        data = get_department(self.access_token, id_)
        return data

    def create_department(self, **dept_info):
        data = create_department(self.access_token, **dept_info)
        return data['id']

    def update_department(self, **dept_info):
        data = update_department(self.access_token, **dept_info)
        return data['id']

    def delete_department(self, id_):
        data = delete_department(self.access_token, id_)
        return data

    def get_user_departments(self, userid):
        """
        查询指定用户的所有上级父部门路径
        查询主管理员时，会返回无此用户，原因不明。
        可能是钉钉有意设置。
        :param userid:
        :return:
        """
        data = get_user_departments(self.access_token, userid)
        return data

    def get_org_user_count(self, only_active):
        """
        获取企业员工人数
        :param only_active: 0:非激活人员数量，1:已经激活人员数量
        :return:
        """
        data = get_org_user_count(self.access_token, only_active)
        return data['count']

    @dingtalk('dingtalk.corp.ext.listlabelgroups')
    def get_label_groups(self, size=20, offset=0):
        """
        获取系统标签
        :param size:
        :param offset:
        :return:
        """
        resp = get_label_groups(access_token=self.access_token, size=size, offset=offset)
        data = json.loads(resp['dingtalk_corp_ext_listlabelgroups_response']['result'])
        return data

    def get_all_label_groups(self):
        """
        获取全部的外部联系人标签
        :return:
        """
        size = 100
        offset = 0
        label_groups = []
        while True:
            # 钉钉接口存在Bug，偏移量已经超出数据数量时，仍会返回数据
            # 对此需要做特殊处理，今后如此Bug被修复，可以简化代码实现
            # 返回的数据是否重复
            valid_data = False
            # 获取钉钉的接口数据
            dd_label_groups = self.get_label_groups(size, offset)
            # 对数据进行循环，整理
            for dd_label_group in dd_label_groups:
                for label in dd_label_group['labels']:
                    label_group = {'color': dd_label_group['color'],
                                   'group': dd_label_group['name'],
                                   'name': label['name'],
                                   'id': label['id']}
                    if label_group not in label_groups:
                        label_groups.append(label_group)
                        valid_data = True
            # 当已经查询不到有效的新数据时，停止请求接口
            if valid_data is False:
                break
        return label_groups

    @dingtalk('dingtalk.corp.ext.list')
    def get_ext_list(self, size=20, offset=0):
        """
        获取外部联系人
        :return:
        """
        resp = get_corp_ext_list(self.access_token, size=size, offset=offset)
        result = json.loads(resp['dingtalk_corp_ext_list_response']['result'])
        return result

    def get_all_ext_list(self):
        """
        获取全部的外部联系人
        :return:
        """
        size = 100
        offset = 0
        dd_customer_list = []
        while True:
            dd_customers = self.get_ext_list(size=size, offset=offset)
            if len(dd_customers) <= 0:
                break
            else:
                dd_customer_list.extend(dd_customers)
                offset += size
        return dd_customer_list

    @dingtalk('dingtalk.corp.ext.add')
    def add_corp_ext(self, contact):
        """
        获取外部联系人
        :return:
        """
        resp = add_corp_ext(self.access_token, contact)
        return resp

    @dingtalk('dingtalk.smartwork.bpms.processinstance.create')
    def create_bpms_instance(self, process_code, originator_user_id, dept_id, approvers,
                             form_component_values, agent_id=None, cc_list=None, cc_position='FINISH'):
        """
        发起审批实例
        :param process_code:
        :param originator_user_id:
        :param dept_id:
        :param approvers:
        :param form_component_values:
        :param agent_id:
        :param cc_list:
        :param cc_position:
        :return:
        """
        agent_id = agent_id or self.agent_id
        data = create_bpms_instance(self.access_token, process_code, originator_user_id,
                                    dept_id, approvers, form_component_values,
                                    agent_id, cc_list, cc_position)
        return {'process_instance_id': data['dingtalk_smartwork_bpms_processinstance_create_response']['result']['process_instance_id'],
                'request_id': data['dingtalk_smartwork_bpms_processinstance_create_response']['request_id']}

    @dingtalk('dingtalk.smartwork.bpms.processinstance.list')
    def get_bpms_instance_list(self, process_code, start_time, end_time=None, size=10, cursor=0):
        """

        :param process_code:
        :param start_time:
        :param end_time:
        :param size: 每次获取的记录条数，最大只能是10
        :param cursor:
        :return:
        """
        data = get_bpms_instance_list(self.access_token, process_code, start_time, end_time, size, cursor)
        instance_list = data['dingtalk_smartwork_bpms_processinstance_list_response']['result']['result']['list'].get('process_instance_top_vo', [])
        next_cursor = data['dingtalk_smartwork_bpms_processinstance_list_response']['result']['result'].get('next_cursor', 0)
        return instance_list, next_cursor

    def get_all_bpms_instance_list(self, process_code, start_time, end_time=None):
        """
        获取"全部"审批实例
        :param process_code:
        :param start_time: 起始时间，如果不传，默认当前时间往前推6个月
        :param end_time: 结束时间，如果不传，默认当前时间
        :return:
        """
        now = datetime.now()
        end_time = end_time or now
        if start_time > now or start_time > end_time:
            raise DingTalkExceptions.timestamp_err('起始时间不能晚于当前时间或结束时间')
        size = 10
        cursor = 0
        bpms_instance_list = []
        while True:
            instance_list, next_cursor = self.get_bpms_instance_list(process_code, start_time, end_time=end_time, size=size, cursor=cursor)
            bpms_instance_list.extend(instance_list)
            if next_cursor > 0:
                cursor = next_cursor
            else:
                break
        return bpms_instance_list

    @dingtalk('dingtalk.corp.message.corpconversation.asyncsend')
    def async_send_msg(self, msgtype, msgcontent, userid_list=None, dept_id_list=None, to_all_user=False):
        resp = async_send_msg(access_token=self.access_token, agent_id=self.agent_id, msgtype=msgtype,
                              userid_list=userid_list, dept_id_list=dept_id_list, to_all_user=to_all_user,
                              msgcontent=msgcontent)
        return resp

    @dingtalk('dingtalk.corp.message.corpconversation.getsendresult')
    def get_msg_send_result(self, task_id, agent_id=None):
        agent_id = agent_id or self.agent_id
        resp = get_msg_send_result(self.access_token, agent_id, task_id)
        return resp

    @dingtalk('dingtalk.corp.message.corpconversation.getsendprogress')
    def get_msg_send_progress(self, task_id, agent_id=None):
        agent_id = agent_id or self.agent_id
        resp = get_msg_send_progress(self.access_token, agent_id, task_id)
        return resp

    @dingtalk('dingtalk.corp.role.list')
    def get_corp_role_list(self, size=20, offset=0):
        resp = get_corp_role_list(self.access_token, size=size, offset=offset)
        data = resp['dingtalk_corp_role_list_response']['result']['list']
        if data.get('role_groups') is None:
            return None
        else:
            role_groups = data.get('role_groups')
            for role_group in role_groups:
                # 钉钉返回的格式嵌套了两层roles，对格式做下处理
                role_group['roles'] = role_group.pop('roles').pop('roles')
            return role_groups

    def get_all_corp_role_list(self):
        size = 100
        offset = 0
        dd_role_list = []
        while True:
            dd_roles = self.get_corp_role_list(size=size, offset=offset)
            if dd_roles is None or len(dd_roles) <= 0:
                break
            else:
                dd_role_list.extend(dd_roles)
                offset += size
        return dd_role_list

    @dingtalk('dingtalk.corp.role.simplelist')
    def get_role_simple_list(self, role_id, size=20, offset=0):
        data = get_role_simple_list(self.access_token, role_id=role_id, size=size, offset=offset)
        # 返回的数据格式，嵌套这么多层，不累吗？
        user_list = data['dingtalk_corp_role_simplelist_response']['result']['list']
        if user_list and 'emp_simple_list' in user_list:
            return user_list['emp_simple_list']

    @dingtalk('dingtalk.corp.role.getrolegroup')
    def get_role_group(self, group_id):
        data = get_role_group(self.access_token, group_id=group_id)
        return data

    def get_customer_space(self):
        data = get_custom_space(self.access_token, self.domain, self.agent_id)
        return data

    @property
    def space_id(self):
        data = self.get_customer_space()
        return data['spaceid']

    def register_callback(self, callback_tag):
        """
        向钉钉注册回调接口。
        注册回调前需要在初始化DingTalk App时传入aes_key和callback_url
        其中callback_url必须返回经过加密的字符串“success”的json数据
        :param callback_tag:
        :return:
        """
        if self.aes_key is None or self.callback_url is None:
            raise RuntimeError('注册回调前需要在初始化DingTalk App时传入aes_key和callback_url')
        data = register_callback(self.access_token, self.token, callback_tag, self.aes_key, self.callback_url)
        return data

    def encrypt(self, plaintext, buf=None):
        """
        钉钉加密数据
        :param plaintext: 明文
        :param buf:
        :return:
        """
        if self.aes_key is None:
            raise RuntimeError('加密解密前需要在初始化DingTalk App时传入aes_key')
        from .crypto import encrypt
        ciphertext = encrypt(aes_key=self.aes_key, plaintext=plaintext, key=self.corp_id, buf=buf)
        return ciphertext

    def encrypt_text(self, plaintext: str):
        """
        对纯文本进行加密
        :param plaintext: 明文
        :return:
        """
        if self.aes_key is None:
            raise RuntimeError('加密解密前需要在初始化DingTalk App时传入aes_key')
        from .crypto import encrypt_text
        ciphertext = encrypt_text(aes_key=self.aes_key, plaintext=plaintext)
        return ciphertext

    def decrypt(self, ciphertext: str):
        """
        钉钉解密数据
        :param ciphertext: 密文
        :return:
        """
        if self.aes_key is None:
            raise RuntimeError('加密解密前需要在初始化DingTalk App时传入aes_key')
        from .crypto import decrypt
        msg, key, buf = decrypt(self.aes_key, ciphertext)
        return msg, key, buf

    def decrypt_text(self, ciphertext: str):
        """
        对纯文本进行解密
        :param ciphertext: 密文
        :return:
        """
        if self.aes_key is None:
            raise RuntimeError('加密解密前需要在初始化DingTalk App时传入aes_key')
        from .crypto import decrypt_text
        temp = decrypt_text(self.aes_key, ciphertext)
        return temp

    def generate_callback_signature(self, data, timestamp, nonce):
        from .crypto import generate_callback_signature
        sign = generate_callback_signature(self.token, data, timestamp, nonce)
        return sign

    def check_callback_signature(self, signature, ciphertext, timestamp, nonce):
        """
        验证钉钉回调接口的签名
        算法请访问
        https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7386797.0.0.EkauZY&source=search&treeId=366&articleId=107524&docType=1
        :param signature: 需要验证的签名
        :param ciphertext: 加密后的数据
        :param timestamp: 时间戳
        :param nonce: 字符串，叫什么忘了
        :return:
        """
        from .crypto import check_callback_signature
        return check_callback_signature(self.token, ciphertext, signature, timestamp, nonce)

    def get_call_back_failed_result(self):
        data = get_callback_failed_result(self.access_token)
        return data['failed_list']

    def return_success(self):
        """
        钉钉回调需要返回含有success的json，提供一个方法，快速返回一个符合钉钉要求的success json
        :return:
        """
        # 加密success数据
        encrypt = self.encrypt('success').decode()
        # 创建时间戳
        timestamp = str(self.timestamp)
        # 获取随机字符串
        nonce = self.noncestr
        # 创建签名
        signature = self.generate_callback_signature(encrypt, timestamp, nonce)
        # 返回结果
        return {'msg_signature': signature, 'timeStamp': timestamp, 'nonce': nonce, 'encrypt': encrypt}

    def check_url(self, ding_nonce, ding_sign, ding_timestamp, ding_encrypt):
        """
        一个钉钉注册回调的check_url方法
        文档：
        https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.x75fVY&treeId=385&articleId=104975&docType=1#s12
        :param ding_nonce: 钉钉返回的随机字符串
        :param ding_sign: 钉钉返回的签名
        :param ding_timestamp: 钉钉返回的时间戳
        :param ding_encrypt: 钉钉返回的加密后数据
        :return: 返回带success的json
        """
        # 验证签名
        if self.check_callback_signature(ding_sign, ding_encrypt, ding_timestamp, ding_nonce) is False:
            raise DingTalkExceptions.sign_err
        # 签名验证成功后，解密数据
        ding_data, corp_id, buf = self.decrypt(ding_encrypt)
        assert ding_data and corp_id and buf
        # 返回结果
        result = self.return_success()
        return result

if __name__ == '__main__':
    pass


from django.core.validators import MinValueValidator, MaxValueValidator
from django.db import models
from django.utils.translation import gettext_lazy as _

class CV(models.Model):
    title_en = models.CharField(_('Title (en)'),
                                help_text=_('Specify the title (occupation/degree) of your new CV entry'),
                                max_length=64, null=False, blank=False
                                )
    title_de = models.CharField(_('Title (de)'),
                                help_text=_('Specify the title (occupation/degree) of your new CV entry'),
                                max_length=64, null=False, blank=False
                                )

    def __str__(self):
        return self.title_en

    class Meta:
        abstract = False

class Experience(CV):
    company = models.CharField(_('Company'),
                               help_text=_('Specify the name of the company'),
                               max_length=64, null=False, blank=False
                               )

    date_from = models.DateField(_('Start of occupation'),
                                 help_text=_('Define the start of your occupation'),
                                 null=False, blank=False
                                 )
    date_to = models.DateField(_('End of your occupation'),
                               help_text=_('Define the end of your occupation\n'
                                            'Leave blank if it is still in progress'),
                               null=True, blank=True
                               )
    content_en = models.TextField(_('Content (en)'),
                                  help_text=_('Specify the main content of your occupation'),
                                  max_length=512, null=False, blank=False
                                  )
    content_de = models.TextField(_('Content (de)'),
                                  help_text=_('Specify the main content of your occupation'),
                                  max_length=512, null=False, blank=False
                                  )

    def __str__(self):
        return ''.join([self.title_en, '@', self.company])

    class Meta:
        ordering = ['-date_from']
        verbose_name = _('Experience')
        verbose_name_plural = _('Experiences')

class Education(CV):
    school = models.CharField(_('School/College'),
                              help_text=_('Specify the name of the school or college'),
                              max_length=64, null=False, blank=False
                              )
    date_from = models.DateField(_('Start of education'),
                                 help_text=_('Define the start of your education'),
                                 null=False, blank=False
                                 )
    date_to = models.DateField(_('End of your education'),
                               help_text=_('Define the end of your education\n'
                                            'Leave blank if it is still in progress'),
                               null=True, blank=True
                               )
    content_en = models.TextField(_('Content (en)'),
                                  help_text=_('Specify the main content of your education'),
                                  max_length=512, null=False, blank=False
                                  )
    content_de = models.TextField(_('Content (de)'),
                                  help_text=_('Specify the main content of your education'),
                                  max_length=512, null=False, blank=False
                                  )

    def __str__(self):
        return ''.join([self.title_en, '@', self.school])

    class Meta:
        ordering = ['-date_from']
        verbose_name = _('Education')
        verbose_name_plural = _('Education')

class Skill(CV):
    level = models.FloatField(_('Level'),
                              help_text=_('level 1 being 100% (don\'t do that...)'),
                              null=False, blank=False,
                              validators=[
                                  MinValueValidator(.05),
                                  MaxValueValidator(1.0)
                              ]
                              )

    # def __str__(self):
    #     return self.title_en

    class Meta:
        abstract = False
        verbose_name = _('Skill')
        verbose_name_plural = _('Skills')

class CodingLanguage(Skill):

    class Meta:
        verbose_name = _('Coding Language')
        verbose_name_plural = _('Coding Languages')

class SoftwareSkill(Skill):

    class Meta:
        verbose_name = _('Software Skill')
        verbose_name_plural = _('Software Skills')

class Language(Skill):

    class Meta:
        verbose_name = _('Language')
        verbose_name_plural = _('Languages')


from engpy.lib.SS import series
from engpy.errors.exceptions import *
from engpy.misc.assist import List
from engpy.misc.gen import getter

class Progression(series.base):
    def members(self, stop = '',  values = '', **kvalues):
        if not stop:
            stop = self.u_bound
        if not values:
            values = {}
        for keys, value in kvalues.items():
            values.update({keys: value})
        return [terms._cal(values) for terms in self]

    @property
    def name(self):
        return 'Progression'

    def __mul__(self,other):
        self.expr *= other

    def __add__(self,other):
        print([self,other],'ol')
        return Progressions([self,other])

    def __sub__(self,other):
        return Progressions([self,-1 * other])

class Progressions(Progression):
    def __new__(cls, expr = '',nth = 'n', l_bound = '', u_bound = '', step = 0, inv = [], **kwargs):
        if not type(expr) == list:
            print(expr,'iopoi')
            raise UnacceptableToken
        l_bound = []; u_bound = []; step = []
        for exprs in expr:
            if getter(exprs,'name') in ('Progression', 'Progressions'):
                raise UnacceptableToken
            l_bound.append(exprs.l_bound); u_bound.append(exprs.u_bound)
            step.append(exprs.step)
        if not List(step).unique:
            raise InvalidOperation
        self = super(Progressions, cls).__new__(cls)
        self.expr = expr
        self.l_bound = min(l_bound)
        self.u_bound = max(u_bound)
        self.step = step[0]
        self.name = 'Progressions'

    def __str__(self):
        return ' + '.join(str(expr) for expr in self.expr)

    @classmethod
    def __add__(cls,other):
        return cls(self.expr + (other if isinstance(other,list) else [other]))

    @classmethod
    def __sub__(cls,other):
        return cls(self.expr + [-1 * other_ for other in (other if isinstance(other,list) else [other])])

    def __getitem__(self,other):
        other *= self.step
        other = num(other)
        if isinstance(other, slice):
            return [self[i] for i in range(other.start, other.stop, 1 if other.step is None else other.step)]
        if not isinstance(other,int):
            raise UnacceptableToken(f'Index must be an integer')
        if not self.l_bound <= other <= self.u_bound or other % self.step != self.l_bound:
            raise OutOfRange(f'This Sequence is valid in range {self.l_bound} - {self.u_bound}' +  f' in the intervals of {self.step}' if self.step - 1 else '')
        if [inv.inv for inv in self.expr].count(other) == len(self.expr):
            raise Void(f'This Sequence is not valid for term {other}')
        total = 0
        for seq in self.expr:
            try:
                total += seq[other]
            except Void:
                pass
            except OutofRange:
                pass
        return total if isinstance(total,(int,float,complex)) else total.simp()

    @classmethod
    def form(cls):
        return cls

    @property
    def recreate(self):
        return self.form()
    __repr__ = __str__

class AP(Progression):
    def __init__(self, expr = ''):
        self.name = 'AP'
        if not expr:
            self.expr = Progression('a + (n-1)d')
            if isinstance(expr,(list,tuple)):
                self.cal(a = expr[0], d = expr[1] - expr[0])

class GP(Progression):
    def __init__(self, expr = ''):
        self.name = 'GP'
        if not expr:
            self.expr = Progression('ar^(n-1)')
            if isinstance(expr,(list,tuple)):
                self.cal(a = expr[0], r = expr[1] / expr[0])


import chess
import random
import os
import time
import copy
from socket import *

BOLD = '\033[1m'
CEND = '\033[0m'
CRED = '\033[31m'
CGRAY = '\033[90m'

def send(sock, move):
    sendData = move
    sock.send(sendData.encode('utf-8'))

def receive(sock):
    recvData = sock.recv(1024)
    decodeMove = recvData.decode('utf-8')
    board.push(chess.Move.from_uci(decodeMove))

# board.move_stack을 사용, 마지막 움직임 출력
def moveStackList():
    moveList=[]
    for i in board.move_stack:
        moveList.append(str(i))
    if len(moveList) > 0 :
        print('\nmove: %s'%(moveList[-1]))

# display
def display():
    reboard = str(board)
    for i in range(len(reboard)):
        if reboard[i].islower() is True:
            print(CRED+BOLD+reboard[i]+CEND, end='')
        elif reboard[i].islower() is False:
            print(reboard[i], end='')
        else:
            print(CGRAY+BOLD+reboard[i]+CEND, end='')
    print()
    moveStackList()

    # p,r,n,b,q,k,P,R,N,B,Q,K = 0
    for i in range(12):
        if piece[i][1] >= 1:
            print('%s: %d'%(piece[i][0],piece[i][1]))

# 잡은 말 count
def capturedCount(before_board, after_board):
    before_list=[0]*12
    after_list=[0]*12

    for i in range(len(before_board)):
        for j in range(len(piece_list)):
            if piece_list[j] is before_board[i]:
                before_list[j] += 1

    for i in range(len(after_board)):
        for j in range(len(piece_list)):
            if piece_list[j] is after_board[i]:
                after_list[j] += 1

    # for i in range(len(before_list)):
    #     if before_list[i] > after_list[i]:
    #         piece[i][1]+=1

    for i in range(len(before_list)):
        if before_list[i] > after_list[i]:
            if (sum(before_list) > sum(after_list)):
                piece[i][1]+=1

port = 8080
serverSock = socket(AF_INET, SOCK_STREAM)
serverSock.bind(('', port))
serverSock.listen(1)
print('%d : LOADING...' % port)
connectionSock, addr = serverSock.accept()
print('CONNECTED in ', str(addr))

board = chess.Board()
piece_list = ['p','r','n','b','q','k','P','R','N','B','Q','K']
piece = []
for i in range(12):
    line = [0, 0]
    piece.append(line)
    piece[i][0]= piece_list[i]

beforeBoard = str(board)
os.system('clear')
display()
while True:
    legal_list = []
    for i in board.legal_moves:
        legal_list.append(str(i))
    rmove = chess.Move.from_uci(random.choice(legal_list))
    board.push(rmove)

    afterBoard = str(board)
    capturedCount(beforeBoard, afterBoard)

    os.system('clear')
    display()

    send(connectionSock, str(rmove))
    beforeBoard = copy.deepcopy(afterBoard)
    if board.is_game_over() is True:
        print()
        print(board.result())
        break

    receive(connectionSock)
    afterBoard = str(board)
    #capturedCount(beforeBoard, afterBoard)
    os.system('clear')
    display()
    if board.is_game_over() is True:
        print()
        print(board.result())
        break


import os.path as osp
import glob
import re
import os
import pdb
import json
from collections import defaultdict

import warnings

from .bases import ImageDataset
from ..datasets import DATASET_REGISTRY

def get_pid_info(trackid_associated_list):
    # pair_set = [x.split(' ') for x in info_match]
    # print('pair_set', pair_set[0:3])
    pair_0 = [k for k, v in trackid_associated_list.items() if len(v)>0]
    pair_1 = [v for k, v in trackid_associated_list.items() if len(v)>0]

    track_set = list(trackid_associated_list.keys())

    info_track = defaultdict(list)
    id_map = {}
    id = 0
    for i in range(len(pair_0)):
        # if pair_0[i] == '2019-11-26_07-50-24-562_ee_bb_03_01_00_e7':
        #     print(i, pair_1[i])
        # if id==221:
        #     aaaa=2
        if pair_0[i] not in id_map.keys() and pair_1[i] not in id_map.keys():
            id_map[pair_0[i]] = id
            id_map[pair_1[i]] = id
            info_track[id] += [pair_0[i], pair_1[i]]
            id+=1
        elif pair_1[i] not in id_map.keys():
            id_map[pair_1[i]] = id_map[pair_0[i]]
            info_track[id_map[pair_1[i]]].append(pair_1[i])
        elif pair_0[i] not in id_map.keys():
            id_map[pair_0[i]] = id_map[pair_1[i]]
            info_track[id_map[pair_0[i]]].append(pair_0[i])
        else:
            id_0 = id_map[pair_0[i]]
            id_1 = id_map[pair_1[i]]
            if id_0 != id_1:
                # print('hi')
                info_track[id_0] += info_track[id_1]
                for trackid in info_track[id_1]:
                    id_map[trackid] = id_0
                info_track.pop(id_1, None)

    print('number of IDs after merge', len(info_track))

    for trackid in track_set:
        if trackid not in id_map.keys():
            id_map[trackid] = id
            info_track[id] = [trackid]
            id+=1
    print('number of IDs', len(set(id_map.values())))

    data_info = {}
    data_info['tracklet_pids'] = id_map
    data_info['pid_info'] = info_track

    return data_info

def get_one_sample_list(data_name, data_path, anno_info, track_info, remove_info):

    samples_all = {}

    trackid_associated_list = anno_info['trackid_associated_list']
    category_list = anno_info['category_list']

    for track_id, _ in category_list.items():
        if track_id not in trackid_associated_list.keys():
            trackid_associated_list[track_id] = []
    print(data_name, '#track in anno:', len(category_list))

    frame_set_selected_list = {}
    for track_id, track in track_info.items():
        frame_set_selected_list[track_id] = list(track['frame_info'].keys())

    data_info = get_pid_info(trackid_associated_list)

    pid_info = data_info['pid_info']

    tid_shown = []
    pid_shown = []
    fname_shown = []
    img_set = []
    count = 0
    print(data_name)
    for pid, tid_list in pid_info.items():
        # print(data_name, pid)
        pid_t = '{}#{}'.format(data_name,pid)
        # pid_container.add(pid_t)
        for tid in tid_list:
            track = track_info[tid]
            att = category_list[tid]
            if att != 'client':
                continue
            # print(tid)
            # save_path_att = '{}/{}'.format(save_path, att)
            # if not os.path.exists(save_path_att):
            #     os.mkdir(save_path_att)
            # cid = '{}#{}'.format(data_name, tid.split('_')[2])
            cid = tid.split('_')[2]
            # cid_container.add(cid)
            for fname in frame_set_selected_list[tid]:
                # if 'bodyBboxOfCurrent' not in track['frame_info'][fname].keys():
                #     continue
                # bodyBboxOfCurrent = track['frame_info'][fname]['bodyBboxOfCurrent']
                # samples_all.append((os.path.join(data_path, tid, fname+'.jpg'), pid_t, cid))

                # samples_all.append(os.path.join(data_path, tid, fname+'.jpg'))
                # print(fname)

                if '{}#{}.jpg'.format(tid, fname) in remove_info:
                    continue

                samples_all[fname] = {'img_path': os.path.join(data_path, tid, fname+'.jpg'), 'pid': pid_t, 'cid': cid}
                    # shutil.copyfile(os.path.join(data_path, tid, fname+'.jpg'), os.path.join(save_path_att, '{}#{}#{}#{}.jpg'.format(data_name,pid,tid,fname)))
    # print('finished!!!!!!!!')
    return samples_all

def get_samples(data_root, remove_path, info_path, data_name_set, path_anno):
    samples_all = {}
    for data_name in data_name_set:
        data_path = '{}/{}'.format(data_root, data_name)
        remove_fname = '{}/{}.txt'.format(remove_path, data_name)

        with open('{}/{}_anno.json'.format(path_anno, data_name), 'r') as f:
            anno_info = json.load(f)

        with open('{}/{}.json'.format(info_path, data_name), 'r') as f:
            track_info = json.load(f)
        remove_info = []
        if os.path.exists(remove_fname):
            with open(remove_fname) as f:
                remove_info = f.read().splitlines()

        sample_one = get_one_sample_list(data_name, data_path, anno_info, track_info, remove_info)

        samples_all.update(sample_one)

    return samples_all

def get_samples_val(data_path_val, data_name, path_anno):

    data_path = '{}/{}'.format(data_path_val, data_name)
    remove_fname = '/hddc/ywu/data/blf/reid/sample_remove_list/{}.txt'.format(data_name)

    with open('{}/{}_anno.json'.format(path_anno, data_name), 'r') as f:
        anno_info = json.load(f)

    with open('/hddc/ywu/data/blf/reid/track_info/{}.json'.format(data_name), 'r') as f:
        track_info = json.load(f)
    remove_info = []
    if os.path.exists(remove_fname):
        with open(remove_fname) as f:
            remove_info = f.read().splitlines()

    sample_one = get_one_sample_list(data_name, data_path, anno_info, track_info, remove_info)

    return sample_one

def get_pid_cid(samples):
    pid_container = set()
    cid_container = set()
    for k, info in samples.items():

        pid_container.add(info['pid'])
        cid_container.add(info['cid'])

    return pid_container, cid_container

def get_all_sample(val_dataname, query_cam):
    date = '2020-06-25'
    data_name_set = ['100000058', '100000078', '100000181', '100000199', '100000225', '100000228', '100000282', '100000298', '100000327', '100003003', '100019002', '123000086', '123000155', '123001073']
    data_name_set = ['{}_{}'.format(x,date) for x in data_name_set]

    data_name_set_2 = ['100000007', '100000305', '100000318', '100002002', '100002006', '100002009', '100009001', '100022002', '100075009', '100076003',
    '100079002', '100079033', '123000081', '123000096']
    data_name_set_2 = ['{}_{}'.format(x,date) for x in data_name_set_2]

    data_name_set += data_name_set_2

    date_3 = '2020-07-16'
    data_name_set_3 = ['100000008', '100000526', '100003002', '100008003', '100033001', '100079008']
    data_name_set_3 = ['{}_{}'.format(x,date_3) for x in data_name_set_3]
    data_name_set_3 += ['100001107_2020-07-23']

    # val_dataname = '100000027_2020-03-17'
    data_root = '/hddc/ywu/data/blf/reid/2020-06-25'
    info_path = '/hddc/ywu/data/blf/reid/track_info'
    remove_path = '/hddc/ywu/data/blf/reid/sample_remove_list'

    data_root_val = '/hddc/ywu/data/blf/reid'

    path_anno = '/hddc/ywu/data/blf/reid/annotation'

    pid_container = set()
    pid_train_container = set()
    cid_container = set()

    # samples_train = get_samples_train(data_name_set, path_anno)
    # samples_val = get_samples_val(data_path_val, val_dataname, path_anno)
    samples_train = get_samples(data_root, remove_path, info_path, data_name_set, path_anno)#get_samples(data_name_set, path_anno)
    samples_train_2 = get_samples(data_root_val, remove_path, info_path, data_name_set_3, path_anno)
    samples_train.update(samples_train_2)

    samples_val = get_samples(data_root_val, remove_path, info_path, val_dataname, path_anno)

    pid_train_container, cid_container = get_pid_cid(samples_train)

    pid2label_train = {pid:label for label, pid in enumerate(pid_train_container)}
    cid2label = {cid:label for label, cid in enumerate(cid_container)}

    train, query, gallery = [], [], []

    for k, sample_info in samples_train.items():
        img_path = sample_info['img_path']
        pid = sample_info['pid']
        cid = sample_info['cid']
        train.append((img_path, pid2label_train[pid], cid2label[cid]))

    pid_container_val, cid_container_val = get_pid_cid(samples_val)
    pid2label_val = {pid:label for label, pid in enumerate(pid_container_val)}
    cid2label_val = {cid:label for label, cid in enumerate(cid_container_val)}
    for k, sample_info in samples_val.items():
        img_path = sample_info['img_path']
        pid = sample_info['pid']
        cid = sample_info['cid']
        # print(cid, query_cam)
        # pdb.set_trace()
        # train.append((img_path, pid2label_train[pid], cid2label[cid]))
        if cid in query_cam:
            query.append((img_path, pid2label_val[pid], cid2label_val[cid]))
        else:
            gallery.append((img_path, pid2label_val[pid], cid2label_val[cid]))

    return train, query, gallery

@DATASET_REGISTRY.register()
class Fangdao_Fanhua_35stores(ImageDataset):
    dataset_dir = ''
    def __init__(self, root='datasets', **kwargs):

        cam_info = {'100000027':{'pos':['ee:bb:03:01:00:c3', 'ee:bb:03:01:02:98'], 'door': ['ee:ee:bb:03:10:05']},#位置重复  保留05
            '100001071':{'pos':['ee:bb:03:01:00:b2', 'ee:bb:03:01:03:37', 'ee:bb:03:01:02:93'], 'door': ['ee:bb:03:01:02:ad']},
            '100000159':{'pos':['ee:bb:03:01:00:e7', 'ee:bb:03:01:02:40'], 'door': ['ee:bb:03:01:01:c6']},
            '100001093':{'pos':['ee:bb:03:01:03:9a', 'ee:bb:03:01:03:31', 'ee:bb:03:01:02:1b', 'ee:bb:03:01:03:25', 'ee:bb:03:01:00:ce', 'ee:bb:03:01:03:05', 'ee:bb:03:01:03:93'], 'door': ['ee:bb:03:01:02:b3', 'ee:bb:03:01:02:00']},
            '100001082':{'pos':['ee:bb:03:01:01:13', 'ee:bb:03:01:03:71', 'ee:bb:03:01:01:cf'], 'door': []}}

        # self.dataset_dir = data_dir

        # fname_list = os.listdir(data_dir)
        val_dataname = ['100000027_2020-03-17']
        # val_dataname = '100001071_2020-02-18'

        query_cam = []
        for name in val_dataname:
            val_storeid = name.split('_')[0]
            query_cam += cam_info[val_storeid]['pos']
        query_cam = [x.replace(':', '-') for x in query_cam]

        train, query, gallery = get_all_sample(val_dataname, query_cam)

        # pdb.set_trace()

        self.train = train
        self.query = query
        self.gallery = gallery

        super(Fangdao_Fanhua_35stores, self).__init__(train, query, gallery, **kwargs)


from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from functools import partial
import os
import numpy as np
import time

# create augmented dataset
TRAIN_DIR = 'split_data/training/'
VAL_DIR = 'split_data/validation/'
IMG_HEIGHT = 224
IMG_WIDTH = 224
BATCH_SIZE = 64

train_datagen = ImageDataGenerator(
    rescale = 1 / 255,
    rotation_range = 45,
    width_shift_range = 0.2,
    height_shift_range = 0.2,
    horizontal_flip = True,
    zoom_range = 0.4,
    shear_range = 0.2,
) # data augmentation applied

val_datagen = ImageDataGenerator(
    rescale = 1 / 255
) # no data augmentation

train_generator = train_datagen.flow_from_directory(
    batch_size = BATCH_SIZE,
    directory = TRAIN_DIR,
    target_size = (IMG_HEIGHT, IMG_WIDTH),
    class_mode = 'sparse'
) # 3916 images
total_train = 3916

val_generator = val_datagen.flow_from_directory(
    batch_size = BATCH_SIZE,
    directory = VAL_DIR,
    target_size = (IMG_HEIGHT, IMG_WIDTH),
    class_mode = 'sparse'
) # 407 images
total_val = 407

# create convolutional base
CHANNELS = 3
IMG_SHAPE = (IMG_HEIGHT, IMG_WIDTH, CHANNELS) # MobileNetV2 is particular about input shapes (Has to be square)
base = keras.applications.MobileNetV2(
    input_shape = IMG_SHAPE,
    include_top = False,
    weights = 'imagenet'
)
base.trainable = False
# base.summary()

# add top layers
model = keras.Sequential([
    base,
    keras.layers.Dropout(0.2),
    keras.layers.GlobalAveragePooling2D(),
    keras.layers.Dropout(0.2),
    keras.layers.Dense(128, activation = 'relu'),
    keras.layers.Dropout(0.2),
    keras.layers.Dense(128, activation = 'relu'),
    keras.layers.Dropout(0.2),
    keras.layers.Dense(5, activation = 'softmax')
])
model.compile(
    optimizer = 'adam',
    loss = 'sparse_categorical_crossentropy',
    metrics = ['accuracy']
)
model.summary()

# define save weights callback
checkpoint_path = 'weights/W'
checkpoint_callback = keras.callbacks.ModelCheckpoint(
    checkpoint_path,
    save_weights_only = True
)

try:
    model.load_weights(checkpoint_path)
    print('Weights detected.')
except:
    print('No weights detected.')

# define tensorboard callback
root_logdir = os.path.join(os.curdir, 'logs')

def get_run_logdir():
    run_id = time.strftime('run_%Y_%m_%d-%H_%M_%S')
    return os.path.join(root_logdir, run_id)
run_logdir = get_run_logdir()

tensorboard_callback = keras.callbacks.TensorBoard(run_logdir)

# define nan callback
nan_callback = keras.callbacks.TerminateOnNaN()

# define early stop callback
early_stop_callback = keras.callbacks.EarlyStopping(
    monitor = 'val_loss',
    patience = 4,
    restore_best_weights = True
)

# train model
EPOCHS = 30

history = model.fit(
    train_generator,
    steps_per_epoch = total_train // BATCH_SIZE,
    epochs = EPOCHS,
    validation_data = val_generator,
    validation_steps = total_val // BATCH_SIZE,
    callbacks = [
        checkpoint_callback,
        tensorboard_callback,
        nan_callback,
        early_stop_callback
    ]
)

# evaluate model
model.evaluate(val_generator)

#