import logging
from typing import Dict, List, Tuple, Union

import numpy as np
import torch
import torch.nn.functional as F
from networkx import DiGraph
from torch import Tensor, nn as nn
from torch.autograd.variable import Variable

from binlin.data.ud import index_data
from binlin.model.nn_utils import get_embed_matrix, pad_seq
from binlin.model.syn.sympairs import SymModel
from binlin.model.syn.utils.bintree import BinTreeBase
from binlin.utils.combinatorics import flatten_nested_lists
from binlin.utils.constants import VocabSymbols

logger = logging.getLogger('main')

class SymGraphEncModel(SymModel):

    @property
    def _data_fields(self):
        return ['LEMMA', 'UPOS', 'XPOS', 'DEPREL']

    @property
    def _num_embedding_feats(self):
        # todo: need to avoid hard-coding
        # 6 = 2 * 3
        # 2 because we consider two nodes at a time,
        # 3 because for a node we consider itself + its context
        # consisting of its parent and a random child
        return 2 * 3 * self._data_fields_num

    def _init_weights(self):
        self._dim_emb = self.config["embedding_dim"]
        self._mat_emb = get_embed_matrix(len(self._id2tok), self._dim_emb, padding_idx=self.PAD_ID)

        self._dim_emb_proj_in = self._num_embedding_feats * self._dim_emb
        self._dim_emb_proj_out = self._num_embedding_feats * self.config['embedding_proj_dim']
        self._mat_emb_proj = nn.Linear(self._dim_emb_proj_in, self._dim_emb_proj_out)

        self._dim_dense_out = self.config["dense_dim"]
        self._mat_dense = nn.Linear(self._dim_emb_proj_out, self._dim_dense_out)

        self._mat_attn = nn.Linear(self._dim_emb, 1)

        self._dim_concat_in = self._dim_dense_out + self._dim_emb
        self._dim_concat_out = self._dim_dense_out
        self._mat_concat = nn.Linear(self._dim_concat_in, self._dim_concat_out)

        self._dim_out = 1
        self._mat_out = nn.Linear(self._dim_dense_out, self._dim_out)

    def forward(self, batch_data: Tuple[Tensor, Tensor, Union[Tensor, None]]) -> Dict:
        other_nodes_var, head_child_var, _ = batch_data

        # head-child pair is encoded using an MLP
        x_head_child = self._mat_emb(head_child_var).view(-1,
                                                          self._dim_emb_proj_in)  # size(num_node_pairs, self.emb_proj_in)
        x_head_child = F.leaky_relu(self._mat_emb_proj(x_head_child))  # size (num_node_pairs, self.emb_proj_out)
        x_head_child = F.leaky_relu(self._mat_dense(x_head_child))  # size (num_node_pairs, self.dense1_dim)
        # x_head_child = F.leaky_relu(x_head_child) # size (num_node_pairs, self.dense1_dim)

        # graph nodes are encoded using embedding lookup --> summing
        node_number = other_nodes_var.shape[-1]
        x_graph = self._mat_emb(other_nodes_var)  # size (max_num_nodes, batch_size, self.emb_dim)

        # variant1: sum over all vecs
        # x_graph = torch.sum(x_graph, dim=[0], keepdim=True).view(-1, self._dim_emb) # size (batch_size, emb_dim)

        # variant2: use attn scores
        attn_unnorm_scores = self._mat_attn(x_graph.view(-1, self._dim_emb))  # num_edges x 1
        attn_weights = F.leaky_relu(attn_unnorm_scores).view(-1, 1, node_number)

        # TODO: find a simpler way to do it w/o squeezing and unsqueezing?
        # apply attention weights to the graph vectors to get weighted average
        # size (1, dense)
        x_graph = torch.bmm(attn_weights, x_graph).squeeze(1)  # size: (bsize, emb_size)

        # Concat head, child and graph representations
        x_combined = torch.cat((x_head_child, x_graph), 1)  # size (bs, emb_dim + self.dense1_dim)

        # size (batch_size, self.dense1_dim)
        x_combined = self._mat_concat(x_combined)
        x_combined = F.leaky_relu(x_combined)
        x_combined = self._mat_out(x_combined)
        logits = torch.sigmoid(x_combined)

        return {'logits': logits}

    def extract_features(self, bt, new_node_nxid, dg, feats_d):

        # pair-level features
        head_nxid = bt.nxid
        head_deptree_feats = feats_d[head_nxid]
        child_deptree_feats = feats_d[new_node_nxid]
        x_pair_ids_l = head_deptree_feats + child_deptree_feats

        # extracting graph-level features
        graph_ids_l = self.extract_graph_level_feats(dg, new_node_nxid, bt, feats_d)

        return (graph_ids_l, x_pair_ids_l)

    def extract_graph_level_feats(self, dg, new_node_nxid, bt, feats_d):
        head_sbl = dg.node[bt.nxid]['sbl']
        if head_sbl is None:
            head_sbl_feats_l = self._dummy_node_feats_vec
        else:
            head_sbl_feats_l = flatten_nested_lists([feats_d[ch] for ch in head_sbl])

        child_sbl = dg.node[new_node_nxid]['sbl']
        if child_sbl is None:
            ch_sbl_feats_l = self._dummy_node_feats_vec
        else:
            ch_sbl_feats_l = flatten_nested_lists([feats_d[ch] for ch in child_sbl])

        child_children = dg[new_node_nxid]
        if len(child_children) == 0:
            ch_ch_feats_l = self._dummy_node_feats_vec
        else:
            ch_ch_feats_l = flatten_nested_lists([feats_d[ch] for ch in child_children])

        graph_ids_l = head_sbl_feats_l + ch_sbl_feats_l + ch_ch_feats_l
        return graph_ids_l

    def init_data_containers(self):
        return {'X': [],
                'Y': [],
                'Xg': []}

    def add_xy_pairs(self, data_containers: Dict, y: int, model_inputs: Tuple[List[int], List[int]]):
        x_graph, x_head_child = model_inputs
        data_containers['X'].append(x_head_child)
        data_containers['Xg'].append(x_graph)
        data_containers['Y'].append(y)

    def _batchify(self, data_containers: Dict, batch_size: int):

        # sort according to the lemma length
        sorted_data = sorted(zip(*(data_containers['Xg'],
                                   data_containers['X'],
                                   data_containers['Y'])), key=lambda p: len(p[0]), reverse=True)
        data_size = len(sorted_data)
        num_batches = data_size // batch_size
        data_indices = index_data(data_size, mode='no_shuffling')
        batch_pairs = []

        for bi in range(num_batches + 1):  # including the last (smaller) batch

            batch_x_pair_feats = []
            batch_x_graph_feats = []
            batch_x_lens = []
            batch_y = []

            curr_batch_indices = data_indices[bi * batch_size: (bi + 1) * batch_size]

            if len(curr_batch_indices) == 0:
                break

            for idx in curr_batch_indices:
                graph_f_ids, node_pairs_f_ids, y_ids = sorted_data[idx]
                batch_x_graph_feats.append(graph_f_ids)
                batch_x_lens.append(len(graph_f_ids))
                batch_x_pair_feats.append(node_pairs_f_ids)
                batch_y.append(y_ids)

            max_graph_f_len = max(batch_x_lens)
            batch_x_graph_feats_padded = [pad_seq(x, max_graph_f_len, pad_id=self.PAD_ID) for x in batch_x_graph_feats]

            # size: (num_nodes, batch_size)
            batch_x_graph_feats_var = Variable(torch.LongTensor(batch_x_graph_feats_padded)).to(self.device)
            # size: (batch_size, 2 * num_node_feats)
            batch_x_pair_feats_var = Variable(torch.LongTensor(batch_x_pair_feats)).to(self.device)
            # size: (batch_size, 1)
            batch_y_var = Variable(torch.FloatTensor(batch_y)).unsqueeze(1).to(self.device)
            batch_pairs.append((batch_x_graph_feats_var, batch_x_pair_feats_var, batch_y_var))

        return batch_pairs

    def make_decision(self,
                      bt: BinTreeBase,
                      new_node_nxid: str,
                      dg: DiGraph,
                      feats_d: Dict, *other_inputs) -> int:

        x_graph_ids_l, x_ids_l = self.extract_features(bt, new_node_nxid, dg, feats_d)
        x_ids_np = np.asarray(x_ids_l)
        x_graph_ids_l = np.asarray([x_graph_ids_l])
        outputs = self.__call__(
            (
                torch.from_numpy(x_graph_ids_l).to(self.device),
                torch.from_numpy(x_ids_np).to(self.device),
                None)
        )
        logit_val = outputs['logits'].cpu().data[0].numpy()

        if logit_val >= 0.5:
            decision = VocabSymbols.RIGHT
        else:
            decision = VocabSymbols.LEFT

        return decision

component = SymGraphEncModel


#!/usr/bin/env/python
"""Module for the formatter base class"""

import logging

from ioccheck.services import Service

class Formatter:  # pylint: disable=too-few-public-methods
    """Base class for creating human-friendly threat intelligence API output"""

    def __init__(self, service: Service):
        self.service = service

        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.INFO)

        f_handler = logging.FileHandler("ioccheck.log")
        f_handler.setLevel(logging.INFO)

        f_format = logging.Formatter(
            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
        )
        f_handler.setFormatter(f_format)

        self.logger.addHandler(f_handler)


"""This is a monster defender game."""

import pygame
import random
import time

from pygame import image

import image_lib
import sprites

def process_events(cmd_spites):
  """Check if user hits the quit button"""
  for event in pygame.event.get():
    if event.type == pygame.QUIT:
      return False
    elif event.type == pygame.KEYDOWN:
      if event.key == pygame.K_ESCAPE:
        return False
    for sprite in cmd_spites:
      sprite.process_event(event)
  return True

class MainCharacter(pygame.sprite.Sprite):
  def __init__(self, screen_size, all_sprites):
    super().__init__()
    size = (100,100)
    images = image_lib.images_from_file('image/SsmWh_4x9.png')
    self.left_image_sequence = images[9:18]
    self.right_image_sequence = images[27:]
    self.image_index = 0
    self.default_image = images[1]
    self.image = self.default_image
    self.ipu = 2
    self.rect = pygame.Rect((screen_size[0] // 2,
                             screen_size[1] - size[1] - 20), size)
    self.boundary = pygame.Rect((0, 0), screen_size)
    self.all_sprites = all_sprites

  def update(self):
    speed = 5
    keys_pressed = pygame.key.get_pressed()
    if keys_pressed[pygame.K_LEFT]:
      self.rect = self.rect.move(-speed, 0)
      self.image_index += 1
      self.image_index %= len(self.left_image_sequence) * self.ipu
      self.image = self.left_image_sequence[self.image_index // self.ipu]
    elif keys_pressed[pygame.K_RIGHT]:
      self.rect = self.rect.move(speed, 0)
      self.image_index += 1
      self.image_index %= len(self.right_image_sequence) * self.ipu
      self.image = self.right_image_sequence[self.image_index // self.ipu]
    else:
      self.image = self.default_image
      self.image_index = 0

  def shoot(self):
    image = pygame.image.load('image/bullet64x64.ico')
    image = pygame.transform.scale(image, (32, 32))
    image_info = sprites.ImageInfo(
        image=image,
        direction=pygame.Vector2(-10, 0))
    position = (self.rect.left + image_info.image.get_width() // 2,
                self.rect.top)
    bullet = sprites.Bullet(image_info, position,
        self.boundary,
        sprites.ShootgingStrategy.NO_TARGET, None)
    bullet.change_to_direction(pygame.Vector2(0, -10))
    self.all_sprites.add(bullet)

  def process_event(self, event):
    # Should not call pygame.event.get() here because it empties
    # the event queue so others won't get their events.
    if event.type == pygame.KEYDOWN and event.key == pygame.K_SPACE:
      self.shoot()

class MainBoard():
  def __init__(self):
    pygame.display.set_caption('Monster Defender')
    self.display_size = (1200, 800)
    self.display = pygame.display.set_mode(self.display_size)

    # Set program icon
    icon = pygame.image.load('image/FOOCHIR.jpeg').convert_alpha()
    pygame.display.set_icon(icon)

    bg_image = pygame.image.load('image/background/bg_lawn.jpg')
    self.bg_image = pygame.transform.scale(bg_image, self.display_size)
    self.all_sprites = pygame.sprite.Group()
    self.cmd_sprites = pygame.sprite.Group()
    self.targets = pygame.sprite.Group()
    self.mc = MainCharacter(self.display_size, self.all_sprites)
    self.all_sprites.add(self.mc)
    self.cmd_sprites.add(self.mc)

  def create_targets(self):
    if random.randint(0, 70) != 0:
      return
    margin = 10
    image = pygame.image.load('image/targets/nectarine.png').convert_alpha()
    x_pos = random.randint(margin, self.display_size[0] - margin)
    y_pos = margin
    velocity = pygame.Vector2(0, random.randint(1, 5))
    target = sprites.Target(image, pygame.Rect((0, 0), self.display_size),
                            (x_pos, y_pos), (50, 50), velocity)
    self.targets.add(target)
    self.all_sprites.add(target)

  def update(self):
    if not process_events(self.cmd_sprites):
      return False
    self.create_targets()
    self.display.blit(self.bg_image, (0, 0))
    self.all_sprites.update()
    self.all_sprites.draw(self.display)

    pygame.display.flip()
    return True

def main():
  """Program main function."""
  pygame.init()

  board = MainBoard()
  clock = pygame.time.Clock()
  while board.update():
    clock.tick(60)

  pygame.quit()

if __name__ == '__main__':
  main()


import sys
import os

dirname = os.path.dirname(__file__) # relative path
cur_dir = os.path.abspath(dirname) # convert relative path to absolute path

with open(os.path.join(cur_dir , '中文.txt'),'r',encoding = 'utf-8') as f:
    # print(f.read())
    for line in f.readlines():
        print(line.strip())

with open(os.path.join(cur_dir , 't.txt'),'r') as f:
    # print(f.read())
    for line in f.readlines():
        print(line.strip())


#!/usr/bin/env python
import time
import RPi.GPIO as GPIO, time, os

def open_for(seconds):
	GPIO.setmode(GPIO.BCM)
	GPIO.setup(17, GPIO.OUT)
	GPIO.output(17, False)
	time.sleep(seconds)
	GPIO.cleanup()


#!/usr/bin/python
""" PN CLI ipv6security-raguard-vlan-add/remove """

# Copyright 2018 Pluribus Networks
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import shlex
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.pn_nvos import pn_cli

DOCUMENTATION = """
---
module: pn_ipv6security_raguard_vlan
author: "Pluribus Networks (devops@pluribusnetworks.com)"
version: 2
short_description: CLI command to add/remove ipv6security-raguard-vlan.
description:
  - C(add): Add vlans to RA Guard Policy
  - C(remove): Remove vlans to RA Guard Policy
options:
  pn_cliswitch:
    description:
      - Target switch to run the CLI on.
    required: False
    type: str
  pn_action:
    description:
      - ipv6security-raguard-vlan configuration command.
    required: true
    choices: ['add', 'remove']
    type: str
  pn_vlans:
    description:
      - Vlans attached to RA Guard Policy
    required: false
    type: str
  pn_name:
    description:
      - RA Guard Policy Name
    required: false
    type: str
"""

EXAMPLES = """
- name: add ipv6 RA guard vlans
  pn_ipv6security_raguard_vlan:
    pn_action: 'add'
    pn_name: 'test'
    pn_vlans: '100'

- name: remove ipv6 RA guard vlans
  pn_ipv6security_raguard_vlan:
    pn_action: 'remove'
    pn_name: 'test'
    pn_vlans: '100'
"""

RETURN = """
command:
  description: the CLI command run on the target node.
stdout:
  description: set of responses from the ipv6security-raguard-vlan command.
  returned: always
  type: list
stderr:
  description: set of error responses from the ipv6security-raguard-vlan command.
  returned: on error
  type: list
changed:
  description: indicates whether the CLI caused changes on the target.
  returned: always
  type: bool
"""

def run_cli(module, cli):
    """
    This method executes the cli command on the target node(s) and returns the
    output. The module then exits based on the output.
    :param cli: the complete cli string to be executed on the target node(s).
    :param module: The Ansible module to fetch command
    """
    action = module.params['pn_action']
    cli = shlex.split(cli)
    rc, out, err = module.run_command(cli)

    # Response in JSON format
    if err:
        module.fail_json(
            command=' '.join(cli),
            stderr=err.strip(),
            msg="ipv6security-raguard-vlan %s operation failed" % action,
            changed=False
        )

    if out:
        module.exit_json(
            command=' '.join(cli),
            stdout=out.strip(),
            msg="ipv6security-raguard-vlan %s operation completed" % action,
            changed=True
        )

    else:
        module.exit_json(
            command=' '.join(cli),
            msg="ipv6security-raguard-vlan %s operation completed" % action,
            changed=True
        )

def main():
    """ This section is for arguments parsing """
    module = AnsibleModule(
        argument_spec=dict(
            pn_cliswitch=dict(required=False, type='str'),
            pn_action=dict(required=True, type='str',
                           choices=['add', 'remove']),
            pn_vlans=dict(required=False, type='str'),
            pn_name=dict(required=False, type='str'),
        )
    )

    # Accessing the arguments
    action = module.params['pn_action']
    vlans = module.params['pn_vlans']
    name = module.params['pn_name']

    # Building the CLI command string
    cli = pn_cli(module)
    cli += 'ipv6security-raguard-vlan-' + action

    if vlans:
        cli += ' vlans ' + vlans
    if name:
        cli += ' name ' + name

    run_cli(module, cli)

if __name__ == '__main__':
    main()


################################
### Mallet Oscar & Malik Jan ###
###     16 November 2018     ###
################################

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import os

#Pour le moment on test les graphiques sur les 50000 premierew lignes(car sinon trop lourd)
# Faudra surement commencer par du PySpark et faissant du Map/Reduce qui sera plus perf à mon avis
#Mais au moins ça fait un petit code qu'on pourra réutiliser

data = pd.read_csv("beer_reviews.csv", nrows=50000)
df1 = pd.read_csv("beer_reviews.csv", nrows=5000) #pour le graph : scatter matrix (sinon trop lourd)
data.dataframeName = 'beer_reviews_50000.csv'
df1.dataframeName = 'beer_reviews_5000.csv'

def plotPerColumnDistribution(df, nGraphShown, nGraphPerRow):
    nunique = df.nunique()
    df = df[[col for col in df if nunique[col] > 1 and nunique[col] < 50]] # For displaying purposes, pick columns that have between 1 and 50 unique values
    nRow, nCol = df.shape
    columnNames = list(df)
    nGraphRow = (nCol + nGraphPerRow - 1) / nGraphPerRow
    plt.figure(num = None, figsize = (6 * nGraphPerRow, 8 * nGraphRow), dpi = 80, facecolor = 'w', edgecolor = 'k')
    for i in range(min(nCol, nGraphShown)):
        plt.subplot(nGraphRow, nGraphPerRow, i + 1)
        columnDf = df.iloc[:, i]
        if (not np.issubdtype(type(columnDf.iloc[0]), np.number)):
            valueCounts = columnDf.value_counts()
            valueCounts.plot.bar()
        else:
            columnDf.hist()
        plt.ylabel('counts')
        plt.xticks(rotation = 90)
        plt.title(f'{columnNames[i]} (column {i})')
    plt.tight_layout(pad = 1.0, w_pad = 1.0, h_pad = 1.0)
    plt.show()

# Correlation matrix
def plotCorrelationMatrix(df, graphWidth):
    filename = df.dataframeName
    df = df.dropna('columns') # drop columns with NaN
    df = df[[col for col in df if df[col].nunique() > 1]] # keep columns where there are more than 1 unique values
    if df.shape[1] < 2:
        print(f'No correlation plots shown: The number of non-NaN or constant columns ({df.shape[1]}) is less than 2')
        return
    corr = df.corr()
    plt.figure(num=None, figsize=(graphWidth, graphWidth), dpi=80, facecolor='w', edgecolor='k')
    corrMat = plt.matshow(corr, fignum = 1)
    plt.xticks(range(len(corr.columns)), corr.columns, rotation=90)
    plt.yticks(range(len(corr.columns)), corr.columns)
    plt.gca().xaxis.tick_bottom()
    plt.colorbar(corrMat)
    plt.title(f'Correlation Matrix for {filename}', fontsize=15)
    plt.show()

# Scatter and density plots
def plotScatterMatrix(df, plotSize, textSize):
    df = df.select_dtypes(include =[np.number]) # keep only numerical columns
    # Remove rows and columns that would lead to df being singular
    df = df.dropna('columns')
    df = df[[col for col in df if df[col].nunique() > 1]] # keep columns where there are more than 1 unique values
    columnNames = list(df)
    if len(columnNames) > 10: # reduce the number of columns for matrix inversion of kernel density plots
        columnNames = columnNames[:10]
    df = df[columnNames]
    ax = pd.plotting.scatter_matrix(df, alpha=0.75, figsize=[plotSize, plotSize], diagonal='kde')
    corrs = df.corr().values
    for i, j in zip(*plt.np.triu_indices_from(ax, k = 1)):
        ax[i, j].annotate('Corr. coef = %.3f' % corrs[i, j], (0.8, 0.2), xycoords='axes fraction', ha='center', va='center', size=textSize)
    plt.suptitle('Scatter and Density Plot')
    plt.show()

plotPerColumnDistribution(data, 10, 5)
plotCorrelationMatrix(data, 12)
plotScatterMatrix(df1, 15, 2)

#Drop some unnecessary columns for plots
data = data.drop(["brewery_id"], axis=1)
data = data.drop(["review_time"], axis=1)
#data = data.drop(["review_profilename"], axis=1)
data = data.drop(["beer_beerid"], axis=1)
data = data.dropna()
data.head()

#Arrange the columns and sort by brewery name
data = data[['brewery_name', 'beer_style', 'beer_name', 'beer_abv', 'review_appearance',
             'review_aroma', 'review_palate', 'review_taste', 'review_overall']]
data = data.sort_values(by=['brewery_name', 'beer_style', 'beer_name' , 'beer_abv', 'review_overall'])
data = data.reset_index()
data = data.drop(["index"], axis=1)
data.head()

# New column review_average
data['review_average'] = data.apply(lambda row: (row["review_overall"] + row["review_aroma"] +
                                                 row["review_appearance"] + row["review_palate"] +
                                                 row["review_taste"]) / 5, axis=1)

data = data.drop(data[(data["review_average"] < 1) | (data["review_average"] > 5)].index)
data.head()
#Visu of top 15
def visualization_function_1(name, condition, ax_left, ax_right):

    unique_names = data[name].unique()
    length = len(unique_names)
    rev_aver = np.zeros(length)
    count = np.zeros(length, dtype=np.int32)

    for _, row in data.iterrows():
        idx = np.where(unique_names == row[name])
        rev_aver[idx] += row["review_average"]
        count[idx] += 1

    for i in range(length):
        if count[i] > condition:
            rev_aver[i] /= count[i]
        else:
            rev_aver[i] = 1

    zipped_left  = sorted(zip(unique_names, rev_aver), key=lambda x: x[1], reverse=True)
    names_left   = [zipped_left[i][0] for i in range(length)]
    sorted_score = [zipped_left[i][1] for i in range(length)]

    zipped_right = sorted(zip(unique_names, count), key=lambda x: x[1], reverse=True)
    names_right  = [zipped_right[i][0] for i in range(length)]
    sorted_count = [zipped_right[i][1] for i in range(length)]

    sns.barplot(sorted_score[:15], names_left[:15], ax=ax_left)
    ax_left.set_xlim(3, 5)
    ax_left.set_xlabel("Scores of review_average")
    # limit the length of names
    ax_left.set_yticklabels([i.get_text()[:17] + "..." if len(i.get_text()) > 17 else i.get_text()
                             for i in ax_left.get_yticklabels()])

    sns.barplot(sorted_count[:15], names_right[:15], ax=ax_right).set_title(name)
    ax_right.set_xlabel("Total number of drinks")
    # limit the length of names
    ax_right.set_yticklabels([i.get_text()[:17] + "..." if len(i.get_text()) > 17 else i.get_text()
                              for i in ax_right.get_yticklabels()])

sns.set(rc={"axes.grid": True})
fig, axs = plt.subplots(ncols=2, nrows=3, figsize=[16, 24])
fig.subplots_adjust(wspace=0.4)

visualization_function_1("brewery_name", 5, axs[0, 0], axs[0, 1])
visualization_function_1("beer_style", 5, axs[1, 0], axs[1, 1])
visualization_function_1("beer_name", 5, axs[2, 0], axs[2, 1])

axs[0, 0].set_title("The best quality Breweries")
axs[0, 1].set_title("Breweries that produce the most drinks")

axs[1, 0].set_title("The best styles of beer")
axs[1, 1].set_title("The most popular styles of beer")

axs[2, 0].set_title("The best quality beer")
axs[2, 1].set_title("The most popular beer");

plt.show()

#Scatter 'to improve'
plt.plot(data['beer_abv'], data['review_average'], 'ro')
plt.xlabel('Alcool %')
plt.ylabel('Appreciation')
plt.title('Alcool vs Appreciation')
plt.show()

#Scatter 'to improve' BIS
plt.plot(data['review_average'], data['beer_abv'], 'ro')
plt.xlabel('Appreciation')
plt.ylabel('Alcool %')
plt.title('Appreciation vs Alcool')
plt.show()

#Hist sur notes
x = data['review_average']
n, bins, patches = plt.hist(x)
plt.xlabel('Mark')
plt.ylabel('Density')
plt.title('Histogram of Appreciations')
plt.show()

#Hist sur abv
x = data['beer_abv']
n, bins, patches = plt.hist(x)
plt.xlabel('Alcool')
plt.ylabel('Density')
plt.title('Histogram of ABV')
plt.show()

Big Data Tools/Beer ABV Project/Scripts/Plots_50000.py


import requests

from pycompat import python as py

if py.is2xx:
    from HTMLParser import HTMLParser
    from urllib import urlencode as url_encode
else:
    from html.parser import HTMLParser
    from urllib.parse import urlencode as url_encode

LO_CGIT_LOG_URL = 'http://cgit.freedesktop.org/libreoffice/core/log'

class CGitParser(HTMLParser):

    def __init__(self):
        HTMLParser.__init__(self)
        self._inside_table = False
        self._inside_table_row = False
        self._inside_td_log = False
        self._index = 0
        self._last_url = None
        self._last_log = None
        self._skip_data = False
        self.items = {}

    def get(self, obj, key):
        if obj:
            for x in obj:
                if x[0] == key:
                    return x[1]

        return None

    def handle_starttag(self, tag, attrs):
        if tag.lower() == 'table' and self.get(attrs, 'class') == 'list nowrap':
            self._inside_table = True

        elif self._inside_table and tag.lower() == 'tr' and self.get(attrs, 'class') != 'nohover':
            self._inside_table_row = True
            self._index += 1

        elif self._inside_table and self._inside_table_row and tag.lower() == 'a' and not self.get(attrs, 'class'):
            self._last_url = self.get(attrs, 'href')

        elif tag.lower() == 'td' and self.get(attrs, 'class') == 'logmsg':
            self._inside_td_log = True

        elif tag.lower() == 'span' and self.get(attrs, 'class') == 'decoration':
            self._skip_data = True

    def handle_endtag(self, tag):
        if self._inside_table and self._inside_table_row and tag.lower() == 'tr':
            self.items[self._index].append(self._last_url)
            self.items[self._index].append(self._last_log)
            self._inside_table_row = False
            self._last_url = None

        elif self._inside_table and tag.lower() == 'table':
            self._inside_table = False

        elif self._inside_td_log and tag.lower() == 'td':
            self._inside_td_log = False

        elif self._skip_data and tag.lower() == 'span':
            self._skip_data = False

    def handle_data(self, data):
        if self._inside_table and self._inside_table_row:
            if self._index > 0:
                if not self.items.get(self._index):
                    self.items[self._index] = []
                if not self._skip_data:
                    self.items[self._index].append(data)
        elif self._inside_table and self._inside_td_log:
            self._last_log = data

parser = CGitParser()

def download_page(page=1):
    if page < 1:
        page = 1

    page_num = (page - 1) * 50

    try:
        data = url_encode({ 'showmsg': 1, 'ofs': page_num })

        headers = { 'Accept-Charset': 'utf-8',
                    'Cache-Control': 'no-cache, max-age=0, s-maxage=0, must-revalidate, proxy-revalidate',
                    'Pragma': 'no-cache',
                    'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
                    # 'Date': datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT'),
                   }

        response = requests.get('%s?%s' % (LO_CGIT_LOG_URL, data), headers=headers)

        if response.status_code == 200:
            return response.text
    except Exception as e:
        print('Error: %s' % e.message)

    return None

def parse_html(html):
    data = []

    if html:
        parser.feed(html)

        for k, v in parser.items.items():
            log = (v[-1] or '').strip()
            data.append({ 'age' : v[0],\
                          'commit_message' : ''.join(v[1:-5]),\
                          'author' : v[-5],
                          'files' : v[-4],\
                          'lines' : v[-3],\
                          'url' : v[-2],\
                          'log' : log })

    return data

def get_commits():
    # l = []
    # l += parse_html(download_page(0))
    # l += parse_html(download_page(1))
    # l += parse_html(download_page(2))
    # return l
    return parse_html(download_page(0))

if __name__ == '__main__':
    commits = get_commits()

    for c in commits:
        print(c)


from django.contrib import admin
from django.urls import include, path
from .views import home, respostas, verifica_pergunta, professor, professor_lista

urlpatterns = [
    path('', home, name='ftc_home'),
    path('professor', professor, name='ftc_professor'),
    path('professor-alunos', professor_lista, name='ftc_professor_lista'),
    path('perguntas/<int:aula>/', respostas, name='ftc_perg'),
    path('verifica-pergunta/<int:aula>/', verifica_pergunta, name='ftc_verifica_pergunta'),
]


from alpha_vantage.timeseries import TimeSeries
import config
from time_sim import *
# Your key here
key = config.avKey
ts = TimeSeries(key)
aapl, meta = ts.get_daily(symbol='AAPL')
#dates = ['2019-09-12','2020-01-02','2020-01-03']
#for i in dates:
#    print(aapl[i])
times = getTimes('2019-12-12','2020-01-03')
aapl, meta = ts.get_intraday(symbol='AAPL',interval='5min',outputsize='full')
for i in times:
    print(i,aapl[i])


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.macosx-10.14-x86_64/egg/limcv/template.py
# Compiled at: 2019-12-11 22:37:24
# Size of source mod 2**32: 5408 bytes
"""模板匹配.

对用户提供的调节参数:
    1. threshod: 筛选阈值，默认为0.8
    2. rgb: 彩色三通道,进行彩色权识别.
"""
import os, cv2
from six import PY2, PY3
import numpy as np, json
from .utils import generate_result, check_source_larger_than_search, img_mat_rgb_2_gray
from .cal_confidence import cal_rgb_confidence
from .match import get_perfect_scale
from .sift import find_sift
from .aircv import imread, resize

def find(im_source_path, im_search_path):
    im_source = imread(im_source_path)
    im_search = imread(im_search_path)
    result = find_all_template_multy_scale(im_source, im_search)
    if result:
        return result
    result = find_sift(im_source, im_search)
    return result

def find_by_sift(im_source_path, im_search_path):
    im_source = imread(im_source_path)
    im_search = imread(im_search_path)
    result = find_sift(im_source, im_search)
    return result

def find_all_template_multy_scale(im_source, im_search):
    result = []
    result = find_all_template(im_source, im_search)
    if result == None:
        scale = get_perfect_scale(im_source, im_search)
        result = find_all_template(im_source, im_search, scale=scale)
    return result

def find_template(im_source, im_search, threshold=0.8, scale=1.0, rgb=False):
    """函数功能：找到最优结果."""
    if scale == 0:
        scale = 1
    if scale != 1:
        im_source = resize(im_source, scale)
    check_source_larger_than_search(im_source, im_search)
    res = _get_template_result_matrix(im_source, im_search)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
    h, w = im_search.shape[:2]
    confidence = _get_confidence_from_matrix(im_source, im_search, max_loc, max_val, w, h, rgb)
    middle_point, rectangle = _get_target_rectangle(max_loc, w, h)
    best_match = generate_result(middle_point, rectangle, confidence)
    if confidence >= threshold:
        return best_match

def find_all_template(im_source, im_search, threshold=0.8, scale=1.0, rgb=False, max_count=10):
    """根据输入图片和参数设置,返回所有的图像识别结果."""
    if scale == 0:
        scale = 1
    if scale != 1:
        im_source = resize(im_source, scale)
    check_source_larger_than_search(im_source, im_search)
    res = _get_template_result_matrix(im_source, im_search)
    result = []
    h, w = im_search.shape[:2]
    while 1:
        min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
        confidence = _get_confidence_from_matrix(im_source, im_search, max_loc, max_val, w, h, rgb)
        if not confidence < threshold:
            if len(result) > max_count:
                break
            middle_point, rectangle = _get_target_rectangle(max_loc, w, h)
            one_good_match = generate_result(middle_point, rectangle, confidence)
            result.append(one_good_match)
            cv2.rectangle(res, (int(max_loc[0] - w / 2), int(max_loc[1] - h / 2)), (int(max_loc[0] + w / 2), int(max_loc[1] + h / 2)), (0,
                                                                                0,
                                                                                0), -1)

    if result:
        return result

def _get_confidence_from_matrix(im_source, im_search, max_loc, max_val, w, h, rgb):
    """根据结果矩阵求出confidence."""
    if rgb:
        img_crop = im_source[max_loc[1]:max_loc[1] + h, max_loc[0]:max_loc[0] + w]
        confidence = cal_rgb_confidence(img_crop, im_search)
    else:
        confidence = max_val
    return confidence

def _get_template_result_matrix(im_source, im_search):
    """求取模板匹配的结果矩阵."""
    s_gray, i_gray = img_mat_rgb_2_gray(im_search), img_mat_rgb_2_gray(im_source)
    return cv2.matchTemplate(i_gray, s_gray, cv2.TM_CCOEFF_NORMED)

def _get_target_rectangle(left_top_pos, w, h):
    """根据左上角点和宽高求出目标区域."""
    x_min, y_min = left_top_pos
    x_middle, y_middle = int(x_min + w / 2), int(y_min + h / 2)
    left_bottom_pos, right_bottom_pos = (
     x_min, y_min + h), (x_min + w, y_min + h)
    right_top_pos = (x_min + w, y_min)
    middle_point = (
     x_middle, y_middle)
    rectangle = (
     left_top_pos, left_bottom_pos, right_bottom_pos, right_top_pos)
    return (
     middle_point, rectangle)


from flask import Flask, redirect, render_template, request, Response
import requests
from typing import List
import json
import nltk

app = Flask(__name__)

def text_into_sentences(file: str) -> List[str]:
    with open(file, 'rt', encoding='cp1251') as in_file:
        sentence_list = []
        for line in in_file:
            sentence_list += nltk.sent_tokenize(line)
        return sentence_list

def find_sentences(sentences: List[str], size: int, word: str) -> List[str]:
    sentence_list = []
    for sent in sentences:
        if size == 0:
            break
        lower_sent = sent.lower()
        word = word.lower()
        if word in lower_sent:
            sentence_list.append(sent)
            size -= 1
    return sentence_list

def find_sentences2(sentences: List[str], _from: int, _to: int, word: str) -> List[str]:
    sentence_list = []
    counter = 1
    for sent in sentences:
        if counter > _to:
            break
        lower_sent = sent.lower()
        word = word.lower()
        if word in lower_sent:
            if counter >= _from:
                sentence_list.append(sent)
            counter += 1
    return sentence_list

@app.route('/')
def redirected():
    return redirect('/search')

@app.route('/search')
def hello():
    return render_template('index.html')

@app.route('/result', methods=['GET', 'POST'])
def func():
    count = int(request.form.get('count'))
    word = str(request.form.get('word'))
    sentences = text_into_sentences('Harry Potter and the Sorcerer.txt')
    sentences = find_sentences(sentences, count, word)
    return render_template('rez.html', data=sentences, kek=word)

@app.route('/request', methods=['POST'])
def function():
    data = request.json
    word = data['word']
    _from = data['from']
    _to = data['to']
    sentences = text_into_sentences('Harry Potter and the Sorcerer.txt')
    sentences = find_sentences2(sentences, _from, _to, word)
    dict = {
        "examples": sentences,
        "examples_count": len(sentences),
        "from": _from,
        "to": _to,
        "word": word
    }
    return dict

if __name__ == '__main__':
    app.run()


# -*- coding: utf-8 -*-
import sys, os

extensions = ['sphinx.ext.autodoc']
templates_path = ['.templates']
source_suffix = '.rst'
master_doc = 'index'

# General information about the project.
project = u'Ella'
copyright = u'2009, Centrum Holdings'

import ella as project

version = ".".join(map(str, project.__version__[:-1]))
# The full version, including alpha/beta/rc tags.
release = project.__versionstr__


#VehicleMovemenManager.py
import time

#TYPE: HNA-LAYER-2
#MODIFICATION: This file does  NOT need to change
#DATE: 05-05-2020
"""
DESCRIPTION:
#Coverts the speed of the car from Km/h to m/seg --> m/millisecond
"""

def speed_m_seg(speed_km_h):
	speed=0
	speed_km_h=float(speed_km_h)
	speed=float(speed_km_h*1000/(3600))
	return speed

#Transforms the speed from km/h into m/ (x) millisecond
#return--> float speed in m/ (x) milliseconds
#Parameters:
#speed_km_h: speed of the vehicle in km/h
#rate_ms indicates the rate of the number of miliseconds we want to get the speed in
    #How many meters does the car move in <rate_ms>ms
def speed_m_mseg(speed_km_h, rate_ms):

	m_s= speed_m_seg(speed_km_h)
	div=1000/rate_ms
	speed= m_s/div

	return speed

#updates the current position of the vehicle. It is called once per iteration (the iteration time must be equal to the <rate_ms>ms
#last_position_car: position of the car before moving it
#speed: distance it moves in (x) milliseconds --> obtained using  speed_m_mseg( )
def move_vehicle(last_position_car, speed):
    actual_position_car=last_position_car+speed #moves the car with real speed each Xms
    return actual_position_car

if __name__ == "__main__":
    speed_km_h=50

    print ("50km/h speed in m/ms: ", speed_m_mseg(speed_km_h, 1))
    print ("50km/h speed in m/5ms: ", speed_m_mseg(speed_km_h, 5))
    print ("50km/h speed in m/10ms: ", speed_m_mseg(speed_km_h, 10)) #How many meters does the car moves in 10ms
    print ("50km/h speed in m/100ms: ", speed_m_mseg(speed_km_h, 100))
    last_position_car=0
    mov_time=100
    speed =speed_m_mseg(speed_km_h, mov_time)
    print( "moving the vehicle each", mov_time, "ms, at a speed of",speed_km_h, "km/h-->" ,speed,"m/",mov_time,"ms")
    for i in range (0,25):

        actual_position_car=move_vehicle(last_position_car,speed)
        last_position_car=actual_position_car
        print("time: " , time.time())
        print ("position vehicle: ",actual_position_car)
        time.sleep(0.1)


# -*- coding: utf-8 -*-
import re
import requests
import pandas as pd
from fake_useragent import UserAgent

url = 'https://www.hao123.com/'
ua = UserAgent()
headers = {'User-Agent':ua.random}

resp = requests.get(url,headers)
data = resp.text
urls = re.findall(r'href="(http.*?)"',data)
print(urls)

df = pd.DataFrame()

#取2000个
df['url'] = urls[:len(urls)]
df.to_csv('TestUrl.csv',index=None)


import pandas as pd                              # tables and data manipulations
import matplotlib.pyplot as plt                  # plots
import numpy as np                               # vectors and matrices

from sklearn.preprocessing import StandardScaler
# import seaborn as sns                            # more plots

from sklearn.linear_model import LassoCV, RidgeCV
# from sklearn.linear_model import LinearRegression
# from xgboost import XGBRegressor
from sklearn.model_selection import cross_val_score, TimeSeriesSplit

# from src.mlModels.ExtractFeatures import extractFeatures
from src.TsUtils import mean_absolute_percentage_error
# from src.mlModels.SelectFeatures import selectFeatures

#
# загрузить данные
#
wavesFilePath = "../../data/waves_price.csv"
candyFilePath = "../../data/candy_production.csv"

# date_column_name = 'Date'
# y_column_name = 'Close'
date_column_name = 'observation_date'
y_column_name = 'IPG3113N'
#
data = pd.read_csv(candyFilePath, index_col=[date_column_name])
#
# data['Volume'] = [x.replace(',', '') for x in data['Volume']]
# data['Market Cap'] = [x.replace(',', '') for x in data['Market Cap']]
#
# data = data.loc[data['Market Cap'] != '-',:]

data.rename(columns={y_column_name:'y'}, inplace=True)

#
# крос валидация
#
tscv = TimeSeriesSplit(n_splits=5)

def timeseries_train_test_split(X, y, test_size, minLag):
    """
        Perform train-test split with respect to time series structure
    """

    X_predict = X.iloc[-minLag:]

    X = X.iloc[:-minLag]

    # get the index after which test set starts
    test_index = int(len(X) * (1 - test_size))

    X_train = X.iloc[:test_index]
    y_train = y.iloc[:test_index]
    X_test = X.iloc[test_index:]
    y_test = y.iloc[test_index:]

    return X_train, X_test, y_train, y_test, X_predict

min_lag = 6
max_lag = 25
# y_column_name = "Close"

dataSize = data.__len__()

columnNames = data.columns

for i in range(dataSize, dataSize + min_lag):
    data.loc[i] = [None for n in range(len(columnNames))]

for name in data.columns:
    for i in range(min_lag, max_lag):
        data["lag_{}_{}".format(name, i)] = data[name].shift(i)

y = data.dropna().y

data.drop(labels=columnNames, axis=1, inplace=True)

X = data.dropna()

#
# разбиваем на тестовую и трееровочную выборку
#

print("-----------------")
print(X.head())
print("-----------------")
print(y[:5])

X_train, X_test, y_train, y_test, X_predict = timeseries_train_test_split(X, y, 0.3, min_lag)

#
# тренеруем модель
#

scaler = StandardScaler()

# X_train = scaler.fit_transform(X_train)
# X_test = scaler.transform(X_test)

# lr = LinearRegression()
lr = LassoCV(cv=tscv)
# lr = RidgeCV(cv=tscv)
# lr = XGBRegressor()
lr.fit(X_train, y_train)

def plotModelResults(model, X_train=X_train, X_test=X_test, plot_intervals=False, plot_anomalies=False):
    """
        Plots modelled vs fact values, prediction intervals and anomalies

    """

    prediction = model.predict(X_test)

    plt.figure(figsize=(15, 7))
    plt.plot(prediction, "g", label="prediction", linewidth=2.0)
    plt.plot(y_test.values, label="actual", linewidth=2.0)

    if plot_intervals:
        cv = cross_val_score(model, X_train, y_train,
                             cv=tscv,
                             scoring="neg_mean_absolute_error")
        mae = cv.mean() * (-1)
        deviation = cv.std()

        scale = 1.96
        lower = prediction - (mae + scale * deviation)
        upper = prediction + (mae + scale * deviation)

        plt.plot(lower, "r--", label="upper bond / lower bond", alpha=0.5)
        plt.plot(upper, "r--", alpha=0.5)

        if plot_anomalies:
            anomalies = np.array([np.NaN] * len(y_test))
            anomalies[y_test < lower] = y_test[y_test < lower]
            anomalies[y_test > upper] = y_test[y_test > upper]
            plt.plot(anomalies, "o", markersize=10, label="Anomalies")

    error = mean_absolute_percentage_error(prediction, y_test)
    plt.title("Mean absolute percentage error {0:.2f}%".format(error))
    plt.legend(loc="best")
    plt.tight_layout()
    plt.grid(True)
    plt.savefig("./waves_Predicted_lag.png")

def plotCoefficients(model):
    """
        Plots sorted coefficient values of the model
    """

    coefs = pd.DataFrame(model.coef_, X_train.columns)
    coefs.columns = ["coef"]
    coefs["abs"] = coefs.coef.apply(np.abs)
    coefs = coefs.sort_values(by="abs", ascending=False).drop(["abs"], axis=1)

    plt.figure(figsize=(15, 7))
    coefs.coef.plot(kind='bar')
    plt.grid(True, axis='y')
    plt.hlines(y=0, xmin=0, xmax=len(coefs), linestyles='dashed')
    plt.savefig("./waves_Coef_lag.png")

def plotPredict(model, x_predict):
    predict = model.predict(x_predict)
    plt.figure(figsize=(15, 7))
    plt.plot(predict, "g", label="new_value", linewidth=2.0)

# predict = lr.predict(x_predict)
plt.figure(figsize=(15, 7))
plt.plot(y_test, "g", label="new_value", linewidth=2.0)
prediction = lr.predict(X_test)
plt.plot(prediction, label="actual", linewidth=2.0)
# plt.plot(lr.predict(X_predict), label="pred", linewidth=2.0)

error = mean_absolute_percentage_error(prediction, y_test)
plt.title("Mean absolute percentage error {0:.2f}%".format(error))
plt.legend(loc="best")
plt.tight_layout()
plt.grid(True)

plotModelResults(lr, plot_intervals=True, plot_anomalies=False)
plotPredict(lr, X_predict)
plotCoefficients(lr)
plt.show()


import pickle
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression

with open("final_project_dataset.pkl", "rb") as data_file:
    data_dict = pickle.load(data_file)

numeric_features = list(data_dict['SULLIVAN-SHAKLOVITZ COLLEEN'].keys())
numeric_features.remove('email_address')
numeric_features.remove('poi')

df = pd.DataFrame(data_dict)
df = df.transpose()

df[numeric_features] = df[numeric_features].astype(np.float)
df['poi'] = df['poi'].astype(bool)
df['email_address'] = df['email_address'].astype(str)

df = df.drop(['TOTAL'])
df = df.drop(['BELFER ROBERT'])
df = df.drop(['BHATNAGAR SANJAY'])
df = df.drop(['THE TRAVEL AGENCY IN THE PARK'])
df = df.drop(['KITCHEN LOUISE'])

for feature in numeric_features:
    df[feature] = df[feature].fillna(df[feature].median())

df['from_poi_proportion'] = df['from_poi_to_this_person'] / df['to_messages']
df['to_poi_proportion'] = df['from_this_person_to_poi'] / df['from_messages']

feature_list = list(df.columns.values.tolist())
feature_list.remove('poi')
feature_list.remove('email_address')
labels = df.poi

feature_list.remove('director_fees')
feature_list.remove('loan_advances')
feature_list.remove('restricted_stock_deferred')

ss = StandardScaler()
ss.fit(df[feature_list])

clf = LogisticRegression(C=0.001, n_jobs=-1)
clf.fit(ss.transform(np.asarray(df[feature_list])), labels)

feature_list.append('poi')
feature_list = feature_list[-1:] + feature_list[:-1]

dataset = df[feature_list].copy()
dataset['poi'] = df['poi'].astype(np.float)

for i in range(1, len(feature_list)):
    dataset[feature_list[i]] = (dataset[feature_list[i]] - ss.mean_[i-1])*ss.scale_[i-1]/ss.var_[i-1]

dataset = dataset.transpose()
dataset = dataset.to_dict()

CLF_PICKLE_FILENAME = "my_classifier.pkl"
DATASET_PICKLE_FILENAME = "my_dataset.pkl"
FEATURE_LIST_FILENAME = "my_feature_list.pkl"

with open(CLF_PICKLE_FILENAME, "wb") as clf_outfile:
    pickle.dump(clf, clf_outfile)
with open(DATASET_PICKLE_FILENAME, "wb") as dataset_outfile:
    pickle.dump(dataset, dataset_outfile)
with open(FEATURE_LIST_FILENAME, "wb") as featurelist_outfile:
    pickle.dump(feature_list, featurelist_outfile)

P5 - Identity Fraud from Enron Email/P5 Submission - Yannick SCHINI/poi_id_with_corrections.py


#calss header
class _SPECTRUM():
	def __init__(self,):
		self.name = "SPECTRUM"
		self.definitions = [u'the set of colours into which a beam of light can be separated, or a range of waves, such as light waves or radio waves: ', u'a range of different positions, opinions, etc. between two extreme points: ']

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'nouns'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


from dex.command.CommandBase import CommandBase
from dex.dextIR import ValueIR

class DexExpectStepOrder(CommandBase):
    def __init__(self, *args):
        if not args:
            raise TypeError('Need at least one order number')

        self.sequence = [int(x) for x in args]
        super(DexExpectStepOrder, self).__init__()

    def __call__(self, debugger):
        step_info = debugger.get_step_info()
        loc = step_info.current_location
        return {'DexExpectStepOrder': ValueIR(expression=str(loc.lineno),
                      value=str(debugger.step_index), type=None,
                      error_string=None,
                      could_evaluate=True,
                      is_optimized_away=True,
                      is_irretrievable=False)}


# -*- coding: utf-8 -*-
"""Views related to project management."""
from flask import (Blueprint, request, render_template, flash, url_for,
                    redirect, session, current_app, jsonify)
from flask_login import login_required, current_user

from dribdat.user.models import Event, Project
from dribdat.public.forms import (
    ProjectNew, ProjectForm, ProjectDetailForm, ProjectPost,
)
from dribdat.database import db
from dribdat.extensions import cache
from dribdat.aggregation import (
    SyncProjectData, GetProjectData,
    ProjectActivity, IsProjectStarred,
)
from dribdat.user import (
    validateProjectData, projectProgressList, isUserActive,
)

blueprint = Blueprint('project', __name__, static_folder="../static", url_prefix='/project')

def current_event(): return Event.current()

@blueprint.route('/<int:project_id>')
def project_view(project_id):
    return project_action(project_id, None)

@blueprint.route('/<int:project_id>/edit', methods=['GET', 'POST'])
@login_required
def project_edit(project_id):
    return project_edit_action(project_id)

@blueprint.route('/<int:project_id>/details', methods=['GET', 'POST'])
@login_required
def project_details(project_id):
    return project_edit_action(project_id, True)

def project_edit_action(project_id, detail_view=False):
    """ Project editing handler """
    project = Project.query.filter_by(id=project_id).first_or_404()
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (isUserActive(current_user) and current_user.is_admin)
    if not allow_edit:
        flash('You do not have access to edit this project.', 'warning')
        return project_action(project_id, None)
    if not detail_view:
        form = ProjectForm(obj=project, next=request.args.get('next'))
        form.category_id.choices = [(c.id, c.name) for c in project.categories_all()]
        if len(form.category_id.choices) > 0:
            form.category_id.choices.insert(0, (-1, ''))
        else:
            del form.category_id
    else:
        form = ProjectDetailForm(obj=project, next=request.args.get('next'))
    if form.validate_on_submit():
        del form.id
        form.populate_obj(project)
        project.update()
        db.session.add(project)
        db.session.commit()
        cache.clear()
        flash('Project updated.', 'success')
        project_action(project_id, 'update', False)
        return redirect(url_for('project.project_view', project_id=project.id))
    return render_template('public/projectedit.html', detail_view=detail_view,
        current_event=project.event, project=project, form=form)

@blueprint.route('/<int:project_id>/resource/add', methods=['GET', 'POST'])
@login_required
def resource_add(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (isUserActive(current_user) and current_user.is_admin)
    if not allow_edit:
        flash('You do not have access to add a resource.', 'warning')
        return project_action(project_id, None)
    resource = Resource()
    form = ResourceForm(obj=resource, next=request.args.get('next'))
    if form.validate_on_submit():
        del form.id
        form.populate_obj(resource)
        db.session.add(resource)
        db.session.commit()
        cache.clear()
        flash('Resource added.', 'success')
        # project_action(project_id, 'resource', False)
        return redirect(url_for('project.project_view', project_id=project.id))
    return render_template('public/resourcenew.html',
        current_event=project.event, project=project, resource=resource, form=form)

@blueprint.route('/<int:project_id>/resource/<int:resource_id>/edit', methods=['GET', 'POST'])
@login_required
def resource_edit(project_id, resource_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    resource = Resource.query.filter_by(id=resource_id).first_or_404()
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (isUserActive(current_user) and current_user.is_admin)
    if not allow_edit:
        flash('You do not have access to edit this resource.', 'warning')
        return project_action(project_id, None)
    form = ResourceForm(obj=resource, next=request.args.get('next'))
    if form.validate_on_submit():
        del form.id
        form.populate_obj(resource)
        db.session.add(resource)
        db.session.commit()
        cache.clear()
        flash('Resource updated.', 'success')
        # project_action(project_id, 'resource', False)
        return redirect(url_for('project.project_view', project_id=project.id))
    return render_template('public/resourceedit.html',
        current_event=project.event, project=project, resource=resource, form=form)

@blueprint.route('/<int:project_id>/resource/<int:resource_id>/del', methods=['GET', 'POST'])
@login_required
def resource_delete(project_id, resource_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    resource = Resource.query.filter_by(id=resource_id).first_or_404()
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (isUserActive(current_user) and current_user.is_admin)
    if not allow_edit:
        flash('You do not have access to edit this resource.', 'warning')
        return project_action(project_id, None)
    resource.is_hidden = True
    db.session.add(resource)
    db.session.commit()
    flash('Resource removed.', 'success')
    # project_action(project_id, 'resource-removed', False)
    return redirect(url_for('project.project_view', project_id=project.id))

@blueprint.route('/<int:project_id>/post', methods=['GET', 'POST'])
@login_required
def project_post(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    event = project.event
    starred = IsProjectStarred(project, current_user)
    allow_post = starred or (not current_user.is_anonymous and current_user.is_admin)
    # allow_post = allow_post and not event.lock_resources
    if not allow_post:
        flash('You do not have access to post to this project.', 'warning')
        return project_action(project_id, None)
    form = ProjectPost(obj=project, next=request.args.get('next'))

    # Evaluate project progress
    stage, all_valid = validateProjectData(project)

    # Process form
    if form.validate_on_submit():

        # Check and update progress
        if form.has_progress.data:
            found_next = False
            if all_valid:
                for a in projectProgressList(True, False):
                    print(a[0])
                    if found_next:
                        project.progress = a[0]
                        flash('Your project has been promoted!', 'info')
                        break
                    if a[0] == project.progress or not project.progress or project.progress < 0:
                        found_next = True
                        print("Founddd")
            if not all_valid or not found_next:
                flash('Your project did not yet meet stage requirements.', 'info')

        # Update project data
        del form.id
        del form.has_progress
        form.populate_obj(project)
        project.update()
        db.session.add(project)
        db.session.commit()
        cache.clear()
        flash('Thanks for your commit', 'success')
        project_action(project_id, 'update', action='post', text=form.note.data)
        # return redirect(url_for('project.project_view', project_id=project.id))
        return project_view(project.id)

    return render_template(
        'public/projectpost.html',
        current_event=event, project=project, form=form, stage=stage, all_valid=all_valid,
    )

def getSuggestionsForStage(progress):
    """ Get all projects which are published in a resource-type event """
    project_list = []
    resource_events = [ e.id for e in Event.query.filter_by(lock_resources=True).all() ]
    for eid in resource_events:
        projects = Project.query.filter_by(event_id=eid, is_hidden=False, progress=progress)
        project_list.extend([ p.data for p in projects.all() ])
    return project_list

def project_action(project_id, of_type=None, as_view=True, then_redirect=False, action=None, text=None):
    project = Project.query.filter_by(id=project_id).first_or_404()
    event = project.event
    if of_type is not None:
        ProjectActivity(project, of_type, current_user, action, text)
    if not as_view:
        return True
    if then_redirect:
        return redirect(url_for('project.project_view', project_id=project.id))
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (not current_user.is_anonymous and current_user.is_admin)
    allow_post = allow_edit # and not event.lock_resources
    allow_edit = allow_edit and not event.lock_editing
    project_team = project.team()
    latest_activity = project.latest_activity()
    project_dribs = project.all_dribs()
    suggestions = None
    if not event.lock_resources:
        suggestions = getSuggestionsForStage(project.progress)
    return render_template('public/project.html', current_event=event, project=project,
        project_starred=starred, project_team=project_team, project_dribs=project_dribs,
        allow_edit=allow_edit, allow_post=allow_post,
        suggestions=suggestions, latest_activity=latest_activity)

@blueprint.route('/<int:project_id>/star', methods=['GET', 'POST'])
@login_required
def project_star(project_id):
    if not isUserActive(current_user):
        return "User not allowed. Please contact event organizers."
    flash('Welcome to the team!', 'success')
    return project_action(project_id, 'star', then_redirect=True)

@blueprint.route('/<int:project_id>/unstar', methods=['GET', 'POST'])
@login_required
def project_unstar(project_id):
    flash('You have left the project', 'success')
    return project_action(project_id, 'unstar', then_redirect=True)

@blueprint.route('/event/<int:event_id>/project/new', methods=['GET', 'POST'])
@login_required
def project_new(event_id):
    if not isUserActive(current_user):
        flash("Your account needs to be activated: please contact an organizer.", 'warning')
        return redirect(url_for('public.event', event_id=event_id))
    form = None
    event = Event.query.filter_by(id=event_id).first_or_404()
    if event.lock_starting:
        flash('Starting a new project is disabled for this event.', 'error')
        return redirect(url_for('public.event', event_id=event.id))
    if isUserActive(current_user):
        project = Project()
        project.user_id = current_user.id
        form = ProjectNew(obj=project, next=request.args.get('next'))
        form.category_id.choices = [(c.id, c.name) for c in project.categories_all(event)]
        if len(form.category_id.choices) > 0:
            form.category_id.choices.insert(0, (-1, ''))
        else:
            del form.category_id
        if form.validate_on_submit():
            del form.id
            form.populate_obj(project)
            project.event = event
            if event.has_started:
                project.progress = 0
            else:
                project.progress = -1 # Start as challenge
            project.update()
            db.session.add(project)
            db.session.commit()
            flash('New challenge added.', 'success')
            project_action(project.id, 'create', False)
            cache.clear()
            if event.has_started:
                project_action(project.id, 'star', False) # Join team
            if len(project.autotext_url)>1:
                return project_autoupdate(project.id)
            else:
                return redirect(url_for('project.project_view', project_id=project.id))
    return render_template('public/projectnew.html', active="projectnew", current_event=event, form=form)

@blueprint.route('/<int:project_id>/autoupdate')
@login_required
def project_autoupdate(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (not current_user.is_anonymous and current_user.is_admin)
    if not allow_edit or project.is_hidden or not project.is_autoupdate:
        flash('You may not sync this project.', 'warning')
        return project_action(project_id)
    data = GetProjectData(project.autotext_url)
    if not 'name' in data:
        flash("Could not sync: check that the Remote Link contains a README.", 'warning')
        return project_action(project_id)
    SyncProjectData(project, data)
    project_action(project.id, 'update', action='sync', text=str(len(project.autotext)) + ' bytes')
    flash("Project data synced from %s" % data['type'], 'success')
    return redirect(url_for('project.project_view', project_id=project.id))


import os

# os.environ["TF_DISABLE_MLC"] = "1"
os.environ["TF_MLC_LOGGING"] = "1"

from tensorflow.python.compiler.mlcompute import mlcompute

# mlcompute.set_mlc_device(device_name='gpu')
print("is_apple_mlc_enabled %s" % mlcompute.is_apple_mlc_enabled())
print("is_tf_compiled_with_apple_mlc %s" % mlcompute.is_tf_compiled_with_apple_mlc())

import tensorflow as tf
from tensorflow.keras import datasets, layers, models

tf.compat.v1.disable_eager_execution()

print(f"eagerly? {tf.executing_eagerly()}")
print(tf.config.list_logical_devices())

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_images, test_images = train_images / 255.0, test_images / 255.0
class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',
               'dog', 'frog', 'horse', 'ship', 'truck']
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10))
model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])
history = model.fit(train_images, train_labels, epochs=10,
                    validation_data=(test_images, test_labels))


from bs4 import BeautifulSoup

class TeachersListParser():
    def __init__(self, html, employment_type):
        page = BeautifulSoup(html, 'html.parser')
        self.table = page.select_one('#p785-content > table')
        self.employment_type = employment_type
        if not self.table:
            raise Exception('Данная html-страница не является валидным списком преподавателей')

    def get_next_teacher(self):
         #первая строка - названия полей, не нужна. Последняя пуста
        for row in self.table.find_all('tr')[1:-1]:
            res = []
            cell_index = 0
            for cell in row.find_all('td'):

                if cell_index in [3, 6, 7]:
                    res.append([x.strip() for x in
                                cell.get_text('\n').split(';')])
                else:
                    res.append(cell.get_text('\n').strip())
                cell_index += 1

            yield self._to_json(res)

    def _to_json(self, arr):
        return {'id': int(arr[0]),
                'name': arr[1],
                'post': arr[2],
                'disciplines': arr[3],
                'degree': arr[4],
                'rank': arr[5],
                'areas': arr[6],
                'training': arr[7],
                'experience': int(arr[8]) if arr[8] else 0,
                'teaching_experience': int(arr[9]) if arr[9] else 0,
                'employment_type': self.employment_type
        }


# -*- coding: utf-8 -*-
"""
Created on Mon Jul  5 08:25:39 2021

@author: fronortin_l
"""
import math

def primes(n):
    """ Returns  a list of primes < n """
    sieve = [True] * n
    for i in range(3,int(n**0.5)+1,2):
        if sieve[i]:
            sieve[i*i::2*i]=[False]*((n-i*i-1)//(2*i)+1)
    return [2] + [i for i in range(3,n,2) if sieve[i]]

def Euler_18(triangle):
    #Transformation de la chaîne de caractère en une liste contenant
    #des sous listes (représentant les lignes) qui contiennent
    #les nombres de chaque ligne
    triangle = [[int(j) for j in i.split()] for i in triangle.split("\n")]
    #Inversion de la liste triangle
    triangle.reverse()
    #Teste de toutes les valeurs à partir de la seconde ligne
    #pour trouver quel est la somme maximale possible en partant de ces valeurs
    for i in range(1, len(triangle)):
        for j, k in enumerate(triangle[i]):
            triangle[i][j] = k + max([triangle[i-1][j], triangle[i-1][j+1]])
    return triangle[-1][0]

def Euler_19():
  dico = {'January':31,'February':28,'March':31,'April':30,'May':31,'June':30,'July':31,'August':31,'September':30,'October':31,'November':30,'December':31}
  year = 1901
  day = 2
  sunday = 0

  while year !=2001:
    for i in dico:
      day += dico[i]

      if (i == 'February') and (year%4==0 or (year%100==0 and year%400==0)):
        day +=1

      day = day%7

      if day==0:
        sunday+=1
        day =7

    year+=1

  return sunday

def sum_factors(n):
     result = []
     for i in range(1, int(n**0.5) + 1):
         if n % i == 0:
             result.extend([i, n//i])
     return sum(set(result)-set([n]))

def Euler_21(number):
    result = []
    for x in range(1,number+1):
        y = sum_factors(x)
        if sum_factors(y) == x and x != y:
            result.append(tuple(sorted((x,y))))

    return sum([sum(i) for i in set(result)])

def Euler_22():
    with open('D:\p022_names.txt') as f:
        lines = f.readlines()

    lines= lines[0].split(',')
    lines = [i.replace('"','') for i in lines]
    lines.sort()

    for k,i in enumerate(lines):
        s=0
        for j in i:
            s+= ord(j)-64
            lines[k] = s

    return sum([lines[i]*(i+1) for i in range(len(lines))])


#!/usr/bin/env python3
# Copyright 2016 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import os.path
import sys
import argparse
import collections
import functools
import re
import copy
try:
  import json
except ImportError:
  import simplejson as json

import pdl

try:
  unicode
except NameError:
  # Define unicode for Py3
  def unicode(s, *_):
    return s

# Path handling for libraries and templates
# Paths have to be normalized because Jinja uses the exact template path to
# determine the hash used in the cache filename, and we need a pre-caching step
# to be concurrency-safe. Use absolute path because __file__ is absolute if
# module is imported, and relative if executed directly.
# If paths differ between pre-caching and individual file compilation, the cache
# is regenerated, which causes a race condition and breaks concurrent build,
# since some compile processes will try to read the partially written cache.
module_path, module_filename = os.path.split(os.path.realpath(__file__))

def read_config():
  # pylint: disable=W0703
  def json_to_object(data, output_base, config_base):
    def json_object_hook(object_dict):
      items = [(k, os.path.join(config_base, v) if k == "path" else v)
               for (k, v) in object_dict.items()]
      items = [(k, os.path.join(output_base, v) if k == "output" else v)
               for (k, v) in items]
      keys, values = list(zip(*items))
      # 'async' is a keyword since Python 3.7.
      # Avoid namedtuple(rename=True) for compatibility with Python 2.X.
      keys = tuple('async_' if k == 'async' else k for k in keys)
      return collections.namedtuple('X', keys)(*values)
    return json.loads(data, object_hook=json_object_hook)

  def init_defaults(config_tuple, path, defaults):
    keys = list(config_tuple._fields)  # pylint: disable=E1101
    values = [getattr(config_tuple, k) for k in keys]
    for i in range(len(keys)):
      if hasattr(values[i], "_fields"):
        values[i] = init_defaults(values[i], path + "." + keys[i], defaults)
    for optional in defaults:
      if optional.find(path + ".") != 0:
        continue
      optional_key = optional[len(path) + 1:]
      if optional_key.find(".") == -1 and optional_key not in keys:
        keys.append(optional_key)
        values.append(defaults[optional])
    return collections.namedtuple('X', keys)(*values)

  try:
    cmdline_parser = argparse.ArgumentParser()
    cmdline_parser.add_argument("--output_base", type=unicode, required=True)
    cmdline_parser.add_argument("--jinja_dir", type=unicode, required=True)
    cmdline_parser.add_argument("--config", type=unicode, required=True)
    cmdline_parser.add_argument("--config_value", default=[], action="append")
    cmdline_parser.add_argument(
      "--inspector_protocol_dir", type=unicode, required=True,
      help=("directory with code_generator.py and C++ encoding / binding "
          "libraries, relative to the root of the source tree."))
    arg_options = cmdline_parser.parse_args()
    jinja_dir = arg_options.jinja_dir
    output_base = arg_options.output_base
    config_file = arg_options.config
    config_base = os.path.dirname(config_file)
    config_values = arg_options.config_value
    inspector_protocol_dir = arg_options.inspector_protocol_dir.lstrip('/')
  except Exception:
    # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html
    exc = sys.exc_info()[1]
    sys.stderr.write("Failed to parse command-line arguments: %s\n\n" % exc)
    exit(1)

  try:
    config_json_file = open(config_file, "r")
    config_json_string = config_json_file.read()
    config_partial = json_to_object(config_json_string, output_base,
                                    config_base)
    config_json_file.close()
    defaults = {
      ".use_snake_file_names": False,
      ".use_title_case_methods": False,
      ".use_embedder_types": False,
      ".imported": False,
      ".imported.export_macro": "",
      ".imported.export_header": False,
      ".imported.header": False,
      ".imported.package": False,
      ".imported.options": False,
      ".protocol.export_macro": "",
      ".protocol.export_header": False,
      ".protocol.options": False,
      ".protocol.file_name_prefix": "",
      ".exported": False,
      ".exported.export_macro": "",
      ".exported.export_header": False,
      ".lib": False,
      ".lib.export_macro": "",
      ".lib.export_header": False,
      ".crdtp": False,
      ".crdtp.dir": os.path.join(inspector_protocol_dir, "crdtp"),
      ".crdtp.namespace": "crdtp",
    }
    for key_value in config_values:
      parts = key_value.split("=")
      if len(parts) == 2:
        defaults["." + parts[0]] = parts[1]
    return (jinja_dir, config_file, init_defaults(config_partial, "", defaults))
  except Exception:
    # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html
    exc = sys.exc_info()[1]
    sys.stderr.write("Failed to parse config file: %s\n\n" % exc)
    exit(1)

# ---- Begin of utilities exposed to generator ----

def to_title_case(name):
  return name[:1].upper() + name[1:]

def dash_to_camelcase(word):
  prefix = ""
  if word[0] == "-":
    prefix = "Negative"
    word = word[1:]
  return prefix + "".join(to_title_case(x) or "-" for x in word.split("-"))

def to_snake_case(name):
  name = re.sub(r"([A-Z]{2,})([A-Z][a-z])", r"\1_\2", name)
  return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name, sys.maxsize).lower()

def to_method_case(config, name):
  if config.use_title_case_methods:
    return to_title_case(name)
  return name

def join_arrays(dict, keys):
  result = []
  for key in keys:
    if key in dict:
      result += dict[key]
  return result

def format_include(config, header, file_name=None):
  if file_name is not None:
    header = header + "/" + file_name + ".h"
  header = "\"" + header + "\"" if header[0] not in "<\"" else header
  if config.use_snake_file_names:
    header = to_snake_case(header)
  return header

def format_domain_include(config, header, file_name):
  return format_include(config, header,
                        config.protocol.file_name_prefix + file_name)

def to_file_name(config, file_name):
  if config.use_snake_file_names:
    return to_snake_case(file_name).replace(".cpp", ".cc")
  return file_name

# ---- End of utilities exposed to generator ----

def initialize_jinja_env(jinja_dir, cache_dir, config):
  # pylint: disable=F0401
  sys.path.insert(1, os.path.abspath(jinja_dir))
  import jinja2

  jinja_env = jinja2.Environment(
    loader=jinja2.FileSystemLoader(module_path),
    # Bytecode cache is not concurrency-safe unless pre-cached:
    # if pre-cached this is read-only, but writing creates a race condition.
    bytecode_cache=jinja2.FileSystemBytecodeCache(cache_dir),
    keep_trailing_newline=True,  # newline-terminate generated files
    lstrip_blocks=True,  # so can indent control flow tags
    trim_blocks=True)
  jinja_env.filters.update({
      "to_title_case": to_title_case,
      "dash_to_camelcase": dash_to_camelcase,
      "to_method_case": functools.partial(to_method_case, config)})
  jinja_env.add_extension("jinja2.ext.loopcontrols")
  return jinja_env

def create_imported_type_definition(domain_name, type, imported_namespace):
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<%s::%s::API::%s>" % (
        imported_namespace, domain_name, type["id"]),
    "pass_type": "std::unique_ptr<%s::%s::API::%s>" % (
        imported_namespace, domain_name, type["id"]),
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<%s::%s::API::%s>" % (
        imported_namespace, domain_name, type["id"]),
    "raw_type": "%s::%s::API::%s" % (
        imported_namespace, domain_name, type["id"]),
    "raw_pass_type": "%s::%s::API::%s*" % (
        imported_namespace, domain_name, type["id"]),
    "raw_return_type": "%s::%s::API::%s*" % (
        imported_namespace, domain_name, type["id"]),
  }

def create_user_type_definition(domain_name, type):
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<protocol::%s::%s>" % (
        domain_name, type["id"]),
    "pass_type": "std::unique_ptr<protocol::%s::%s>" % (
        domain_name, type["id"]),
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<protocol::%s::%s>" % (domain_name, type["id"]),
    "raw_type": "protocol::%s::%s" % (domain_name, type["id"]),
    "raw_pass_type": "protocol::%s::%s*" % (domain_name, type["id"]),
    "raw_return_type": "protocol::%s::%s*" % (domain_name, type["id"]),
  }

def create_object_type_definition():
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<protocol::DictionaryValue>",
    "pass_type": "std::unique_ptr<protocol::DictionaryValue>",
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<protocol::DictionaryValue>",
    "raw_type": "protocol::DictionaryValue",
    "raw_pass_type": "protocol::DictionaryValue*",
    "raw_return_type": "protocol::DictionaryValue*",
  }

def create_any_type_definition():
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<protocol::Value>",
    "pass_type": "std::unique_ptr<protocol::Value>",
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<protocol::Value>",
    "raw_type": "protocol::Value",
    "raw_pass_type": "protocol::Value*",
    "raw_return_type": "protocol::Value*",
  }

def create_string_type_definition():
  # pylint: disable=W0622
  return {
    "return_type": "String",
    "pass_type": "const String&",
    "to_pass_type": "%s",
    "to_raw_type": "%s",
    "to_rvalue": "%s",
    "type": "String",
    "raw_type": "String",
    "raw_pass_type": "const String&",
    "raw_return_type": "String",
    "is_primitive": True
  }

def create_binary_type_definition():
  # pylint: disable=W0622
  return {
    "return_type": "Binary",
    "pass_type": "const Binary&",
    "to_pass_type": "%s",
    "to_raw_type": "%s",
    "to_rvalue": "%s",
    "type": "Binary",
    "raw_type": "Binary",
    "raw_pass_type": "const Binary&",
    "raw_return_type": "Binary",
    "is_primitive": True
  }

def create_primitive_type_definition(type):
  # pylint: disable=W0622
  typedefs = {
    "number": "double",
    "integer": "int",
    "boolean": "bool"
  }
  defaults = {
    "number": "0",
    "integer": "0",
    "boolean": "false"
  }
  jsontypes = {
    "number": "TypeDouble",
    "integer": "TypeInteger",
    "boolean": "TypeBoolean",
  }
  return {
    "return_type": typedefs[type],
    "pass_type": typedefs[type],
    "to_pass_type": "%s",
    "to_raw_type": "%s",
    "to_rvalue": "%s",
    "type": typedefs[type],
    "raw_type": typedefs[type],
    "raw_pass_type": typedefs[type],
    "raw_return_type": typedefs[type],
    "default_value": defaults[type],
    "is_primitive": True
  }

def wrap_array_definition(type):
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<protocol::Array<%s>>" % type["raw_type"],
    "pass_type": "std::unique_ptr<protocol::Array<%s>>" % type["raw_type"],
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<protocol::Array<%s>>" % type["raw_type"],
    "raw_type": "protocol::Array<%s>" % type["raw_type"],
    "raw_pass_type": "protocol::Array<%s>*" % type["raw_type"],
    "raw_return_type": "protocol::Array<%s>*" % type["raw_type"],
    "out_type": "protocol::Array<%s>&" % type["raw_type"],
  }

class Protocol(object):

  def __init__(self, config):
    self.config = config
    self.json_api = {"domains": []}
    self.imported_domains = []
    self.exported_domains = []
    self.generate_domains = self.read_protocol_file(config.protocol.path)

    if config.protocol.options:
      self.generate_domains = [rule.domain for rule in config.protocol.options]
      self.exported_domains = [rule.domain for rule in config.protocol.options
                               if hasattr(rule, "exported")]

    if config.imported:
      self.imported_domains = self.read_protocol_file(config.imported.path)
      if config.imported.options:
        self.imported_domains = [rule.domain
                                 for rule in config.imported.options]

    self.patch_full_qualified_refs()
    self.create_type_definitions()
    self.generate_used_types()

  def read_protocol_file(self, file_name):
    input_file = open(file_name, "r")
    parsed_json = pdl.loads(input_file.read(), file_name)
    input_file.close()
    version = '%s.%s' % (parsed_json["version"]["major"],
                         parsed_json["version"]["minor"])
    domains = []
    for domain in parsed_json["domains"]:
      domains.append(domain["domain"])
      domain["version"] = version
    self.json_api["domains"] += parsed_json["domains"]
    return domains

  def patch_full_qualified_refs(self):
    def patch_full_qualified_refs_in_domain(json, domain_name):
      if isinstance(json, list):
        for item in json:
          patch_full_qualified_refs_in_domain(item, domain_name)
      if not isinstance(json, dict):
        return
      for key in json:
        if key == "type" and json[key] == "string":
          json[key] = domain_name + ".string"
        if key != "$ref":
          patch_full_qualified_refs_in_domain(json[key], domain_name)
          continue
        if json["$ref"].find(".") == -1:
          json["$ref"] = domain_name + "." + json["$ref"]
      return

    for domain in self.json_api["domains"]:
      patch_full_qualified_refs_in_domain(domain, domain["domain"])

  def all_references(self, json):
    refs = set()
    if isinstance(json, list):
      for item in json:
        refs |= self.all_references(item)
    if not isinstance(json, dict):
      return refs
    for key in json:
      if key != "$ref":
        refs |= self.all_references(json[key])
      else:
        refs.add(json["$ref"])
    return refs

  def check_if_dependency_declared(self, domain, refs):
    dependencies = domain.get('dependencies', set())
    for ref in refs:
      type_definition = self.type_definitions[ref]
      if type_definition.get('is_primitive', False):
        continue
      domain_match = re.match(r'^(.*)[.]', ref)
      if domain_match:
        referenced_domain_name = domain_match.group(1)
        if referenced_domain_name != domain['domain'] and not referenced_domain_name in dependencies:
            sys.stderr.write(("Domains [%s] uses type [%s] from domain [%s], but did not declare the dependency\n\n"
                    ) % (domain["domain"], ref, referenced_domain_name))
            exit(1)

  def generate_used_types(self):
    all_refs = set()
    for domain in self.json_api["domains"]:
      domain_name = domain["domain"]
      if "commands" in domain:
        for command in domain["commands"]:
          if self.generate_command(domain_name, command["name"]):
            all_refs_command = self.all_references(command)
            # If the command has a redirect, it is as if it didn't exist on this domain.
            if not command.get('redirect', False):
              self.check_if_dependency_declared(domain, all_refs_command)
            all_refs |= all_refs_command

      if "events" in domain:
        for event in domain["events"]:
          if self.generate_event(domain_name, event["name"]):
            all_refs_event = self.all_references(event)
            self.check_if_dependency_declared(domain, all_refs_event)
            all_refs |= all_refs_event

    dependencies = self.generate_type_dependencies()
    queue = set(all_refs)
    while len(queue):
      ref = queue.pop()
      if ref in dependencies:
        queue |= dependencies[ref] - all_refs
        all_refs |= dependencies[ref]
    self.used_types = all_refs

  def generate_type_dependencies(self):
    dependencies = dict()
    domains_with_types = (x for x in self.json_api["domains"] if "types" in x)
    for domain in domains_with_types:
      domain_name = domain["domain"]
      for type in domain["types"]:
        related_types = self.all_references(type)
        if len(related_types):
          dependencies[domain_name + "." + type["id"]] = related_types
    return dependencies

  def create_type_definitions(self):
    imported_namespace = ""
    if self.config.imported:
      imported_namespace = "::".join(self.config.imported.namespace)
    self.type_definitions = {}
    self.type_definitions["number"] = create_primitive_type_definition("number")
    self.type_definitions["integer"] = create_primitive_type_definition("integer")
    self.type_definitions["boolean"] = create_primitive_type_definition("boolean")
    self.type_definitions["object"] = create_object_type_definition()
    self.type_definitions["any"] = create_any_type_definition()
    self.type_definitions["binary"] = create_binary_type_definition()
    for domain in self.json_api["domains"]:
      self.type_definitions[domain["domain"] + ".string"] = (
          create_string_type_definition())
      self.type_definitions[domain["domain"] + ".binary"] = (
          create_binary_type_definition())
      if not ("types" in domain):
        continue
      for type in domain["types"]:
        type_name = domain["domain"] + "." + type["id"]
        if type["type"] == "object" and domain["domain"] in self.imported_domains:
          self.type_definitions[type_name] = create_imported_type_definition(
              domain["domain"], type, imported_namespace)
        elif type["type"] == "object":
          self.type_definitions[type_name] = create_user_type_definition(
              domain["domain"], type)
        elif type["type"] == "array":
          self.type_definitions[type_name] = self.resolve_type(type)
        elif type["type"] == domain["domain"] + ".string":
          self.type_definitions[type_name] = create_string_type_definition()
        elif type["type"] == domain["domain"] + ".binary":
          self.type_definitions[type_name] = create_binary_type_definition()
        else:
          self.type_definitions[type_name] = create_primitive_type_definition(
              type["type"])

  def check_options(self, options, domain, name, include_attr, exclude_attr,
                    default):
    for rule in options:
      if rule.domain != domain:
        continue
      if include_attr and hasattr(rule, include_attr):
        return name in getattr(rule, include_attr)
      if exclude_attr and hasattr(rule, exclude_attr):
        return name not in getattr(rule, exclude_attr)
      return default
    return False

  # ---- Begin of methods exposed to generator

  def type_definition(self, name):
    return self.type_definitions[name]

  def resolve_type(self, prop):
    if "$ref" in prop:
      return self.type_definitions[prop["$ref"]]
    if prop["type"] == "array":
      return wrap_array_definition(self.resolve_type(prop["items"]))
    return self.type_definitions[prop["type"]]

  def generate_command(self, domain, command):
    if not self.config.protocol.options:
      return domain in self.generate_domains
    return self.check_options(self.config.protocol.options, domain, command,
                              "include", "exclude", True)

  def generate_event(self, domain, event):
    if not self.config.protocol.options:
      return domain in self.generate_domains
    return self.check_options(self.config.protocol.options, domain, event,
                              "include_events", "exclude_events", True)

  def generate_type(self, domain, typename):
    return domain + "." + typename in self.used_types

  def is_async_command(self, domain, command):
    if not self.config.protocol.options:
      return False
    return self.check_options(self.config.protocol.options, domain, command,
                              "async_", None, False)

  def is_exported(self, domain, name):
    if not self.config.protocol.options:
      return False
    return self.check_options(self.config.protocol.options, domain, name,
                              "exported", None, False)

  def is_imported(self, domain, name):
    if not self.config.imported:
      return False
    if not self.config.imported.options:
      return domain in self.imported_domains
    return self.check_options(self.config.imported.options, domain, name,
                              "imported", None, False)

  def is_exported_domain(self, domain):
    return domain in self.exported_domains

  def generate_disable(self, domain):
    if "commands" not in domain:
      return True
    for command in domain["commands"]:
      if command["name"] == "disable" and self.generate_command(
          domain["domain"], "disable"):
        return False
    return True

  def is_imported_dependency(self, domain):
    return domain in self.generate_domains or domain in self.imported_domains

def main():
  jinja_dir, config_file, config = read_config()

  protocol = Protocol(config)

  if not config.exported and len(protocol.exported_domains):
    sys.stderr.write(("Domains [%s] are exported, but config is missing export "
                      "entry\n\n") % ", ".join(protocol.exported_domains))
    exit(1)

  if not os.path.exists(config.protocol.output):
    os.mkdir(config.protocol.output)
  if len(protocol.exported_domains) and not os.path.exists(
      config.exported.output):
    os.mkdir(config.exported.output)
  jinja_env = initialize_jinja_env(jinja_dir, config.protocol.output, config)

  inputs = []
  inputs.append(__file__)
  inputs.append(config_file)
  inputs.append(config.protocol.path)
  if config.imported:
    inputs.append(config.imported.path)
  templates_dir = os.path.join(module_path, "templates")
  inputs.append(os.path.join(templates_dir, "TypeBuilder_h.template"))
  inputs.append(os.path.join(templates_dir, "TypeBuilder_cpp.template"))
  inputs.append(os.path.join(templates_dir, "Exported_h.template"))
  inputs.append(os.path.join(templates_dir, "Imported_h.template"))

  h_template = jinja_env.get_template("templates/TypeBuilder_h.template")
  cpp_template = jinja_env.get_template("templates/TypeBuilder_cpp.template")
  exported_template = jinja_env.get_template("templates/Exported_h.template")
  imported_template = jinja_env.get_template("templates/Imported_h.template")

  outputs = dict()

  for domain in protocol.json_api["domains"]:
    class_name = domain["domain"]
    file_name = config.protocol.file_name_prefix + class_name
    template_context = {
      "protocol": protocol,
      "config": config,
      "domain": domain,
      "join_arrays": join_arrays,
      "format_include": functools.partial(format_include, config),
      "format_domain_include": functools.partial(format_domain_include, config),
    }

    if domain["domain"] in protocol.generate_domains:
      outputs[os.path.join(config.protocol.output, to_file_name(
          config, file_name + ".h"))] = h_template.render(template_context)
      outputs[os.path.join(config.protocol.output, to_file_name(
          config, file_name + ".cpp"))] = cpp_template.render(template_context)
      if domain["domain"] in protocol.exported_domains:
        outputs[os.path.join(config.exported.output, to_file_name(
            config, file_name + ".h"))] = exported_template.render(
                template_context)
    if domain["domain"] in protocol.imported_domains:
      outputs[os.path.join(config.protocol.output, to_file_name(
          config, file_name + ".h"))] = imported_template.render(
              template_context)

  if config.lib:
    template_context = {
      "config": config,
      "format_include": functools.partial(format_include, config),
    }

    lib_templates_dir = os.path.join(module_path, "lib")
    # Note these should be sorted in the right order.

    # TODO(dgozman): sort them programmatically based on commented includes.

    forward_h_templates = [
      "Forward_h.template",
    ]

    protocol_h_templates = []
    protocol_cpp_templates = []

    if not config.use_embedder_types:
      protocol_h_templates += [
        "Values_h.template",
        "Object_h.template",
        "ValueConversions_h.template",
      ]
      protocol_cpp_templates += [
        "Protocol_cpp.template",
        "Values_cpp.template",
        "Object_cpp.template",
        "ValueConversions_cpp.template",
      ]
    else:
      protocol_h_templates += [
        "Forward_h.template",
      ]

    def generate_lib_file(file_name, template_files):
      parts = []
      for template_file in template_files:
        inputs.append(os.path.join(lib_templates_dir, template_file))
        template = jinja_env.get_template("lib/" + template_file)
        parts.append(template.render(template_context))
      outputs[file_name] = "\n\n".join(parts)

    generate_lib_file(os.path.join(config.lib.output, to_file_name(
        config, "Forward.h")), forward_h_templates)
    generate_lib_file(os.path.join(config.lib.output, to_file_name(
        config, "Protocol.h")), protocol_h_templates)

    if not config.use_embedder_types:
      generate_lib_file(os.path.join(config.lib.output, to_file_name(
          config, "Protocol.cpp")), protocol_cpp_templates)

  # Make gyp / make generatos happy, otherwise make rebuilds world.
  inputs_ts = max(map(os.path.getmtime, inputs))
  up_to_date = True
  for output_file in outputs.keys():
    if (not os.path.exists(output_file)
        or os.path.getmtime(output_file) < inputs_ts):
      up_to_date = False
      break
  if up_to_date:
    sys.exit()

  for file_name, content in outputs.items():
    # Remove output file first to account for potential case changes.
    try:
      os.remove(file_name)
    except OSError:
      pass
    out_file = open(file_name, "w")
    out_file.write(content)
    out_file.close()

if __name__ == "__main__":
  main()


dataset = [
    [['Зеленый', 3], 'Яблоко'],
    [['Желтый', 3], 'Яблоко'],
    [['Красный', 1], 'Виноград'],
    [['Красный', 1], 'Виноград'],
    [['Желтый', 3], 'Лимон'],
]

header = [['цвет', 'размер'], 'метка']

def unique_values(rows, col):
    return set([row[0][col] for row in rows])

def class_counts(dataset):
    counts = {}
    for row in dataset:
        label = row[1]
        if label not in counts:
            counts[label] = 0
        counts[label] += 1
    return counts

def is_numeric(value):
    return isinstance(value, (int, float))

class Question:
    def __init__(self, column, value):
        self.column = column
        self.value = value

    def match(self, example):
        val = example[self.column]
        if is_numeric(val):
            return val >= self.value
        else:
            return val == self.value

    def __repr__(self):
        condition = "=="
        if is_numeric(self.value):
            condition = ">="
        return "%s %s %s?" % (header[0][self.column], condition, str(self.value))

def partition(rows, question):
    true_rows, false_rows = [], []
    for row in rows:
        if question.match(row[0]):
            true_rows.append(row)
        else:
            false_rows.append(row)
    return true_rows, false_rows

def gini(rows):
    counts = class_counts(rows)
    impurity = 1
    for label in counts:
        prob_label = counts[label] / float(len(rows))
        impurity -= prob_label ** 2
    return impurity

def info_gain(left, right, current):
    p = float(len(left)) / (len(left) + len(right))
    return current - p * gini(left) - (1 - p) * gini(right)

def find_best_split(rows):
    best_gain = 0
    best_question = None
    current = gini(rows)
    n_features = len(rows[0][0])

    for col in range(n_features):
        values = set([row[0][col] for row in rows])

        for val in values:
            question = Question(col, val)
            true_rows, false_rows = partition(rows, question)
            if len(true_rows) == 0 or len(false_rows) == 0:
                continue
            gain = info_gain(true_rows, false_rows, current)

            if gain >= best_gain:
                best_gain, best_question = gain, question

    return best_gain, best_question

class Leaf:
    def __init__(self, rows):
        self.predictions = class_counts(rows)

class Decision_node:
    def __init__(self, question, true_branch, false_branch):
        self.question = question
        self.true_branch = true_branch
        self.false_branch = false_branch

def build_tree(rows):
    gain, question = find_best_split(rows)

    if gain == 0:
        return Leaf(rows)
    true_rows, false_rows = partition(rows, question)

    true_branch = build_tree(true_rows)
    false_branch = build_tree(false_rows)

    return Decision_node(question, true_branch, false_branch)

def print_tree(node, spacing=""):
    if isinstance(node, Leaf):
        print(spacing + "Предположение", node.predictions)
        return
    print(spacing + str(node.question))
    print(spacing + "--> Да")
    print_tree(node.true_branch, spacing + '    ')
    print(spacing + "--> Нет")
    print_tree(node.false_branch, spacing + '    ')

def classify(row, node):
    if isinstance(node, Leaf):
        return node.predictions

    if node.question.match(row[0]):
        return classify(row, node.true_branch)
    else:
        return classify(row, node.false_branch)

def print_leaf(counts):
    total = sum(counts.values()) * 1.0
    probs = {}
    for label in counts.keys():
        probs[label] = str(int(counts[label] / total * 100)) + "%"
    return  probs

# print(find_best_split(dataset))
tree = build_tree(dataset)
# print_tree(tree)
# print(classify(dataset[0], tree))
# print(print_leaf(classify(dataset[1], tree)))

test = [
    [['Зеленый', 3]],
    [['Желтый', 4]],
    [['Зеленый', 2]],
    [['Зеленый', 1]],
    [['Желтый', 3]]
]

for row in test:
    print("Класс: %s" % (print_leaf(classify(row,tree))))