import gym
from gym import error, spaces, utils
from gym.utils import seeding
from gym.spaces import Box, Tuple
from .env_generator import EnvironmentCollection

from math import pi, cos, sin
import numpy as np

#from gym.envs.classic_control.rendering  import make_circle, Transform
from gym_extensions.continuous.gym_navigation_2d import gym_rendering
import os
import logging

class LimitedRangeBasedPOMDPNavigation2DEnv(gym.Env):
    logger = logging.getLogger(__name__)
    metadata = {'render.modes': ['human']}

    def __init__(self,
                 worlds_pickle_filename=os.path.join(os.path.dirname(__file__), "assets", "worlds_640x480_v0.pkl"),
                 world_idx=0,
                 initial_position = np.array([-20.0, -20.0]),
                 destination = np.array([520.0, 400.0]),
                 max_observation_range = 100.0,
                 destination_tolerance_range=20.0,
                 add_self_position_to_observation=False,
                 add_goal_position_to_observation=False):

        worlds = EnvironmentCollection()
        worlds.read(worlds_pickle_filename)

        self.world = worlds.map_collection[world_idx]
        self.set_destination(destination)

        assert not (self.destination is None)
        self.init_position = initial_position
        self.state = self.init_position.copy()

        self.max_observation_range = max_observation_range
        self.destination_tolerance_range = destination_tolerance_range
        self.viewer = None
        self.num_beams = 16
        self.max_speed = 5
        self.add_self_position_to_observation = add_self_position_to_observation
        self.add_goal_position_to_observation = add_goal_position_to_observation

        low = np.array([0.0, 0.0])
        high = np.array([self.max_speed, 2*pi])
        self.action_space = Box(low, high)#Tuple( (Box(0.0, self.max_speed, (1,)), Box(0.0, 2*pi, (1,))) )
        low = [-1.0] * self.num_beams
        high = [self.max_observation_range] * self.num_beams
        if add_self_position_to_observation:
            low.extend([-10000., -10000.]) # x and y coords
            high.extend([10000., 10000.])
        if add_goal_position_to_observation:
            low.extend([-10000., -10000.]) # x and y coords
            high.extend([10000., 10000.])

        self.observation_space = Box(np.array(low), np.array(high))
        self.observation = []

    def set_destination(self, destination):
        self.destination = destination

    def _get_observation(self, state):
        delta_angle = 2*pi/self.num_beams
        ranges = [self.world.raytrace(self.state,
                                      i * delta_angle,
                                      self.max_observation_range) for i in range(self.num_beams)]

        ranges = np.array(ranges)
        if self.add_self_position_to_observation:
            ranges = np.concatenate([ranges, self.state])
        if self.add_goal_position_to_observation:
            ranges = np.concatenate([ranges, self.destination])
        return ranges

    def _step(self, action):
        old_state = self.state.copy()
        v = action[0]
        theta = action[1]
        dx = v*cos(theta)
        dy = v*sin(theta)

        self.state += np.array([dx, dy])

        reward = -1 # minus 1 for every timestep you're not in the goal
        done = False
        info = {}

        if np.linalg.norm(self.destination - self.state) < self.destination_tolerance_range:
            reward = 20 # for reaching the goal
            done = True

        if not self.world.point_is_in_free_space(self.state[0], self.state[1], epsilon=0.25):
            reward = -5 # for hitting an obstacle

        if not self.world.segment_is_in_free_space(old_state[0], old_state[1],
                                                   self.state[0], self.state[1],
                                                   epsilon=0.25):
            reward = -5 # for hitting an obstacle

        self.observation = self._get_observation(self.state)
        return self.observation, reward, done, info

    def _reset(self):
        self.state = self.init_position
        return self._get_observation(self.state)

    def _plot_state(self, viewer, state):
        polygon = gym_rendering.make_circle(radius=5, res=30, filled=True)
        state_tr = gym_rendering.Transform(translation=(state[0], state[1]))
        polygon.add_attr(state_tr)
        viewer.add_onetime(polygon)

    def _plot_observation(self, viewer, state, observation):
        delta_angle = 2*pi/self.num_beams
        for i in range(len(observation)):
            r = observation[i]
            if r < 0:
                r = self.max_observation_range

            theta = i*delta_angle
            start = (state[0], state[1])
            end = (state[0] + r*cos(theta), state[1] + r*sin(theta))

            line = gym_rendering.Line(start=start, end=end)
            line.set_color(.5, 0.5, 0.5)
            viewer.add_onetime(line)

    def _append_elements_to_viewer(self, viewer,
                                   screen_width,
                                   screen_height,
                                   obstacles,
                                   destination=None,
                                   destination_tolerance_range=None):

        viewer.set_bounds(left=-100, right=screen_width+100, bottom=-100, top=screen_height+100)

        L = len(obstacles)
        for i in range(L):

            obs = obstacles[i]
            for c,w,h in zip(obs.rectangle_centers, obs.rectangle_widths, obs.rectangle_heights):
                l = -w/2.0
                r = w/2.0
                t = h/2.0
                b = -h/2.0

                rectangle = gym_rendering.FilledPolygon([(l,b), (l,t), (r,t), (r,b)])
                tr = gym_rendering.Transform(translation=(c[0], c[1]))
                rectangle.add_attr(tr)
                rectangle.set_color(.8,.6,.4)
                viewer.add_geom(rectangle)

        if not (destination is None):
            tr = gym_rendering.Transform(translation=(destination[0], destination[1]))
            polygon = gym_rendering.make_circle(radius=destination_tolerance_range, res=30, filled=True)
            polygon.add_attr(tr)
            polygon.set_color(1.0, 0., 0.)
            viewer.add_geom(polygon)

    def _render(self, mode='human', close=False):

        if close:
            if self.viewer is not None:
                self.viewer.close()
            self.viewer = None
            return

        screen_width = (self.world.x_range[1] - self.world.x_range[0])
        screen_height = (self.world.y_range[1] - self.world.y_range[0])

        if self.viewer is None:
            self.viewer = gym_rendering.Viewer(screen_width, screen_height)
            self._append_elements_to_viewer(self.viewer,
                                            screen_width,
                                            screen_height,
                                            obstacles=self.world.obstacles,
                                            destination=self.destination,
                                            destination_tolerance_range=self.destination_tolerance_range)

        self._plot_state(self.viewer, self.state)
        self._plot_observation(self.viewer, self.state, self.observation)

        return self.viewer.render(return_rgb_array = mode=='rgb_array')

class StateBasedMDPNavigation2DEnv(LimitedRangeBasedPOMDPNavigation2DEnv):
    logger = logging.getLogger(__name__)
    def __init__(self, *args, **kwargs):
        LimitedRangeBasedPOMDPNavigation2DEnv.__init__(self, *args, **kwargs)
        low = [-float('inf'), -float('inf'), 0.0, 0.0]
        high = [float('inf'), float('inf'), float('inf'), 2*pi]

        if self.add_goal_position_to_observation:
            low.extend([-10000., -10000.]) # x and y coords
            high.extend([10000., 10000.])

        self.observation_space = Box(np.array(low), np.array(high))

    def _plot_observation(self, viewer, state, observation):
        pass

    def _get_observation(self, state):
        # return state
        dist_to_closest_obstacle, absolute_angle_to_closest_obstacle = self.world.range_and_bearing_to_closest_obstacle(state[0], state[1])
        obs = np.array([state[0], state[1], dist_to_closest_obstacle, absolute_angle_to_closest_obstacle])
        if self.add_goal_position_to_observation:
            obs = np.concatenate([obs, self.destination])
        return obs


from lxml import html
import requests
import urllib
import random
import sys

#########################################################
# Main
#########################################################
def main(argv):
    file_write = argv[1]
    # Select a closed chanlenge from the first page
    page = requests.get('http://www.dpreview.com/challenges/ChallengesFinished.aspx')
    tree = html.fromstring(page.text)
    winners = tree.xpath('//a[contains(@id,"lnkChallengeDetail")]/@href')
    selected = random.choice(winners)
    # Select an entry from that chanlenge - top 5 only
    page_selected = requests.get(selected)
    tree_selected = html.fromstring(page_selected.text)
    first_place = tree_selected.xpath('//a[contains(@class, "imgIcon zoomIcon")]/@href')
    first_place_bigger = random.choice(first_place[:5])
    # Get image and write it to a file
    id = first_place_bigger[49:].replace('&View=Results&Rows=4','')
    url =  "http://www.dpreview.com/challenges/DownloadOriginal.aspx?id=%s"% id
    urllib.urlretrieve(url, file_write)

# Call main if not imported
if __name__ == "__main__":
    main(sys.argv)


import os
import cv2

from .utils import *
from ..data import cfg
from .augmentation import *
from .dataset import BaseDatset

import torch
import torch.nn.functional as F

class YoloDatset(BaseDatset):
    def __init__(self, image_path, label_path, txt_path, is_augment=False) -> None:
        self.aug = AlbumAug()
        self.txt_path = txt_path
        self.image_path = image_path
        self.label_path = label_path
        self.is_augment = is_augment
        self.image_size = cfg.models.image_size
        self.tranform = Transform(self.image_size)
        self.dataset_voc = self.load_dataset_voc_format(self.image_path, self.label_path, self.txt_path)

    def get_image_label(self, image_pth, bboxes, labels):
        image = cv2.imread(image_pth)
        if self.is_augment:
            image, bboxes, labels = self.aug(image, bboxes, labels)

        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        image, bboxes, labels = self.tranform(image, bboxes, labels)
        return image, bboxes, labels

    def make_grid_cells(self, cls_ids, boxes):
        S = cfg.models.grid_size
        B = cfg.models.num_bboxes
        C = cfg.models.num_classes
        # Divide the input image into SXS grid cells
        grid_cell_i = self.image_size[0] / S
        grid_cell_j = self.image_size[1] / S
        # Define input shape
        target_cxcywh = torch.zeros((S, S, 5 * B + C), dtype=torch.float32)

        for class_id, bbox in zip(cls_ids, boxes):
            x_min, y_min, x_max, y_max = bbox.copy()
            # Compute center of an object
            x_c = (x_min + x_max) / 2
            y_c = (y_min + y_max) / 2

            # Determine cell position of object
            pos_i = int(x_c // grid_cell_i)
            pos_j = int(y_c // grid_cell_j)

            if target_cxcywh[pos_j, pos_i, 8] > 0 or target_cxcywh[pos_j, pos_i, 9] > 0:
                continue
            # Each grid cell contains:
            #   bbox1: [x_c, y_c, w_box, h_box, p_c, p0, p1, ..., pn]
            #   bbox2: [x_c, y_c, w_box, h_box, p_c, p0, p1, ..., pn]

            p_cls = torch.zeros(C, dtype=torch.float32)
            p_cls[int(class_id)-1] = 1.0
            conf_cls = torch.ones((2, )).long()
            box = torch.FloatTensor([
                (x_c - (pos_i * grid_cell_i)) / grid_cell_i,
                (y_c - (pos_j * grid_cell_j)) / grid_cell_j,
                np.sqrt((x_max - x_min) / self.image_size[0]),
                np.sqrt((y_max - y_min) / self.image_size[1])]).repeat(B)

            # Assign bboxes to each grid cell
            grid_cell = torch.cat([box, conf_cls, p_cls], dim=-1)
            target_cxcywh[pos_j, pos_i, :] = grid_cell

        return target_cxcywh

    def __getitem__(self, index):
        image_path, labels = self.dataset_voc[index]
        cls_ids, bboxes = labels[:, 0], labels[:, 1:]
        image, bboxes, cls_ids = self.get_image_label(image_path, bboxes, cls_ids)
        target = self.make_grid_cells(cls_ids, bboxes)
        return image, target

    def __len__(self):
        return len(self.dataset_voc)


from typing import List, Optional

class Node:
    """node chainable storage unit

    each node has the same basic structure: u, min, max, summary, clusters
    """

    def __init__(self, u: int):
        self._u = 2
        self.min = self.max = None
        self.clusters = {}
        self.summary = None
        # find the size of this universe (next power of 2 containing all the keys)
        while self._u < u:
            self._u = self._u << 1

    def __repr__(self):
        return f"{(self.min, self.max)}u{self._u}"

class VEB:
    """implementation of the van emde boas tree

    this one uses hash tables to achive `O(n*log(log(U)))` space efficiency, instead of the original `O(U)`
    where: `n` is the number of elements in the tree, and `U` is the size of the `universe`

    https://en.wikipedia.org/wiki/Van_Emde_Boas_tree
    https://www.youtube.com/watch?v=hmReJCupbNU&t=1838s

    the ADT contains the following methods:
        - insert: insert a new node in the tree in O(lg lg u)
        - search: return the node with the given value in O(lg lg u)
        - min: return the min element in O(1)
        - max: returns the max element in O(1)
        - delete: delete the given value from the tree in O(lg lg U)
        - successor: return the next in order successor in O(lg lg U)

    usage example:
        # from a list of keys
        v = VEB([33, 66, 1, 65, 5, 7, 41])
        # set universe size
        v = VEB(u=16, keys=[1,3,5,11])
    """

    def __init__(self, keys: List[int] = [], u: Optional[int] = None):
        if u is not None and not 2 <= u:
            raise ValueError(f"{u} must be greater or equal than 2")
        # find the size of universe (next power of 2 containing all the keys)
        self._u = 2
        maximum = u or max(keys, default=2)
        # choose ideal size of the universe
        while self._u < maximum:
            self._u = self._u << 1
        # double it once more for safety
        self._u = self._u << 1
        # build tree from root
        self.root: Node = Node(self._u)
        # insert elements in the tree
        for key in keys:
            self.insert(key)

    def min(self) -> int:
        """returns the min in the tree O(1)

        Returns:
            int: current min in the tree
        """
        if self.root.min is None:
            raise IndexError("empty tree")
        return self.root.min

    def max(self) -> int:
        """returns the max in the tree O(1)

        Returns:
            int: current max in the tree
        """
        if self.root.min is None:
            raise IndexError("empty tree")
        return self.root.max

    def search(self, key: int) -> bool:
        """find a given `key` in the tree O(log log u)

        Args:
            key (int): key to look for

        Raises:
            KeyError: raised when not found

        Returns:
            bool: if the key is contained
        """
        # search recursively
        def r(node, x):
            if node.min == x or node.max == x:
                return True
            # get cluster -> i (high) and offset -> j (low)
            i = x // int(node._u ** (1 / 2))
            j = x % int(node._u ** (1 / 2))
            # don't give up... keep looking!
            if i in node.clusters:
                return r(node.clusters[i], j)
            return False

        res = r(self.root, key)
        if not res:
            raise KeyError(f"{key} not found")
        return res

    def insert(self, key: int) -> None:
        """insert a new key in O(log log u)

        Args:
            key (int): `key` to be inserted

        Raises:
            ValueError: raised for invalid keys...
        """
        if not 0 <= key < self._u:
            raise ValueError(f"{key} out of the universe")

        def r(node, partial_key: int) -> None:
            """insert recursively"""
            # pseudo lazy propagation, makes the insertion log(log(U))
            if node.min is None:
                node.min = node.max = partial_key
                return
            if partial_key < node.min:
                partial_key, node.min = node.min, partial_key
            if partial_key > node.max:
                node.max = partial_key
            # get cluster -> i (high) and offset -> j (low)
            i = partial_key // int(node._u ** (1 / 2))
            j = partial_key % int(node._u ** (1 / 2))
            u = int(node._u ** (1 / 2))
            # update summary if the corresponding cluster was empty
            if node._u > 2 and i not in node.clusters:
                if not node.summary:
                    node.summary = Node(u)
                r(node.summary, i)
            # update the corresponding cluster
            if node._u > 2:
                r(node.clusters.setdefault(i, Node(u)), j)

        r(self.root, key)

    def successor(self, key: int) -> int:
        """returns the successor of the given key in the tree

        the classic data structure returns the size of the universe `self._u`
        if the successor is not found... we will raise an exception

        Args:
            key (int): key of the predecessor

        Raises:
            KeyError: raised when the successor is not in the tree

        Returns:
            int: the key of the sucessor
        """
        if self.root.min is None:
            raise KeyError(f"successor of {key} not found")

        def r(node, x):
            # trivial case
            if x < node.min:
                return node.min
            # no successor availible...
            if x >= node.max:
                return node._u
            # get cluster -> i (high) and offset -> j (low)
            i = x // int(node._u ** (1 / 2))
            j = x % int(node._u ** (1 / 2))
            # professor Erik Demaine was missing this next line...
            if not node._u > 2 and x < node.max:
                return node.max
            if node._u > 2 and i in node.clusters and j < node.clusters[i].max:
                # if key < max in the cluster for sure the successor can be found here
                j = r(node.clusters[i], j)
            else:
                # take a look in the summary first
                i = r(node.summary, i)
                j = node.clusters[i].min
            return i * int(node._u ** (1 / 2)) + j

        res = r(self.root, key)
        if res == self._u:
            raise KeyError(f"successor of {key} not found")
        return res

    def delete(self, key: int) -> None:
        """deletes a `key` from the tree in O(log log U)"""
        # special case: empty tree
        if self.root.min is None:
            return

        def r(node, x: int) -> bool:
            """delete recursively"""
            # base case: we reached a node with a single element
            # return a signal to the parent to erase this empty child
            if node.min is None or x < node.min or node.max < x:
                return False
            if node.min == node.max:
                return True
            # special case: erasing min
            if x == node.min:
                # we dont want to recurse into a node with no summary...
                if not node._u > 2:
                    node.min = node.max
                    return False
                tmp = node.summary.min
                # new min discovered, update
                x = node.min = tmp * int(node._u ** (1 / 2)) + node.clusters.get(tmp, Node(2)).min or 0
            # get cluster -> i (high) and offset -> j (low)
            i = x // int(node._u ** (1 / 2))
            j = x % int(node._u ** (1 / 2))
            # simetrical with respect to insert, we delete first
            if node._u > 2 and i in node.clusters:
                if r(node.clusters[i], j):
                    del node.clusters[i]
            # then if is the cluster is empty, update the summary
            if node._u > 2 and i not in node.clusters:
                r(node.summary, i)
            # special case: erasing max, very similar to the min case
            if x == node.max:
                if not node.clusters:
                    node.max = node.min
                    node.summary = None
                    return
                tmp = node.summary.max
                if tmp in node.clusters:
                    node.max = tmp * int(node._u ** (1 / 2)) + node.clusters.get(tmp, Node(2)).max or 0
            return False

        # base case: we reached a node with a single element
        # return a signal to the parent to erase this empty child
        if r(self.root, key):
            self.root = Node(self._u)

    def __repr__(self):
        """prints first the main node and summary trees recursevly"""
        main, summary = [], []

        def r(tmp, node, level=0):
            if not node:
                return
            tmp.append("\t" * level + f"-->{node}")
            for cluster in node.clusters.values():
                r(tmp, cluster, level + 1)

        # recurse on both the main and the summary tree
        r(main, self.root)
        r(summary, self.root.summary)
        main, summary = "\n".join(main), "\n".join(summary)
        return f"main:\n{main}\nsummary:\n{summary}"

    def __len__(self):
        return self._u


# %% [markdown]
# # 작업형2 기출 유형(심화)
# - 본 문제는 변형한 심화 문제 입니다.
# - 오리지널 3회 기출 유형을 보고 싶은 분은 아래 클래스-커리큘럼 탭에 무료공개(3회 작업형2)로 영상과 데이터셋을 올려놨어요!
# - https://class101.net/products/467P0ZPH0lVX9FwFBDz7
#
# ### 여행 보험 패키지 상품을 구매할 확률 값을 구하시오
# - 예측할 값(y): TravelInsurance (여행보험 패지지를 구매 했는지 여부 0:구매안함, 1:구매)
# - 평가: roc-auc 평가지표
# - data: t2-1-train.csv, t2-1-test.csv
# - 제출 형식
#
# ~~~
# id,TravelInsurance
# 0,0.3
# 1,0.48
# 2,0.3
# 3,0.83
# ~~~
#
# # Baseline
# ### 3회 기출문제에서 데이터 셋을 편집해 조금 더 어렵게 만들었어요
# - 결측치 추가
# - Employment Type 컬럼에 카테고리 추가
# - sample_submission 파일은 제공된 적 없음(3회 때 제출 형식에 대한 이슈가 있어 제공하거나 제출 형식을 명확하게 설명할 가능성 있어 보임)

# %% [code] {"execution":{"iopub.status.busy":"2022-06-14T14:55:19.742731Z","iopub.execute_input":"2022-06-14T14:55:19.743242Z","iopub.status.idle":"2022-06-14T14:55:19.771533Z","shell.execute_reply.started":"2022-06-14T14:55:19.743149Z","shell.execute_reply":"2022-06-14T14:55:19.770761Z"},"jupyter":{"outputs_hidden":false}}
# 라이브러리 불러오기
import pandas as pd

# %% [code] {"execution":{"iopub.status.busy":"2022-06-14T14:55:19.772981Z","iopub.execute_input":"2022-06-14T14:55:19.773329Z","iopub.status.idle":"2022-06-14T14:55:19.805999Z","shell.execute_reply.started":"2022-06-14T14:55:19.773291Z","shell.execute_reply":"2022-06-14T14:55:19.805117Z"},"jupyter":{"outputs_hidden":false}}
# 데이터 불러오기
train = pd.read_csv("../input/big-data-analytics-certification/t2-1-train.csv")
test = pd.read_csv("../input/big-data-analytics-certification/t2-1-test.csv")

train.shape, test.shape

X_train = train.iloc[:,1:-1]
y_train = train['TravelInsurance']

X_test = test.iloc[:,1:]
X_test_id = test.iloc[:,0]
# y_test = test['TravelInsurance']

# dir(X_train)

num_cols = list(X_train.select_dtypes(exclude = 'object').columns)
cat_cols = list(X_train.select_dtypes(include = 'object').columns)

X_train[num_cols].describe()
X_train[cat_cols].describe()

# Missing value handling

X_train[num_cols].isnull().sum()
X_train['AnnualIncome'].fillna(X_train['AnnualIncome'].median(), inplace = True)
X_test[num_cols].isnull().sum()
X_test['AnnualIncome'].fillna(X_train['AnnualIncome'].median(), inplace = True)

# X_train[cat_cols].isnull().sum()
X_test[cat_cols].isnull().sum()

# Feature engineering

from sklearn.preprocessing import MinMaxScaler, LabelEncoder

scaler = MinMaxScaler()
X_train[num_cols] = scaler.fit_transform(X_train[num_cols])
X_test[num_cols] = scaler.transform(X_test[num_cols])

for c in cat_cols:
    le = LabelEncoder()
    X_train[c] = le.fit_transform(X_train[c])
    X_test[c] = le.fit_transform(X_test[c])

# Modelling

from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier()
model.fit(X_train, y_train)
model.score(X_train, y_train)

pred = model.predict_proba(X_test)

# submission

submit = pd.DataFrame()
submit['id'] = test['id']
submit['TravelInsurance'] = pred[:,1]

submit


import numpy as np
from baselines.ecbp.agents.buffer.lru_knn_combine_bp import LRU_KNN_COMBINE_BP
from baselines.ecbp.agents.buffer.lru_knn_combine_bp_2 import LRU_KNN_COMBINE_BP_2
from baselines.ecbp.agents.buffer.lru_knn_prioritizedsweeping import LRU_KNN_PRIORITIZEDSWEEPING
from baselines.ecbp.agents.buffer.lru_knn_kbps import LRU_KNN_KBPS
from baselines.ecbp.agents.graph.model import *
import tensorflow as tf
from baselines.ecbp.agents.graph.build_graph_dueling import *
from baselines import logger
import copy
import logging

class KBPSAgent(object):
    def __init__(self, model_func, exploration_schedule, obs_shape, vector_input, lr=1e-4, buffer_size=1000000,
                 num_actions=6, latent_dim=32,
                 gamma=0.99, knn=10, eval_epsilon=0.01,
                 tf_writer=None, bp=True, debug=True):
        self.ec_buffer = LRU_KNN_KBPS(num_actions, buffer_size, latent_dim, latent_dim, gamma, bp, debug)
        self.obs = None
        self.z = None
        self.ind = -1
        self.writer = tf_writer
        self.sequence = []
        self.gamma = gamma
        self.bp = bp
        self.num_actions = num_actions
        self.exploration_schedule = exploration_schedule
        self.latent_dim = latent_dim
        self.knn = knn
        self.steps = 0
        self.rmax = 100000
        self.debug = debug
        self.logger = logging.getLogger("ecbp")
        self.heuristic_exploration = True
        self.eval_epsilon = eval_epsilon
        input_type = U.Float32Input if vector_input else U.Uint8Input
        self.hash_func, _, _ = build_train_dueling(
            make_obs_ph=lambda name: input_type(obs_shape, name=name),
            model_func=model_func,
            q_func=model,
            imitate=False,
            num_actions=num_actions,
            optimizer=tf.train.AdamOptimizer(learning_rate=lr, epsilon=1e-4),
            gamma=gamma,
            grad_norm_clipping=10,
        )

    def log(self, *args, logtype='debug', sep=' '):
        getattr(self.logger, logtype)(sep.join(str(a) for a in args))

    def act(self, obs, is_train=True):
        self.obs = obs
        z = np.array(self.hash_func(np.array(obs))).reshape((self.latent_dim,))
        self.z = z
        if self.ind == -1:
            self.ind, knn_dist, knn_ind = self.ec_buffer.peek(z)
            knn_dist = np.array(knn_dist).reshape(-1).tolist()
            knn_ind = np.array(knn_ind).reshape(-1).tolist()
            if self.ind == -1:
                self.ind, _ = self.ec_buffer.ec_buffer.add_node(z)
                knn_dist = [0] + knn_dist
                knn_ind = [self.ind] + knn_ind
                if self.debug:
                    print("add node for first ob ", self.ind)
            self.ec_buffer.dist = knn_dist
            self.ec_buffer.ind = knn_ind
        else:
            ind, knn_dist, knn_ind = self.ec_buffer.peek(z)
            knn_dist = np.array(knn_dist).reshape(-1).tolist()
            knn_ind = np.array(knn_ind).reshape(-1).tolist()
            assert self.ind == ind, "{} {}".format(self.ind, ind)
            self.ec_buffer.dist = knn_dist
            self.ec_buffer.ind = knn_ind

        self.steps += 1
        epsilon = max(0, self.exploration_schedule.value(self.steps)) if is_train else self.eval_epsilon
        if np.random.random() < epsilon:
            action = np.random.randint(0, self.num_actions)
            return action
        else:
            finds = np.zeros((1,))
            extrinsic_qs, intrinsic_qs, find = self.ec_buffer.ec_buffer.act_value(np.array([z]), self.knn)
            extrinsic_qs, intrinsic_qs = np.array(extrinsic_qs).squeeze(), np.array(intrinsic_qs).squeeze()
            finds += sum(find)
            if is_train:
                # self.log("extrinsic q shaps",extrinsic_qs.shape,extrinsic_qs)
                # extrinsic_qs = np.array([x if x > -self.rmax else 0 for x in extrinsic_qs])
                q = intrinsic_qs + extrinsic_qs
            else:
                q = extrinsic_qs
            q = np.squeeze(q)
            q_max = np.max(q)

            max_action = np.where(q >= q_max - 1e-7)[0]
            self.log("action selection", max_action)
            self.log("q", q, q_max)
            action_selected = np.random.randint(0, len(max_action))
            return max_action[action_selected]

    def observe(self, action, reward, state_tp1, done, train=True):

        z_tp1 = np.array(self.hash_func(np.array(state_tp1)[np.newaxis, ...])).reshape((self.latent_dim,))
        if train:
            self.ind = self.ec_buffer.prioritized_sweeping((self.ind, action, reward, z_tp1, done))
        else:
            self.ind, self.ec_buffer.dist, self.ec_buffer.ind = self.ec_buffer.peek(z_tp1)
        if done:
            self.ind = -1
            self.steps = 0

    def finish(self):
        pass


import telebot

bot = telebot.TeleBot('5530513392:AAG89CLkxSe_ytQvCMTJQZOfRF-NqbOov4A')

# Функция, обрабатывающая команду /start
@bot.message_handler(commands=["start"])
def start(m, res=False):
    bot.send_message(m.chat.id, 'Я на связи. Пришлю информацию по просроченным поставкам )')
    file = open("chats_ud.txt", "a")
    file.write(str(m.chat.id)+'\n')
    file.close()

# Запускаем бота
bot.polling(none_stop=True, interval=10)


import faiss, os
from llama_index.vector_stores import FaissVectorStore
from llama_index import VectorStoreIndex, SimpleDirectoryReader, Document
from llama_index import StorageContext, load_index_from_storage
from llama_index.vector_stores.simple import SimpleVectorStore
from llama_index.storage.docstore.simple_docstore import SimpleDocumentStore
from llama_index.storage.index_store.simple_index_store import SimpleIndexStore

class FaissEmbeddingStorage:

    def __init__(self, data_dir, dimension=384):
        self.d = dimension
        self.data_dir = data_dir
        self.index = self.initialize_index()

    def initialize_index(self):
        if os.path.exists("storage-default") and os.listdir("storage-default"):
            print("Using the presisted value")
            vector_store = FaissVectorStore.from_persist_dir("storage-default")
            storage_context = StorageContext.from_defaults(
                vector_store=vector_store, persist_dir="storage-default"
            )
            index = load_index_from_storage(storage_context=storage_context)
            return index
        else:
            print("generating new values")
            documents = SimpleDirectoryReader(self.data_dir).load_data()
            faiss_index = faiss.IndexFlatL2(self.d)
            vector_store = FaissVectorStore(faiss_index=faiss_index)
            storage_context = StorageContext.from_defaults(vector_store=vector_store)
            index = VectorStoreIndex.from_documents(documents, storage_context=storage_context)
            index.storage_context.persist(persist_dir = "storage-default")
            return index

    def get_query_engine(self):
        return self.index.as_query_engine()


import logging
import os
from flask import Flask, abort
from google.cloud import storage

app = Flask(__name__)

# Configure this environment variable via app.yaml
CLOUD_STORAGE_BUCKET = os.environ.get('CLOUD_STORAGE_BUCKET')
CLOUD_STORAGE_PREFIX = os.environ.get('CLOUD_STORAGE_PREFIX', '')

if not CLOUD_STORAGE_BUCKET:
    raise RuntimeError('CLOUD_STORAGE_BUCKET environment variable must be specified')

@app.route('/', defaults={'path': 'index.html'})
@app.route('/<path:path>')
def index(path):
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(CLOUD_STORAGE_BUCKET)
    blob = bucket.get_blob(os.path.join(CLOUD_STORAGE_PREFIX, path))
    if blob is None:
        abort(404)

    content = blob.download_as_string()
    if blob.content_encoding:
        return content.decode(blob.content_encoding)
    else:
        return content

@app.errorhandler(404)
def handle_404(_):
    return '404: This page does not exist', 404

@app.errorhandler(Exception)
def server_error(e):
    logging.exception(e)
    return """An internal error occurred: <pre>{}</pre>
See logs for full stacktrace.""".format(e)

if __name__ == '__main__':
    app.run(debug=True)


"""
Tests for the press of the "Q" key or the end of the video.
"""

import cv2

from pipeline import Stage
from pipeline.stage_result import StageResult

class TestExit(Stage):
    """
    Tests for the press of the "Q" key or the end of the video.
    """

    def __init__(self) -> None:
        Stage.__init__(self)

    def execute(self) -> StageResult:
        """
        Tests for the press of the "Q" key or the end of the video.
        """

        if cv2.waitKey(50) & 0xFF == ord("q"):
            return StageResult(False, None)

        return StageResult(True, True)


# this is a demo for dynamic programming of Reinforcement Learning
import numpy as np
from pandas import DataFrame
from simple_env import SimpleEnv

class Dp:
    def __init__(self, env, reward_discount=0.9):
        self.actions = env.actions_space()
        self.gamma = reward_discount
        self.env = env
        self.total_states = env.total_states()
        action_p = np.ones(env.total_states())
        self.policy = DataFrame(data=list(zip(action_p, action_p, action_p, action_p)),
                                columns=['up', 'down', 'left', 'right'])
        self.state_value = np.zeros(env.total_states())

    def choose_action(self):
        pass

    def learn(self):
        pass

    def print(self):
        print(self.policy)
        print(self.state_value.reshape(self.env.size()))

    def compute_state_value(self, state):
        sum_value = 0.0
        sum_rate = 0.0
        for action in self.policy.columns:
            next_state, reward = self.env.step_with_state(action, state)
            sum_value += self.policy[action][state] * self.state_value[next_state] * self.gamma + reward
            sum_rate += self.policy[action][state]
        if sum_rate == 0.0:
            return 0.0
        return sum_value / sum_rate

    def policy_evaluation(self):
        new_state_value = np.zeros(self.total_states)
        for i in range(1, self.total_states - 1):
            new_state_value[i] = self.compute_state_value(i)
        self.state_value = new_state_value

    def single_state_policy(self, state):
        values = np.array([float('-inf') for _ in range(0, len(self.actions))])
        new_policy = np.zeros(len(self.actions))
        for action in self.policy.columns:
            action_index = self.policy.columns.get_loc(action)
            next_state, _ = self.env.step_with_state(action, state)
            if self.policy[action][state] > 0.0 and state != next_state:
                values[action_index] = self.state_value[next_state]
        max_value = values.max()
        for i in range(0, len(self.actions)):
            if values[i] == max_value:
                new_policy[i] = 1.0
            else:
                new_policy[i] = 0.0
        return new_policy

    def policy_iteration(self):
        for s in range(1, self.total_states - 1):
            self.policy.loc[s] = self.single_state_policy(s)

if __name__ == '__main__':
    env = SimpleEnv(4, 4)
    dp = Dp(env, 1)
    for i in range(200):
        env.reset()
        dp.policy_evaluation()
        dp.policy_iteration()
        dp.print()


import sys
sys.stdin = open("5202.txt")

T = int(input())
for tc in range(1,T+1):
    N = int(input())
    data = [list(map(int, input().split())) for _ in range(N)]
    data.sort(key=lambda x : x[1])
    cnt = 0
    end = 0
    for i in range(N):
        if end <= data[i][0]:
            cnt += 1
            end = data[i][1]
    print('#{} {}'.format(tc, cnt))


from flask import Flask, request, jsonify
from gpt import ask
app = Flask(__name__)

@app.route('/')
def hello_world():  # put application's code here
    return 'Hello World!'

@app.post("/api/v1/chat-bot")
def detectSystemModel():
    question = request.form.get('question');
    # Encode the bytes using base64
    result_str = ask(question)

    return jsonify({'result': result_str})

if __name__ == '__main__':
    app.run()


import math

def minEatingSpeed(piles, h):
    # binary search init
    l, r = 1, max(piles)

    k = 0

    while l <= r:
        m = (l + r) // 2

        totalTime = 0
        for p in piles:
            totalTime += math.ceil(p / m)

        if totalTime <= h:
            r = m - 1
            k = m
        else:
            l = m + 1

    return k


from tkinter import *
master = Tk()
w = Canvas(master, width=500, height=500,bg="red")
w.pack()
# canvas_height=20
# canvas_width=200
# y = int(canvas_height / 2)
w.create_line(10, 10, 100, 10 )
coord = 10, 50, 240, 210
arc = w.create_arc(coord, start=0, extent=150, fill="blue")
mainloop()


import struct
import socket
HOST = '192.168.1.1'
PORT = 6340
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))

for x in range(200):
      size = struct.unpack('i', s.recv(4)) [0]
      str_data= s.recv(size)
      print ('Data size: %s Data value: %s' % (size, str_data.decode('ascii')))
s.sendall(b'Enough data :) Thanks')


#%%
from pickle import FALSE
import pandas as pd
import numpy as np
from datetime import date, timedelta, datetime
from tqdm import tqdm
import plotly.graph_objects as go
import numpy as np
import streamlit as stream

stream.set_page_config(
     page_title="Cartologia | Cartola 2022",
     page_icon="ðŸŽ²",
     layout="wide",
     initial_sidebar_state="expanded",
     menu_items={
         'Get Help': 'https://www.extremelycoolapp.com/help',
         'Report a bug': "https://www.extremelycoolapp.com/bug",
         'About': "# This is a header. This is an *extremely* cool app!"
     }
 )

#%%

def apply_debito(df, qtd_):
    debt_ = qtd_
    k = 0
    while(debt_>0 and k<len(df)):

        if(df.iloc[k,1]>=debt_):
            df.iloc[k,1] = df.iloc[k,1] - debt_
            debt_ = 0
        elif(df.iloc[k,1]<debt_):
            df.iloc[k,1] = 0
            debt_ = debt_ - df.iloc[k,1]

        k = k+1
    return debt_, df

#%%

def generate_forecast_(MEAN_DEMAND = 100, STD_DEMAND = 50, SIZE_FORECAST = 1000, MAPE_OFFSET = 0.3, MAPE = 1):

    #CREATE FORECAST
    MEAN_DEMAND = 100
    STD_DEMAND = 50
    SIZE_FORECAST = 1000

    day_vec = []
    for i in range(0,SIZE_FORECAST):
        day_ = date.today() + timedelta(days=i)
        day_vec.append(day_)

    data = np.random.normal(MEAN_DEMAND,STD_DEMAND,SIZE_FORECAST)

    dataframe_forecast = pd.DataFrame(columns=['data','forecast'])
    dataframe_forecast['data'] = day_vec
    dataframe_forecast['forecast'] = data

    dataframe_forecast['forecast'] = [0 if x<=0 else int(x) for x in dataframe_forecast['forecast']]

    real_sales = dataframe_forecast.copy()
    real_sales['sales'] = real_sales['forecast']
    real_sales = real_sales.drop('forecast', axis=1)

    real_sales['sales'] = [int(x*(1+np.random.normal(MAPE_OFFSET, MAPE, 1))) for x in real_sales['sales']]
    real_sales['sales'] = [0 if x<=0 else int(x) for x in real_sales['sales']]

    return dataframe_forecast, real_sales

#%%

def get_cashflow_and_stock_sim(stock_map_dataframe, pay_dataframe, TIME_RANGE = 240, flag_backtest=True):

    CASH_FLOW_VEC = []
    STOCK_VEC = []

    CASH_BANK = 0
    COUNT_FIRST_SALE = 0
    for i in range(0, TIME_RANGE):

        day_sim = date.today()+timedelta(days=i)

        if(flag_backtest):
            sale_  = dataframe_forecast[dataframe_forecast['data']==day_sim].forecast.sum()
        else:
            sale_  = real_sales[real_sales['data']==day_sim].sales.sum()

        qtd_estoque = stock_map_dataframe[stock_map_dataframe['data']<=day_sim].lote.sum()

        if(COUNT_FIRST_SALE>=0 and COUNT_FIRST_SALE<LTREC_CLIENTE and qtd_estoque>0):
            COUNT_FIRST_SALE = COUNT_FIRST_SALE + 1
            #stock_map_dataframe = apply_debito(stock_map_dataframe, AVG_SALE_MULT)[1]
            stock_map_dataframe = apply_debito(stock_map_dataframe, sale_)[1]

        if(COUNT_FIRST_SALE>=LTREC_CLIENTE and qtd_estoque>0):
            #CASH_BANK = CASH_BANK + AVG_SALE_MULT*PRICE
            #stock_map_dataframe = apply_debito(stock_map_dataframe, AVG_SALE_MULT)[1]
            stock_map_dataframe = apply_debito(stock_map_dataframe, sale_)[1]
            CASH_BANK = CASH_BANK + sale_*PRICE

        stock_map_dataframe = stock_map_dataframe[stock_map_dataframe['lote']>0]

        DEBT_ = -1*pay_dataframe[pay_dataframe['data']==day_sim].valor_pay.sum()
        CASH_BANK = CASH_BANK + DEBT_

        qtd_estoque = stock_map_dataframe[stock_map_dataframe['data']<=day_sim].lote.sum()

        STOCK_VEC.append([day_sim, qtd_estoque])
        CASH_FLOW_VEC.append([day_sim, CASH_BANK])

        if(len(stock_map_dataframe)==0 and flag_backtest):
            break

    dataframe_stock_simulation = pd.DataFrame(data = STOCK_VEC, columns = ['data_sim','pos_estoque'])
    dataframe_cashflow_simulation = pd.DataFrame(data = CASH_FLOW_VEC, columns = ['data_sim','cash_flow'])

    return dataframe_stock_simulation, dataframe_cashflow_simulation

#%%

def position_order(pay_dataframe, stock_map_dataframe, order_to_be, order_date_, lerror1, lerror2):

    PAYMENT_DATE_VEC_FUT = [[order_date_+timedelta(days=LTPAG2+(LT1+lerror1)+(LT2+lerror2)), order_to_be*COST_OF_LABOR],
                [order_date_+timedelta(days=LTPAG1+LT1), order_to_be*COST_OF_GOODS]
                ]

    pay_dataframe = pay_dataframe.append(pd.DataFrame(data = PAYMENT_DATE_VEC_FUT, columns = ['data','valor_pay']))

    STOCK_MAP_FUT = [[order_date_+timedelta(days=(LT1+lerror1)+(LT2+lerror2)), order_to_be]]
    stock_map_dataframe = stock_map_dataframe.append(pd.DataFrame(data = STOCK_MAP_FUT, columns = ['data','lote']))

    return stock_map_dataframe, pay_dataframe

#%%
#PARAMETERS FROM SUPPLIERS
LT1 = 30
LT2 = 30
LTREC_CLIENTE = 3
LTPAG2 = 30
LTPAG1 = 30

#PARAMETERS OF SUPPLY
STOCK_MAX_ = 60
MEAN_DEMAND = 100
STD_DEMAND = 50
SIZE_FORECAST = 1000
MAPE_OFFSET = 0
MAPE = 0
LEADTIME_ERROR = 0

#PARAMETERS OF FINANCE
PRICE = 10
MARKUP = 2.5
TOTAL_COST = PRICE/3.5
COST_OF_GOODS = 0.7*TOTAL_COST
COST_OF_LABOR = 0.3*TOTAL_COST

#PARAMETERS OF SIMULATION
N_MAX_ORDERS = 3
TIME_RANGE = 1000

dataframe_forecast, real_sales = generate_forecast_(MEAN_DEMAND = MEAN_DEMAND,
                                                     STD_DEMAND = STD_DEMAND,
                                                      SIZE_FORECAST = SIZE_FORECAST,
                                                         MAPE_OFFSET = MAPE_OFFSET,
                                                             MAPE = MAPE)

ORDER = dataframe_forecast[(dataframe_forecast['data']>=pd.to_datetime(date.today()+timedelta(days=LT2+LT1))) &
 (dataframe_forecast['data']<=pd.to_datetime(date.today()+timedelta(days=LT2+LT1+STOCK_MAX_)))].forecast.sum()

PAYMENT_DATE_VEC = [[date.today()+timedelta(days=LTPAG2+LT1+LT2), ORDER*COST_OF_LABOR],
                    [date.today()+timedelta(days=LTPAG1+LT1), ORDER*COST_OF_GOODS]
]

pay_dataframe = pd.DataFrame(data = PAYMENT_DATE_VEC, columns = ['data','valor_pay'])

STOCK_MAP = [[date.today()+timedelta(days=LT2+LT1), ORDER]]
stock_map_dataframe = pd.DataFrame(data = STOCK_MAP, columns = ['data','lote'])

ORDER_BACKLOG_VEC = []

n_orders_ = 0
while(n_orders_<=N_MAX_ORDERS):
    print("At order n_ : {}".format(n_orders_))
    dataframe_stock_simulation, dataframe_cashflow_simulation  = get_cashflow_and_stock_sim(stock_map_dataframe, pay_dataframe, TIME_RANGE = TIME_RANGE, flag_backtest=True)

    first_rupture_dataframe = dataframe_stock_simulation[(dataframe_stock_simulation['data_sim']>=date.today()+timedelta(days=LT1+LT2)) &
    (dataframe_stock_simulation['pos_estoque']==0)]

    if(len(first_rupture_dataframe)>0):
        data_ruptura = first_rupture_dataframe.head(1).data_sim.tolist()[0]
        order_date_ = data_ruptura - timedelta(days=LT1+LT2)

        new_order_ = dataframe_forecast[(dataframe_forecast['data']>=pd.to_datetime(order_date_+timedelta(days=LT2+LT1))) &
        (dataframe_forecast['data']<=pd.to_datetime(order_date_+timedelta(days=LT2+LT1+STOCK_MAX_)))].forecast.sum()

        ld_order_error1 = int(np.random.normal(0, LEADTIME_ERROR, 1))
        ld_order_error2 = int(np.random.normal(0, LEADTIME_ERROR, 1))

        stock_map_dataframe, pay_dataframe =  position_order(pay_dataframe, stock_map_dataframe, new_order_, order_date_, ld_order_error1, ld_order_error2)

        real_sales_backlog = real_sales[(real_sales['data']>=pd.to_datetime(order_date_+timedelta(days=LT2+LT1))) &
        (real_sales['data']<=pd.to_datetime(order_date_+timedelta(days=LT2+LT1+STOCK_MAX_)))].sales.sum()

        demand_mape_error = abs(new_order_ - real_sales_backlog)/real_sales_backlog

        ORDER_BACKLOG_VEC.append([order_date_, new_order_, ld_order_error1, ld_order_error2, demand_mape_error])

        n_orders_ = n_orders_ + 1
    else:
        break

DF_ORDER = pd.DataFrame(data = ORDER_BACKLOG_VEC, columns = ['order_date_', 'new_order_', 'ld_order_error1', 'ld_order_error2', 'demand_mape_error'])

#%%

dataframe_stock_simulation, dataframe_cashflow_simulation  = get_cashflow_and_stock_sim(stock_map_dataframe, pay_dataframe, TIME_RANGE = TIME_RANGE, flag_backtest=False)

#%%

valid_range = dataframe_stock_simulation[(dataframe_stock_simulation['data_sim']>=dataframe_stock_simulation[dataframe_stock_simulation['pos_estoque']>0].head(1).data_sim.tolist()[0]) &
(dataframe_stock_simulation['data_sim']<=dataframe_stock_simulation[dataframe_stock_simulation['pos_estoque']>0].tail(1).data_sim.tolist()[0])]

#%%

indicador_disponibilidade = 1 - len(valid_range[valid_range['pos_estoque']==0])/len(valid_range)
cashflow_final = dataframe_cashflow_simulation.tail(1).cash_flow.tolist()[0]
cashflow_max = dataframe_cashflow_simulation.cash_flow.max()
cashflow_min = dataframe_cashflow_simulation.cash_flow.min()

dataframe_cashflow_simulation_cp = dataframe_cashflow_simulation.reset_index()
cond = dataframe_cashflow_simulation_cp['cash_flow']<0
dataframe_cashflow_simulation_cp.date=pd.to_datetime(dataframe_cashflow_simulation_cp.data_sim)

max_time_negative_cashflow = dataframe_cashflow_simulation_cp[~cond].groupby(cond.cumsum())['data_sim'].agg(np.ptp).reset_index().sort_values(by='data_sim', ascending=False).cash_flow.tolist()[0]

#%%

dataframe_stock_simulation = dataframe_stock_simulation[dataframe_stock_simulation['data_sim']<=dataframe_stock_simulation[dataframe_stock_simulation['pos_estoque']>0].tail(1).data_sim.tolist()[0]]
dataframe_cashflow_simulation = dataframe_cashflow_simulation[dataframe_cashflow_simulation['data_sim']<=dataframe_stock_simulation[dataframe_stock_simulation['pos_estoque']>0].tail(1).data_sim.tolist()[0]]

# Create traces
fig = go.Figure()
fig.add_trace(go.Scatter(x=dataframe_stock_simulation['data_sim'],
                         y=dataframe_stock_simulation['pos_estoque'],
                         name='STOCK',
                    mode='lines+markers',
                    yaxis='y2'))

fig.add_trace(go.Scatter(x=dataframe_cashflow_simulation['data_sim'],
                         y=dataframe_cashflow_simulation['cash_flow'],
                         name='CASH FLOW',
                    mode='lines+markers'))

fig.update_layout(
    height=500,
    width = 1200,
    yaxis=dict(
        title="CASHFLOW",
        titlefont=dict(
            color="#1f77b4"
        ),
        tickfont=dict(
            color="#1f77b4"
        )
    ),
    yaxis2=dict(
        title="STOCK",
        titlefont=dict(
            color="#ff7f0e"
        ),
        tickfont=dict(
            color="#ff7f0e"
        ),
        overlaying="y",
        side="right"
    ))

stream.plotly_chart(fig, use_container_width=True)
# %%

import plotly.graph_objects as go

fig = go.Figure()

fig.add_trace(go.Indicator(
    mode = "number",
    value = int(indicador_disponibilidade*100),
    domain = {'row': 0, 'column': 0}))

fig.update_layout(height=200,
    width = 400)

fig.show()

stream.plotly_chart(fig)

# %%

with stream.expander("GlossÃ¡rio & Premissas"):
        stream.write("""
            WebApp dedidcado para anÃ¡lises do Cartola FC 2022 \n
            Scouts = Eventos que ocorrem durantse uma partida de futebol \n
            A = AssistÃªncia - (Passes que resultam imediatamente em gol) \n
            G = Gols \n
            FS = Faltas sofridas \n
            FD = FinalizaÃ§Ãµes Defendidados - (Chutem que vÃ£o no gol mas sÃ£o defendidas) \n
            DS = Desarmes - (Admitidos com InterceptaÃ§Ãµes declaradas + 50% de dividias vencidas) \n
            FF = FinalizaÃ§Ãµes para Fora - (Chutes que nÃ£o vÃ£o no gol) \n
            F =  FinalizaÃ§Ãµes - (Representa a soma de FF + FD + FT) \n
        """)


import logging

from fuel_agent_ci.objects import Object

LOG = logging.getLogger(__name__)

class Http(Object):
    __typename__ = 'http'

    def __init__(self, env, name, http_root, port, network,
                 status_url='/status', shutdown_url='/shutdown'):
        self.name = name
        self.env = env
        self.http_root = http_root
        self.port = port
        self.network = network
        self.status_url = status_url
        self.shutdown_url = shutdown_url

    def start(self):
        if not self.status():
            LOG.debug('Starting HTTP server')
            self.env.driver.http_start(self)

    def stop(self):
        if self.status():
            LOG.debug('Stopping HTTP server')
            self.env.driver.http_stop(self)

    def status(self):
        status = self.env.driver.http_status(self)
        LOG.debug('HTTP status %s' % status)
        return status


from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.reverse import reverse
from django.core.urlresolvers import resolve

@api_view(('GET',))
def version_root(request, format=None):
    """List urls of this directory
    """
    from . import urls
    r = resolve(request.path_info)
    version_root_namespaces = r.namespaces
    view_url_dict = {}
    instance_api_resolvers = urls.instance_api_urlpatterns
    for instance_api_resolver in instance_api_resolvers:
        instance_api_namespaces = version_root_namespaces
        instance_api_namespaces.append(instance_api_resolver.namespace)
        for endpoint_url_pattern in instance_api_resolver.url_patterns:
            endpoint_namespaces = instance_api_namespaces
            endpoint_name = endpoint_url_pattern.name
            endpoint_namespaces.append(endpoint_name)
            viewname = ':'.join(endpoint_namespaces)
            view_url_dict[endpoint_name] = reverse(viewname, request=request, format=format)
    return Response(view_url_dict)


# (I): Create a dictionary of coordinates of all stations
# (II): Computing the distances between stations and export into excel file, print LaTeX table string. Also compute
#       the distance of stations from the meteorological station, print to LaTeX and excel
# (III.1): Create a function which calculates inverse-distance based weights for stations based on coordinates
# (III.2): Create a parsimonious version of (III.1) based on distances

#Importing dependencies
import pandas as pd
import csv

if __name__=='__main__':
    #Reading stations
    #all stations
    with open(r"C:\Users\Máté\Dropbox\CEU\2017 Winter\Econ\Smog\Data\All_Stations.csv", 'r') as myfile:
        rd=csv.reader(myfile, delimiter=',')
        #list the headers
        allstations=list(rd)
        #correct for the returned list of list file only to have a simple a list
        allstations=allstations[0]
    #  kept stations
    with open(r"C:\Users\Máté\Dropbox\CEU\2017 Winter\Econ\Smog\Data\Kept_Stations.csv", 'r') as myfile:
        rd=csv.reader(myfile, delimiter=',')
        #list the headers
        keptstations=list(rd)
        #correct for the returned list of list file only to have a simple a list
        keptstations=keptstations[0]
    print('The kept stations are \n', keptstations, '\n\n')

    #(I): Create a DataFrame containing all the stations and the coordinates in decimal degrees, print to LaTex
    # and export to excel
    #DataFrame
    d={'Teteny':[47.4061243, 19.0091796],'Csepel':[47.4047352, 19.0911827], 'Erzsebet':[47.4975260, 19.0527480],
     'Gergely':[47.4674130, 19.1558940], 'Gilice':[47.4309037, 19.1811572], 'Honved':[47.5217225, 19.0684197],
     'Kaposztas':[47.5853036, 19.1147661], 'Korakas':[47.5433350,19.1465781], 'Kosztolanyi':[47.4759677, 19.0400691],
    'Pesthidegkut':[47.5620970, 18.9614345], 'Szena':[47.5078719, 19.0267938], 'Teleki':[47.4934215, 19.0847853]}
    coordinates=pd.DataFrame(d, index=['latitude', 'longitude'])
    d={'Teteny':[47.4061243, 19.0091796],'Csepel':[47.4047352, 19.0911827], 'Erzsebet':[47.4975260, 19.0527480],
     'Gergely':[47.4674130, 19.1558940], 'Gilice':[47.4309037, 19.1811572], 'Honved':[47.5217225, 19.0684197],
     'Kaposztas':[47.5853036, 19.1147661], 'Korakas':[47.5433350,19.1465781], 'Kosztolanyi':[47.4759677, 19.0400691],
    'Pesthidegkut':[47.5620970, 18.9614345], 'Szena':[47.5078719, 19.0267938], 'Teleki':[47.4934215, 19.0847853],
       'Csillaghegy_met': [47.5957565308, 19.0407772064]}
    coordinates_extended=pd.DataFrame(d, index=['latitude', 'longitude'])
    #print to LaTeX
    print('###### LaTeX table for the coordinates of stations\n', coordinates.T.to_latex(),'\n ########\n')
    #export to excel
    coordinates.to_excel(r"C:\Users\Máté\Dropbox\CEU\2017 Winter\Econ\Smog\Data\Coordinates.xlsx")
    print('The coordinates of all PM10 stations are\n', coordinates, '\n\n')

#(II) Distance matrix
#(II.1) Haversine function
#(II.2) Distance matrix: function taking the DataFrame of all coordinates and the name of the station
#     and returns its distance from every station as an array
#(II.3) Distance matrix: call '(II.2)' for every station to make a dictionary, conversion to DataFrame, export to excel,
#     print LaTeX table string
#(II.4) Distance matrix extended version: with meteorological station included, export to excel, print to LaTeX

#(II.1) Haversine, credits to Michael Dunn, Uppsala University,
# posted http://stackoverflow.com/questions/4913349/haversine-formula-in-python-bearing
# -and-distance-between-two-gps-points
def haversine(lon1, lat1, lon2, lat2):
    """
    Calculate the great circle distance between two points
    on the earth (specified in decimal degrees)
    """
    #import math dependencies
    from math import radians, cos, sin, asin, sqrt
    # convert decimal degrees to radians
    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
    # haversine formula
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
    c = 2 * asin(sqrt(a))
    r = 6371 # Radius of earth in kilometers. Use 3956 for miles
    return c * r

#(II.2) distance function
def distancearray(df,s):
    """
    Accepts  (1) a DataFrame object (df) which contains locations as columns and the rows
    are indexed by 'latitude' and 'longitude' corresponding to coordinates of the
    locations in the columns in decimal degrees;
    (2) the name of the column (s) for which the distances from the other locations are
     to be determined.
     Returns an array of distances (dist) in kilometers, where the first element corresponds
     to the first location in the DataFrame.
     Dependencies: pandas, Haversine
    :param s:
    :return:
    """
    #empty array to be filled
    dist=list()
    #looping through all the locations in columns and determine the distance from
    #the location of interest, df['s']
    for key in df:
        dist.append(haversine(df.ix['longitude'][s], df.ix['latitude'][s],
                                   df.ix['longitude'][key],df.ix['latitude'][key]))
    #return the array
    return dist

if __name__=='__main__':
    #(II.3)-(II.4) call 'distancearray' function for every station to create a dictionary, convert to DataFrame,
    # export to excel, print LaTeX table string code
    #distancearray function
    d1={key: distancearray(coordinates,key) for key in coordinates}
    d2={key: distancearray(coordinates_extended,key) for key in coordinates_extended}
    #convert the dictionary to DataFrame
    distancematrix=pd.DataFrame(d1,index=coordinates.columns)
    distancematrix_extended=pd.DataFrame(d2, index=coordinates_extended.columns)
    #export to excel
    distancematrix.to_excel(r"C:\Users\Máté\Dropbox\CEU\2017 Winter\Econ\Smog\Data\Distancematrix.xlsx")
    distancematrix_extended.to_excel(r"C:\Users\Máté\Dropbox\CEU\2017 Winter\Econ\Smog\Data\Distancematrix_extend.xlsx")
    #print to LaTeX
    #rounding to 4 decimals
    def rounding(x):
        return x.round(decimals=4)
    #printing
    print('######## \n\n LaTeX table for distancematrix \n',
       distancematrix.apply(rounding).to_latex(), '\n ########\n')
    print('######## \n\n LaTeX table for extended distancematrix \n',
       distancematrix_extended.apply(rounding).to_latex(), '\n ########\n')

#(III.1) InvDistWeights: function taking the DataFrame of all coordinates, the name of the station,
#      the list of kept stations, and returns the inverse-distance weights, based on
#       station's distance from all the kept stations, as an array
# InvDistWeights
def InvDistWeights(df_coord,s,keptlist):
    """
    Accepts  (1) a DataFrame object (df) which contains locations as columns and the rows
    are indexed by 'latitude' and 'longitude' corresponding to coordinates of the
    locations in the columns in decimal degrees;
    (2) the name of the column (s) for which the distances from the other locations are
     to be determined
    (3) a list of kept locations for which the inverse-distance weights are calculated for
     the station (s)
     Returns an array of inverse-distance weights (weights), where the first element corresponds
     to the first kept location in the DataFrame, the array adds up to 1
     Dependencies: pandas, Haversine
    :param df_coord: DataFrame containing the locations as columns and the rows are indexed by 'latitude' and 'longitude'
            corresponding to coordinates of the locations in the columns
    :param s: name of the location for which the weights are to be determined
    :param keptlist: a list of kept locations for which the inverse-distance weights are calculated for
            the station (s)
    :return: array of the weights, of length same as 'keptlist'
    """
    #Distance from locations in 'keptlist'
    #empty array to be filled
    dist=list()
    # looping through all the locations in columns of DataFrame which is also in 'keptlist'
    #  and determine the distance from the location of interest, df['s']
    for key in df_coord:
         if key in keptlist:
             dist.append(haversine(df.ix['longitude'][s], df.ix['latitude'][s],
                                   df.ix['longitude'][key], df.ix['latitude'][key]))
    #Weights:
    #dist2weights function
    def dist2weights(dlist):
        #reciprocal
        def reciprocal(x):
            if x==0:
                return x
            else:
                return 1/x
        out=list(map(reciprocal, dlist))
        out=list(map(lambda x: x/sum(out), out))
        return out
    return dist2weights(dist)

#(III.2) Parsimonius InvDistWeights function (doesn't calculate the distances at every single call, but relies on
#the DataFrame of distances)
def InvDistWeightsP(df_dist,s,keptlist):
    """
    Returns inverse-distance weights for a location from a distance matrix, based on the distance of the given location
    from selected locations (contributors to weighting)
    (Parsimonious version of InvDistWeights)
    :param df_dist: DataFrame of distance matrix between locations
    :param s:  the location in DataFrame for which the weights are required
    :param keptlist: the stations that are used as weighting ones (contributors)
    :return: an array of weights, of length same as 'keptlist'
    """
    #Extract the appropriate distances from the appropriate column
    dist=list()
    for idx in keptlist:
        dist.append(df_dist.ix[idx, s])
    #Turn the selected distances to weights
    #dist2weights function
    def dist2weights(dlist):
        #reciprocal
        def reciprocal(x):
            if x==0:
                return x
            else:
                return 1/x
        out=list(map(reciprocal, dlist))
        out=list(map(lambda x: x/sum(out), out))
        return out
    return dist2weights(dist)


import pandas as pd
from datetime import timedelta

xls = pd.ExcelFile("E:\PD12 - System errors.xlsx")

autolog = pd.read_excel(xls,"Automatic Error log")
manlog = pd.read_excel(xls,"Manual capture error list")

for k in ['Start','End']:
    manlog[k+'DateTime'] = manlog.apply(lambda x: pd.to_datetime(x[k+' Date'].strftime('%Y-%m-%d')+' '+ x[k+' Time'].strftime('%H:%M:%S')),axis=1)
    autolog[k+'DateTime'] = autolog.apply(lambda x: x[k+' Date / Time']+timedelta(hours=0.5)*(1 if k=='End' else -1),axis=1)
    manlog.drop([k+' Date',k+' Time'],axis=1,inplace = True)

df_dup = pd.merge(manlog, autolog, on='System',suffixes=('_man', '_auto'))
df_dup = df_dup.query('StartDateTime_auto <= StartDateTime_man and EndDateTime_auto >= EndDateTime_man')

final = pd.merge(autolog, df_dup, how='left', on=['System','Start Date / Time','End Date / Time'])
final = final[['System','Error','Start Date / Time','End Date / Time','StartDateTime_man','EndDateTime_man']].copy()

final = pd.merge(final,manlog, how='outer', left_on=['System','StartDateTime_man','EndDateTime_man'], \
                 right_on=['System','StartDateTime','EndDateTime'] ,suffixes=('_a', '_m'), indicator='Check')

for k in ['Start','End']:
    final[k+' Date / Time'] = final.apply(lambda x: x[k+'DateTime'] if pd.isnull(x[k+' Date / Time']) else x[k+' Date / Time'],axis=1)
    final.drop([k+'DateTime',k+'DateTime_man'],axis=1,inplace = True)
final['Error'] = final.apply(lambda x: x['Error_m'] if pd.isnull(x['Error_a']) else x['Error_a'],axis=1)
final['Error source'] = final.apply(lambda x: 'Manual capture error list' if x['Check']=='right_only' else 'Automatic Error log',axis=1)
final['Downtime in Hours'] = final.apply(lambda x: (x['End Date / Time']-x['Start Date / Time'])/timedelta(minutes=60),axis=1)
final['Total Downtime in Hours'] = final['Downtime in Hours'].groupby(final['System']).transform('sum')
final['% of system downtime'] = final['Downtime in Hours']/final['Total Downtime in Hours']
final.drop(['Error_m','Error_a','Check'],axis=1,inplace = True)


import os
import Pyro4

@Pyro4.expose
@Pyro4.behavior(instance_mode="single")

class Server(object):
    def __init__(self):
        self.clients = [] #client -> (name,callback)
        self.names = []
        self.tree = []  # nodes (parent,type,mod,name,route)
        self.currDir = 'root'
        self.pastDir = ''
        self.dirIndex = 0
        self.pastDirIndex = 0
        self.tree.append([-1, 0, 0, "root",'root'])
        self.fillTree()

    def join(self, name, callback):
        if not name:
            raise ValueError('Nombre inválido')
        if (name in self.names):
            raise ValueError('El nombre ya está en uso')
        self.clients.append((name,callback))
        self.names.append(name)

        print("Se ha unido un nuevo cliente")
    def leave(self, name):
        if name not in self.names:
            print("Este cliente no está registrado")
            return
        for (n,c) in self.clients:
            if(n == name):
                self.names.remove(name)
                self.clients.remove((n,c))
                print("Se ha desconectado un cliente")
                break

    def createFile(self,parent,name,dir):
        writeBUffer = open(dir,"w+")
        self.tree.append([parent,1,0,name,dir])
        #writeBUffer.write(content)
        for (n,c) in self.clients:
            c.updateTree()
        writeBUffer.close()

    def mkDir(self,parent,name,dir):
        os.mkdir(dir)#poner 0777 si no funciona
        self.tree.append([parent,0,0,name,dir])
        for (n,c) in self.clients:
            c.updateTree()

    def save(self,index,client,dir,content):
        writeBUffer = open(dir,"w+")
        #self.tree.append((parent,1,0,name,dir))
        writeBUffer.write(content)
        writeBUffer.close()
        for (n,c) in self.clients:
            if(n != client):
                c.update(index)

    def fillTree(self):
        tem = self.pastDir
        temI = self.pastDirIndex
        for filename in os.listdir(self.currDir):
            if(os.path.isdir(self.currDir+'/'+filename)):

                self.pastDir = self.currDir

                self.currDir = self.currDir+'/'+filename
                node = [self.dirIndex,0,0,filename,self.currDir]
                self.tree.append(node)
                self.pastDirIndex = self.dirIndex
                self.dirIndex = self.tree.index(node)

                self.fillTree()
            else:
                node = [self.dirIndex,1,0,filename,self.currDir+'/'+filename]
                self.tree.append(node)

        self.dirIndex = temI
        self.currDir = tem

    def change(self,index,client,content):
        for (n,c) in self.clients:
            if(n != client):
                #c.change()
                break

    def getTree(self):
        return self.tree

    def getNames(self):
        return self.names

    def fetch(self,dir):
        f = open(dir,"r")
        contents = f.read()
        f.close()
        return contents

Pyro4.Daemon.serveSimple(
    {Server: "dfs.server"},
    host = "172.16.14.71"
    )


#七参数的输入
def Coordinate_Input():
    coordinate_before=[]
    coordinate_after=[]

    num=int(input("请输入坐标对的个数："))

    for counter in range(num):
        coordinate_before.append([])
        coordinate_after.append([])

    #if __name__=="__main__":
    for counter in range(num):
        temp=input('请输入坐标系1下的第%s个坐标：'% str(counter+1))
        coordinate_before[counter]=list(map(float,temp.strip().split()))
        print (coordinate_before[counter])

    for counter in range(num):
        temp=input('请输入坐标系2下的第%s个坐标：'% str(counter+1))
        coordinate_after[counter]=list(map(float,temp.strip().split()))
        print (coordinate_after[counter])

    print('坐标1下的坐标对：',coordinate_before)
    print('坐标2下的坐标对：',coordinate_after)

    return num,coordinate_before,coordinate_after

if __name__=='__main__':
    Coordinate_Input()


"""用户订单查询"""
'''
@Time    : 2018/4/6 下午10:16
@Author  : scrappy_zhang
@File    : user_order.py
'''
import logging

from app.user import user
from flask import request, current_app, jsonify, g

from app.utils.response_code import RET
from app.utils.common import login_required

from app.models import House, Order
from app import db, constants

@user.route('/orders')
@login_required
def user_orders():
    """
    查询订单,通过参数判断要查询房东/房客的订单
    :return:
    """
    # 1. 获取用户参数
    user_id = g.user_id
    role = request.args.get('role')
    # 2. 校验参数
    if not role:
        return jsonify(errno=RET.PARAMERR, errmsg='参数有误')

    if role == "custom":
        # 查询当前自己下了哪些订单
        orders_list = Order.query.filter_by(user_id=user_id).order_by(Order.create_time.desc())
    else:
        # 查询自己房屋都有哪些订单
        houses_self = House.query.filter_by(user_id=user_id).all()
        house_ids = [house.id for house in houses_self]
        orders_list = Order.query.filter(Order.house_id.in_(house_ids)).order_by(Order.create_time.desc())

    orders_dict = []

    for order_each in orders_list:
        orders_dict.append(order_each.to_dict())
    return jsonify(errno=RET.OK, errmsg="请求成功", data={"orders": orders_dict})


n, K = map(int, input().split())
rest = 0
result = 0
while(n >= K):
    if n % K != 0:
        rest = n % K
        n -= rest
        result += rest
    n //= K
    result += 1
result += n - 1
print(result)

greedy/이것이 코딩 테스트다/1이 될 때까지.py


from scipy.optimize import linear_sum_assignment
from sklearn.metrics.cluster import supervised

import numpy as np

def clustering_accuracy(labels_true, labels_pred):
    """Clustering Accuracy between two clusterings.
    Clustering Accuracy is a measure of the similarity between two labels of
    the same data. Assume that both labels_true and labels_pred contain n
    distinct labels. Clustering Accuracy is the maximum accuracy over all
    possible permutations of the labels, i.e.
    \max_{\sigma} \sum_i labels_true[i] == \sigma(labels_pred[i])
    where \sigma is a mapping from the set of unique labels of labels_pred to
    the set of unique labels of labels_true. Clustering accuracy is one if
    and only if there is a permutation of the labels such that there is an
    exact match
    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.
    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
    	A clustering of the data into disjoint subsets.
    labels_pred : array, shape = [n_samples]
    	A clustering of the data into disjoint subsets.

    Returns
    -------
    accuracy : float
       return clustering accuracy in the range of [0, 1]
    """
    labels_true, labels_pred = supervised.check_clusterings(labels_true, labels_pred)
    # value = supervised.contingency_matrix(labels_true, labels_pred, sparse=False)
    value = supervised.contingency_matrix(labels_true, labels_pred)
    [r, c] = linear_sum_assignment(-value)
    return value[r, c].sum() / len(labels_true)

def self_representation_loss(labels_true, representation_matrix):
    """Evaluation of self-representation error for self-expressive subspace clustering methods
    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
    	A clustering of the data into disjoint subsets.
    representation_matrix : array, shape = [n_samples, n_samples]
    	Each row is a representation vector

    Returns
    -------
    loss : float
       return self_representation_loss in the range of [0, 1]
    """
    n_samples = labels_true.shape[0]
    loss = 0.0
    for i in range(n_samples):
        representation_vec = np.abs(representation_matrix[i, :])
        label = labels_true[i]
        loss += np.sum(representation_vec[labels_true != label]) / np.sum(representation_vec)

    return loss / n_samples


#!/usr/bin/env python
import sys
import os
home = os.path.expanduser("~")
wavypath = home + '/wavyMini/wavyMini'
sys.path.append(wavypath)

from datetime import datetime, timedelta
from satmod import satellite_altimeter as sa
from stationmod import matchtime
from modelmod import get_model
from collocmod import collocate
from validationmod import validate
from copy import deepcopy
from model_specs import model_dict
from ncmod import dumptonc_stats
import argparse
from argparse import RawTextHelpFormatter
from utils import grab_PID

# parser
parser = argparse.ArgumentParser(
    description="""
Validate wave model output against s3a data and dump to monthly nc-file.
If file exists, data is appended.

Usage:
./validate.py -sd 2019100118 -ed 2019100118 -sat c2 -mod SWAN
    """,
    formatter_class = RawTextHelpFormatter
    )
parser.add_argument("-sd", metavar='startdate',
    help="start date of time period")
parser.add_argument("-ed", metavar='enddate',
    help="end date of time period")
parser.add_argument("-mod", metavar='model',
    help="wave model")
parser.add_argument("-sat", metavar='satellite',
    help="satellite mission [s3a,s3b,c2,j3,al]")
parser.add_argument("-path", metavar='destination',
    help="destination of collocated data")

args = parser.parse_args()

now = datetime.now()

if args.sd is None:
    sdate = datetime(now.year,now.month,now.day)-timedelta(days=1)
else:
    sdate = datetime(int(args.sd[0:4]),int(args.sd[4:6]),
                int(args.sd[6:8]),int(args.sd[8:10]))
if args.ed is None:
    edate = datetime(now.year,now.month,now.day)-timedelta(hours=1)
else:
    edate = datetime(int(args.ed[0:4]),int(args.ed[4:6]),
                int(args.ed[6:8]),int(args.ed[8:10]))

if args.path is None:
    args.path = home + '/wavyMini/data'

model = args.mod

# retrieve PID
grab_PID()

tmpdate = deepcopy(sdate)

#forecasts = [12, 36, 60, 84, 108, 132, 156, 180, 204, 228]
forecasts = [6,12,24,30,36,42,48]

while tmpdate <= edate:
    print(tmpdate)
    for element in forecasts:
        # settings
        fc_date = deepcopy(tmpdate)
        basetime=model_dict[model]['basetime']
        # get model collocated values
        from ncmod import get_coll_ts
        inpath = (args.path + '/CollocationFiles/'
                + args.sat
                + fc_date.strftime('/%Y/%m/'))
        filename_ts=fc_date.strftime(model
                                    + "_vs_"
                                    + args.sat
                                    + "_coll_ts_lt"
                                    + "{:0>3d}".format(element)
                                    + "h_%Y%m.nc")
        print(inpath)
        print(filename_ts)
        try:
            dtime, sHs, mHs = get_coll_ts(inpath + filename_ts)
            del filename_ts
            # find collocations for given model time step and validate
            from stationmod import matchtime
            time_lst = []
            for dt in dtime:
                time_lst.append((dt-basetime).total_seconds())
            ctime,idx = matchtime(tmpdate, tmpdate, time_lst, basetime, timewin=30)
            if len(idx)==0:
                pass
            else:
                results_dict = {'date_matches':dtime[idx],
                        'model_Hs_matches':mHs[idx],
                        'sat_Hs_matches':sHs[idx]}
                valid_dict=validate(results_dict)
                print(valid_dict)
                # dump to nc-file: validation
                outpath=(args.path + '/ValidationFiles/'
                    + args.sat
                    + fc_date.strftime('/%Y/%m/'))
                os.system('mkdir -p ' + outpath)
                title_stat='validation file'
                filename_stat=fc_date.strftime(model
                                        + "_vs_"
                                        + args.sat
                                        + "_val_ts_lt"
                                        + "{:0>3d}".format(element)
                                        + "h_%Y%m.nc")
                time_dt = fc_date
                dumptonc_stats(outpath,filename_stat,title_stat,basetime,
                          time_dt,valid_dict)
        except Exception as e:
            print(e)
    tmpdate = tmpdate + timedelta(hours=6)


import os, subprocess, pysam, shutil
try:
    import deeptools.bamCoverage
except ValueError:
    import matplotlib
    matplotlib.colormaps.unregister("rocket")
    matplotlib.colormaps.unregister("rocket_r")
    matplotlib.colormaps.unregister("mako")
    matplotlib.colormaps.unregister("mako_r")
    matplotlib.colormaps.unregister("vlag")
    matplotlib.colormaps.unregister("vlag_r")
    matplotlib.colormaps.unregister("icefire")
    matplotlib.colormaps.unregister("icefire_r")
    import deeptools.bamCoverage

### run hicpro
def run_HiC_Pro(filepairs, inpath, outpath, config):
    for filepair in filepairs:
        sample_dir = os.path.commonprefix([os.path.basename(filepair[0]),os.path.basename(filepair[1])])[:-2]
        os.makedirs(os.path.join(inpath,sample_dir), exist_ok=True)
        for file in filepair:
            file = os.path.join(os.getcwd(),file)
            subprocess.check_output(f'ln -s -f {file} {os.path.join(inpath,sample_dir,os.path.basename(file))}', shell=True)
    if os.path.exists(outpath):
        os.rmdir(outpath)
    subprocess.check_output(f'HiC-Pro -i {inpath} -o {outpath} -c {config}', shell=True)

### run macs2
def run_macs2(rep, outpath, adapter, minlen=20, utrim=0, dtrim=0, threads=6, genome_index='/home/ljw/hg19_with_bowtie2_index/hg19', qvalue=0.001):
    if dtrim>0:
        raise Exception("dtrim must be negative (trim reversely from 3' end)")
    for file in rep:
        subprocess.check_output(f'cutadapt -u {utrim} -u {dtrim} -a {adapter} -m {minlen} -j {threads} {file} -o {f"{file}.adp"}', shell=True)
        subprocess.check_output(f'bowtie2 -p {threads} -x {genome_index} -U {f"{file}.adp"} | samtools view -@ {threads} -b | samtools sort -@ {threads} > {f"{file}.adp.bam"}', shell=True)
        pysam.index("-b", "-@", f"{threads}", f"{file}.adp.bam")
        deeptools.bamCoverage.main(["-b", f"{file}.adp.bam", "-o", f"{file}.adp.bam.bw"])
    out_name = os.path.join(os.path.dirname(rep[0]), os.path.commonprefix([os.path.basename(rep[0]),os.path.basename(rep[1])])[:-4])
    pysam.merge("-@", f"{threads}", "-f", "-o", f"{out_name}.adp.bam", f"{rep[0]}.adp.bam", f"{rep[1]}.adp.bam")
    pysam.sort("-@", f"{threads}", "-o", f"{out_name}.adp.sort.bam", f"{out_name}.adp.bam")
    subprocess.check_output(f"mv {out_name}.adp.sort.bam {out_name}.adp.bam", shell=True)
    pysam.index("-b", "-@", f"{threads}", f"{out_name}.adp.bam")
    deeptools.bamCoverage.main(["--numberOfProcessors", f"{threads}", "-b", f"{out_name}.adp.bam", "-o", f"{out_name}.adp.bam.bw"])
    for inputs, on in zip([f"{rep[0]}.adp.bam {rep[1]}.adp.bam", f"{rep[0]}.adp.bam", f"{rep[1]}.adp.bam"],[out_name,f"{out_name}_rep1",f"{out_name}_rep2"]):
        subprocess.check_output(f'macs2 callpeak -t {inputs} -g hs -n {on} -B -q {qvalue} -f BAM --out {outpath} -m 1 50', shell=True)

### run hichipper
def run_hichipper(hichipper_outpath, hicpro_outpath, macs2_outpath, resfrags_file):
    shutil.rmtree(os.path.join(hichipper_outpath,"hicpro_div"))
    commons = os.listdir(os.path.join(hicpro_outpath,"bowtie_results","bwt2"))
    for common in commons:
        os.makedirs(os.path.join(hichipper_outpath,"hicpro_div",common,"bowtie_results","bwt2"), exist_ok=True)
        os.makedirs(os.path.join(hichipper_outpath,"hicpro_div",common,"hic_results","data"), exist_ok=True)
        os.makedirs(os.path.join(hichipper_outpath,"hicpro_div","yamls"), exist_ok=True)
        with open(os.path.join(hichipper_outpath,"hicpro_div","yamls",f"{common}.yaml"), "w") as f:
            f.write(f'peaks:\n - {os.path.join(os.getcwd(),macs2_outpath,common.replace("HiChIP", "ChIP-nexus"))}_peaks.narrowPeak\nresfrags:\n - {os.path.join(os.getcwd(),resfrags_file)}\nhicpro_output:\n - {os.path.join(os.getcwd(),hichipper_outpath,"hicpro_div",common)}\n')
        subprocess.check_output(f'ln -f -s {os.path.join(os.getcwd(),hicpro_outpath,"bowtie_results","bwt2",common)} {os.path.join(hichipper_outpath,"hicpro_div",common,"bowtie_results","bwt2",common)}', shell=True)
        subprocess.check_output(f'ln -f -s {os.path.join(os.getcwd(),hicpro_outpath,"hic_results","data",common)} {os.path.join(hichipper_outpath,"hicpro_div",common,"hic_results","data",common)}', shell=True)

    for common in commons:
        if os.path.exists(os.path.join(hichipper_outpath, common)):
            os.rmdir(os.path.join(hichipper_outpath, common))

    current_path = os.getcwd()
    os.chdir(hichipper_outpath)
    for common in commons:
        subprocess.check_output(f'hichipper --out {common} {os.path.join("hicpro_div","yamls",f"{common}.yaml")}', shell=True)
    os.chdir(current_path)


#Objective: Introduce Matplotlib Python library
from matplotlib import pyplot as plt #Import basic plot functions

x = [5,2,9,4,7] #X axis values
y = [10,5,8,4,2] #Y axis values
hist = [2,2,2,3,54,6,4,3,2,2,24,5,6] #Histogram values

x2 = 10*x
y2 = 5*y

#plt.plot(x,y) #Plot points on graph
#plt.bar(x,y) #Plot points on bar graph
#plt.hist(hist) #Plot values on histogram
#plt.scatter(x,y) #Plot points on scatter graph

fig = plt.figure(figsize = (25,10)) #Create 25 x 10 figure
ax = fig.add_subplot(111) #
ax.bar(x,y) #Create a bar graph in the subplot
ax.bar(y,x,color = 'red') #Append an additional bar graph in the subplot
ax2 = ax.twinx() #Create a twin axes sharing the xaxis
ax2.plot(hist, color = 'green') #Plot aditional data

ax.set_xlabel('Test X-axis') #Set the x-axis label
ax.set_ylabel('Test Y-axis') #Set the y-axis label

ax.set_xticks([1,3,5,7,9,11]) #Sets the x-axis tick marks
ax.set_yticks([1,3,5,7,9,11]) #Sets the y-axis tick marks

ax.set_xticklabels(['a','b','c','d','e','f']) #For the x-axis ticks, set a label
ax.set_yticklabels(['g','h','i','j','k','l']) #For the y-axis ticks, set a label

ax.legend(loc = "Upper right",fontsize = 18)

plt.title("Test graph") #Add title to the graph

plt.axvspan(2,10,0.4,0.5) #Add demarkation regions to the graph
plt.axvline(1,0,1) #Adds demarkation line to the graph

plt.gca().invert_xaxis() #Flip graph on x-axis
plt.savefig("result.png") #Save the picture to a file
plt.show() #Display plot


with open('/root/work/moses-test-corpus/model/phrase-table.ja-en','r') as phrase_table :
    with open('/root/work/nmt/alignment/test/test_pre_word_human_18088_epoch50_lr0.0003_dr0.3_bz16','r') as sentences:
        with open('./phrase-by-phrase.noremovedic.human.en','w') as word_translation:
            phrase_table_lines = phrase_table.readlines()
            phrase=[]
            for phrase_ in phrase_table_lines:
                phrase_ = phrase_.split('|||')
                scores = phrase_[2].split()
                avg_score = float(scores[2]) + float(scores[3])
                avg_score = avg_score / 2
                phrase.append([phrase_[0],phrase_[1],avg_score])
            phrase.sort(key = lambda score: score[2], reverse=True)
            sentences_lines = sentences.readlines()
            count = 0
            for sentence in sentences_lines:
                sentence_line = sentence.split()
                for phrases in phrase:
                    ja_phrase = phrases[0].split()
                    en_phrase = phrases[1].split()
                    if len(ja_phrase) == 1:
                        for j in range(len(sentence_line)):
                            if sentence_line[j] == ja_phrase[0]:
                                sentence_line[j] = en_phrase
                    if len(ja_phrase) == 2:
                        for j in range(len(sentence_line)-1):
                            if sentence_line[j] == ja_phrase[0] and sentence_line[j+1] == ja_phrase[1]:
                                sentence_line[j],sentence_line[j+1] = en_phrase,''
                    if len(ja_phrase) == 3:
                        for j in range(len(sentence_line)-2):
                            if sentence_line[j] == ja_phrase[0] and sentence_line[j+1] == ja_phrase[1] and sentence_line[j+2] == ja_phrase[2]:
                                sentence_line[j],sentence_line[j+1],sentence_line[j+2] = en_phrase,'',''
                count = count+1
                print(count)

                for word in sentence_line:
                    if type(word) is str:
                        word_translation.write(word + ' ')
                    elif type(word) is list:
                        for pp in word:
                            word_translation.write(pp + ' ')
                word_translation.write('\n')

        word_translation.close()
    sentences.close()
phrase_table.close()


from pytube import YouTube
from moviepy.editor import AudioFileClip
import cv2
import os
from PIL import Image
import imagehash
import argparse
import numpy as np
import platform
import subprocess
import zipfile
import webbrowser
import os
import re
from numpy.fft import fftshift, fft2

def download_youtube_video(url, save_path='', audio_only=False):
    print(f"Downloading {'audio' if audio_only else 'video'} from {url} ...")
    yt = YouTube(url)
    if audio_only:
        stream = yt.streams.get_audio_only()
    else:
        stream = yt.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first()
    if not save_path:
        save_path = stream.default_filename

    stream.download(output_path=save_path)
    print(f"{'Audio' if audio_only else 'Video'} downloaded: {stream.default_filename}")

    return os.path.join(save_path, stream.default_filename)

def is_mostly_black_or_white(image, threshold, white_threshold=225):
    """
    Check if the given image is mostly black or white.
    :param image: Image to be checked.
    :param threshold: Threshold below which the image is considered 'black'.
    :param white_threshold: Threshold above which the image is considered 'white'.
    :return: True if the image is mostly black or white; False otherwise.
    """
    # Convert image to grayscale
    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    # Calculate the average brightness of the image
    average_brightness = cv2.mean(gray_image)[0]

    return average_brightness < threshold or average_brightness > white_threshold

def extract_frames(video_path, frame_interval=50, save_path='', black_white_threshold=10, hash_func=imagehash.average_hash, hash_size=8, hash_diff_threshold=10, remove_blur=True, motion_blur_threshold=-0.02):
    if save_path and not os.path.exists(save_path):
        os.makedirs(save_path)

    cap = cv2.VideoCapture(video_path)

    if not cap.isOpened():
        print("Error: Could not open video.")
        return

    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    fps = cap.get(cv2.CAP_PROP_FPS)
    print(f"Total frames: {total_frames}, FPS: {fps}")

    frame_index = 0
    saved_frame_count = 0
    previous_hash = None

    while True:
        ret, frame = cap.read()
        if ret:
            if frame_index % frame_interval == 0:
                pil_image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))

                is_black_or_white = is_mostly_black_or_white(frame, threshold=black_white_threshold)
                current_hash = hash_func(pil_image, hash_size=hash_size)

                is_blurry, blur_score = detect_motion_blur(frame, motion_blur_threshold)

                should_save = True  # Initialize the flag assuming the frame will be saved

                if previous_hash is not None:
                    hash_diff = current_hash - previous_hash
                    if hash_diff < hash_diff_threshold:
                        should_save = False  # If frames are similar, do not save
                        print(f"Skipping frame {frame_index} due to perceived duplication (hash diff: {hash_diff})")

                if is_black_or_white:
                    should_save = False  # If frame is mostly black or white, do not save
                    print(f"Skipping frame {frame_index} because it's mostly black or white")

                if remove_blur and is_blurry:
                    should_save = False
                    print(f"Skipping frame {frame_index} because it's blurry (blur score: {blur_score}))")

                if should_save:
                    save_filename = os.path.join(save_path, f'frame_{frame_index}.jpg')
                    cv2.imwrite(save_filename, frame)
                    saved_frame_count += 1

                previous_hash = current_hash

            frame_index += 1
        else:
            break

    cap.release()
    print(f"Done extracting frames. {saved_frame_count} images are saved in '{save_path}'.")

def open_file_explorer(path):
    """
    Open the file explorer at the specified path.
    """
    if platform.system() == "Windows":
        os.startfile(path)
    elif platform.system() == "Darwin":  # macOS
        subprocess.Popen(["open", path])
    else:  # linux
        subprocess.Popen(["xdg-open", path])

def user_image_confirmation(path):
    """
    Prompt the user to check the images before proceeding.
    """
    confirmation = input("Press Enter to open finder to check the images.")
    open_file_explorer(path)  # This will open the file explorer so you can check the images

    confirmation = input("Have you checked the images and do you want to proceed with posting a training? (y/n): ")
    return confirmation.lower() == "y"

def user_audio_confirmation(path):
    """
    Prompt the user to check the audio before proceeding.
    """
    confirmation = input("Press Enter to check the audio file")
    open_file_explorer(path)

    confirmation = input("Have you checked the audio and do you want to proceed with posting a training? (y/n): ")
    return confirmation.lower() == "y"

def user_model():
    """
    Prompt the user to input the fine tune model name
    """
    does_model_exist = input("Have you already created the model on Replicate? (y/n): ")

    if does_model_exist.lower() == "y":
        model_name = input("Please input the model name (owner/model_name): ")
        return model_name
    else:
        name = input("What do you want to call the model? Pick a short and memorable name. Use lowercase characters and dashes. (eg: sdxl-barbie, musicgen-ye): ")
        webbrowser.open(f"https://replicate.com/create?name={name}")
        input("Once you have created the model (click Create on the webpage that just opened), press Enter to continue.")
        owner = input("What is your Replicate username? ")
        return f"{owner}/{name}"

def zip_directory(folder_path, zip_path):
    """
    Compress a directory (with all files in it) into a zip file.

    :param folder_path: Path of the folder you want to compress.
    :param zip_path: Destination file path, including the filename of the new zip file.
    """
    print(f"Zipping {folder_path} to {zip_path} ...")
    with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
        for root, _, files in os.walk(folder_path):
            for file in files:
                file_path = os.path.join(root, file)
                zipf.write(file_path, os.path.relpath(file_path, folder_path))

    return zip_path

def create_sdxl_training(model, save_dir, caption_prefix="in the style of TOK"):
    try:
        # Please make sure that 'replicate' is installed and available in your system's PATH.
        # The command assumes that "nightmare.zip" is correctly placed and accessible.
        command = [
            "replicate",
            "train",
            "stability-ai/sdxl",
            "--destination",
            model,
            "--web",
            f"input_images=@{save_dir}",
            f"caption_prefix={caption_prefix}",
        ]
        subprocess.run(command, check=True)
    except subprocess.CalledProcessError as e:
        print("Error executing the command:", str(e))
    except FileNotFoundError:
        print("Error: 'replicate' command not found. Is it installed correctly?")
    except Exception as e:
        print(f"An error occurred: {str(e)}")

def create_musicgen_training(model, save_dir, audio_description, drop_vocals=False):
    try:
        # Please make sure that 'replicate' is installed and available in your system's PATH.
        # The command assumes that "your-audio.mp3" is correctly placed and accessible.
        command = [
            "replicate",
            "train",
            "sakemin/musicgen-fine-tuner",
            "--destination",
            model,
            "--web",
            "model=medium",
            f"drop_vocals={drop_vocals}",
            f"one_same_description={audio_description}",
            f"dataset_path=@{save_dir}",
        ]
        subprocess.run(command, check=True)
    except subprocess.CalledProcessError as e:
        print("Error executing the command:", str(e))
    except FileNotFoundError:
        print("Error: 'replicate' command not found. Is it installed correctly?")
    except Exception as e:
        print(f"An error occurred: {str(e)}")

def is_replicate_api_token_set():
    return 'REPLICATE_API_TOKEN' in os.environ

def is_replicate_cli_installed():
    try:
        subprocess.check_output(["replicate", "--version"])
        return True
    except subprocess.CalledProcessError:
        return False
    except FileNotFoundError:
        return False

def slugify(title):
    """
    Slugify a YouTube title.

    :param title: The title to slugify.
    :return: The slugified title.
    """
    return re.sub(r'\W+', '-', title).lower()

def detect_motion_blur(image, motion_blur_threshold):
   # Load the image
   image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    # Scale the image to the range [0, 1]
   image = image / 255.0

   # Apply DCT
   dct = cv2.dct(np.float32(image))

   # Compute the average of the DCT coefficients in the high-frequency region
   avg = np.mean(dct[1:int(dct.shape[0]/2), 1:int(dct.shape[1]/2)])

   avg = avg * 10000 if avg else 0

   # If the average is below a certain threshold, the image is likely blurred
   # After some trial and error, around -0.02 seems to be a good threshold
   if avg < motion_blur_threshold:
       return (True, avg)
   else:
       return (False, avg)

def convert_mp4_to_mp3(mp4_file_path):
    print(f"Converting {mp4_file_path} to MP3 ...")
    audio_clip = AudioFileClip(mp4_file_path)
    mp3_file_path = mp4_file_path.replace('.mp4', '.mp3')
    audio_clip.write_audiofile(mp3_file_path, codec='mp3')
    audio_clip.close()
    os.remove(mp4_file_path)
    return mp3_file_path

def process_audio(audio_file_path):
    audio_file_path = convert_mp4_to_mp3(audio_file_path)

    if user_audio_confirmation(audio_file_path):
        # If the user confirms, proceed with the posting function
        model = user_model()

        print("Please describe the audio, use 2 to 3 comma separated keywords. This could be a band name, genre or something unique. Youâ€™ll use this when prompting your fine-tune.")
        audio_description = input("Describe the audio: ")

        # ask user if they want to drop vocals
        print("MusicGen does not train well with audio that has vocals")
        drop_vocals_input = input("Do you want to automatically drop vocals from your audio? (y/n): ")
        drop_vocals = True if drop_vocals_input.lower() == 'y' else False

        create_musicgen_training(model, audio_file_path, audio_description, drop_vocals)
    else:
        print("Operation cancelled by the user.")

def process_video(video_file_path, interval, caption_prefix):
    # slugify the video title
    video_name = video_file_path.split('/')[-1]
    output_directory = f"./extracted_frames/{slugify(video_name)}"

    extract_frames(video_file_path, frame_interval=interval, save_path=output_directory)

    # After extracting and saving images, ask the user to confirm
    if user_image_confirmation(output_directory):
        # If the user confirms, proceed with the posting function
        model = user_model()

        # Compress the directory with the images
        zip_path = zip_directory(output_directory, output_directory + ".zip")

        create_sdxl_training(model, zip_path, caption_prefix=caption_prefix)
    else:
        print("Operation cancelled by the user.")

def main():
    parser = argparse.ArgumentParser(description='Download a video from YouTube and extract frames or audio')
    parser.add_argument('url', help='URL of the YouTube video')
    parser.add_argument('--interval', help='Interval between frames', default=50, type=int)
    parser.add_argument('--caption_prefix', help='automatically add this to the start of each caption', default="in the style of TOK", type=str)
    parser.add_argument('--audio', help='Download audio only', action='store_true')
    parser.add_argument('--remove_blur', help='remove blurry frames', default=True, action='store_true')
    args = parser.parse_args()

    if not is_replicate_cli_installed():
        input("ðŸš« Replicate CLI is not installed. Please install it before proceeding. Link: https://github.com/replicate/cli. Press any key to open the webpage.")
        webbrowser.open(f"https://github.com/replicate/cli")
    else:
        print("âœ… Replicate CLI is installed. Proceeding...")

    if not is_replicate_api_token_set():
        print("ðŸš« REPLICATE_API_TOKEN is not set. Please set it with `export REPLICATE_API_TOKEN=<your-token>`, then try again.")
        return
    else:
        print("âœ… REPLICATE_API_TOKEN is set. Proceeding...")

    if args.audio:
        print("ðŸŽµ Audio training mode. Proceeding...")
    else:
        print("ðŸŽ¥ Video training mode. Proceeding...")

    video_url = args.url
    interval = args.interval
    caption_prefix = args.caption_prefix

    # Directory where you want to save the downloaded video
    download_directory = './downloaded_audio' if args.audio else './downloaded_videos'

    if video_url.startswith("http"):
        video_file_path = download_youtube_video(video_url, save_path=download_directory, audio_only=args.audio)
    else:
        video_file_path = video_url

    if args.audio:
        process_audio(video_file_path)
    else:
        process_video(video_file_path, interval, caption_prefix)

if __name__ == "__main__":
    main()