import os
import pickle
import shutil
import time
from collections import deque
from os import path

import numpy as np
import DDPG_baseline_v2.baselines.common.tf_util as U
import tensorflow as tf
from mpi4py import MPI
from DDPG_baseline_v2.baselines import logger
from DDPG_baseline_v2.baselines.ddpg.ddpg import DDPG
from DDPG_baseline_v2.baselines.ddpg.memory import load_from_cedric, load_from_geppg

from DDPG_baseline_v2.baselines.ddpg.util import mpi_mean, mpi_std, mpi_sum

def run_agent(env, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic,
    normalize_returns, normalize_observations, critic_l2_reg, actor_lr, critic_lr, action_noise,
    popart, gamma, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_steps, batch_size, memory, study,
    buffer_location, trial_id, data_path, nb_eval_episodes,  tau=0.01, eval_env=None, param_noise_adaption_interval=50,
    gep_memory=None, **kwargs):
    rank = MPI.COMM_WORLD.Get_rank()

    assert (np.abs(env.action_space.low) == env.action_space.high).all()  # we assume symmetric actions.
    max_action = env.action_space.high
    logger.info('scaling actions by {} before executing in env'.format(max_action))
    agent = DDPG(actor, critic, memory, env.observation_space.shape, env.action_space.shape,
                 gamma=gamma, tau=tau, normalize_returns=normalize_returns, normalize_observations=normalize_observations,
                 batch_size=batch_size, action_noise=action_noise, param_noise=param_noise, critic_l2_reg=critic_l2_reg,
                 actor_lr=actor_lr, critic_lr=critic_lr, enable_popart=popart, clip_norm=clip_norm,
                 reward_scale=reward_scale)
    logger.info('Using agent with the following configuration:')
    logger.info(str(agent.__dict__.items()))

    # Set up logging stuff only for a single worker.
    if rank == 0:
        saver = tf.train.Saver()
    else:
        saver = None

    episode = 0
    best_score = -1000 # initialize to negative values to keep track of best score
    eval_episode_rewards_history = deque(maxlen=100)
    episode_rewards_history = deque(maxlen=100)
    saver_best = tf.train.Saver(max_to_keep=1)

    with U.single_threaded_session() as sess:
        # Prepare everything.
        agent.initialize(sess)
        sess.graph.finalize()

        size_buffer = 0
        # load buffer if necessary
        if study == 'GEP_PG' or study == 'GEP_FPG':
            if study ==  'GEP_FPG':
                logger.info('This is a frozen buffer study')
            if gep_memory is not None:
                buffer_gep = load_from_geppg(gep_memory)
                del gep_memory
                logger.info('Load buffer from gep memory')
            else:
                assert buffer_location != '', 'study is ' + study + ', a buffer location should be provided.'
                # fill replay buffer
                buffer_gep = load_from_cedric(filename=buffer_location)
                logger.info('Load buffer from file')

            size_buffer = len(buffer_gep[0])
            for i in range(size_buffer):
                agent.store_transition(np.array(buffer_gep[0][i]), np.array(buffer_gep[1][i]),
                                       buffer_gep[2][i][0], np.array(buffer_gep[3][i]), buffer_gep[4][i])
            logger.info('Buffer of size ' + str(size_buffer) + ' has been loaded')

        agent.reset()
        obs = env.reset()
        if eval_env is not None:
            eval_obs = eval_env.reset()
        done = False
        episode_reward = 0.
        episode_step = 0
        episodes = 0
        t = size_buffer # t statt at size buffer if a buffer is provided, 0 otherwise

        epoch = 0
        start_time = time.time()

        epoch_episode_rewards = []
        epoch_episode_steps = []
        epoch_episode_eval_rewards = []
        epoch_episode_eval_steps = []
        epoch_start_time = time.time()
        epoch_actions = []
        epoch_qs = []
        epoch_episodes = 0

        max_steps = nb_rollout_steps*nb_epoch_cycles*nb_epochs

        for epoch in range(nb_epochs):
            if t>=max_steps:
                break
            for cycle in range(nb_epoch_cycles):
                if t>=max_steps:
                    break
                # Perform rollouts.
                if study != 'GEP_FPG':
                    agent, env, episode_reward, episode_step, epoch_actions, epoch_qs, epoch_episode_rewards, episode_rewards_history, \
                    epoch_episode_steps, epoch_episodes, episodes, t = rollout(nb_rollout_steps, agent, obs, rank, env, max_action,
                                                                               episode_reward, episode_step, epoch_actions,
                                                                               epoch_qs,epoch_episode_rewards, episode_rewards_history,
                                                                               epoch_episode_steps, epoch_episodes, episodes, render, t, epoch
                                                                               )
                # Train.
                agent, epoch_actor_losses, epoch_critic_losses, epoch_adaptive_distances = train(nb_train_steps, memory,
                                                                                batch_size, t,
                                                                                param_noise_adaption_interval, agent
                                                                                )
            # Evaluate.
            agent, eval_episode_rewards, eval_qs, best_score, eval_obs, eval_env, \
            eval_episode_rewards_history = evaluate(agent, eval_obs, max_action, eval_env, render_eval, sess, t, eval_episode_rewards_history,
                                                     best_score, data_path, saver_best, nb_eval_episodes, study, max_steps, size_buffer
                                                    )

            # update decreasing_ou if necessary
            try:
                action_noise.adapt()
                print(action_noise.sigma)
            except:
                pass

            # Log stats.
            epoch_train_duration = time.time() - epoch_start_time
            duration = time.time() - start_time
            stats = agent.get_stats()
            combined_stats = {}
            for key in sorted(stats.keys()):
                combined_stats[key] = mpi_mean(stats[key])

            # Added statistics

            # Rollout statistics.
            combined_stats['rollout/return'] = mpi_mean(epoch_episode_rewards)
            combined_stats['rollout/return_history'] = mpi_mean(np.mean(episode_rewards_history))
            combined_stats['rollout/episode_steps'] = mpi_mean(epoch_episode_steps)
            combined_stats['rollout/episodes'] = mpi_sum(epoch_episodes)
            combined_stats['rollout/actions_mean'] = mpi_mean(epoch_actions)
            combined_stats['rollout/actions_std'] = mpi_std(epoch_actions)
            combined_stats['rollout/Q_mean'] = mpi_mean(epoch_qs)

            # Train statistics.
            try:
                combined_stats['train/loss_actor'] = mpi_mean(epoch_actor_losses)
                combined_stats['train/loss_critic'] = mpi_mean(epoch_critic_losses)
                combined_stats['train/param_noise_distance'] = mpi_mean(epoch_adaptive_distances)
            except:
                pass

            # Evaluation statistics.
            if eval_env is not None:
                combined_stats['eval/return'] = mpi_mean(eval_episode_rewards)
                combined_stats['eval/return_history'] = mpi_mean(np.mean(eval_episode_rewards_history))
                combined_stats['eval/Q'] = mpi_mean(eval_qs)
                combined_stats['eval/episodes'] = mpi_mean(len(eval_episode_rewards))

            # Total statistics.
            combined_stats['total/duration'] = mpi_mean(duration)
            combined_stats['total/steps_per_second'] = mpi_mean(float(t) / float(duration))
            combined_stats['total/episodes'] = mpi_mean(episodes)
            combined_stats['total/epochs'] = epoch + 1
            combined_stats['total/steps'] = t

            for key in sorted(combined_stats.keys()):
                logger.record_tabular(key, combined_stats[key])
            logger.dump_tabular()
            logger.info('')
            logdir = logger.get_dir()
            if rank == 0 and logdir:
                if hasattr(env, 'get_state'):
                    with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as f:
                        pickle.dump(env.get_state(), f)
                if eval_env and hasattr(eval_env, 'get_state'):
                    with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as f:
                        pickle.dump(eval_env.get_state(), f)

def rollout(nb_rollout_steps, agent, obs, rank, env, max_action, episode_reward,
            episode_step, epoch_actions, epoch_qs,epoch_episode_rewards, episode_rewards_history,
            epoch_episode_steps, epoch_episodes, episodes, render, t, epoch):

    for t_rollout in range(nb_rollout_steps):
        # Predict next action.
        action, q = agent.pi(obs, apply_noise=True, compute_Q=True)
        assert action.shape == env.action_space.shape

        # Execute next action.
        if rank == 0 and render:
            env.render()
        assert max_action.shape == action.shape
        new_obs, r, done, info = env.step(max_action * action)  # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
        t += 1
        if rank == 0 and render:
            env.render()
        episode_reward += r
        episode_step += 1
        #print ("rollout episode_step : ", episode_step, " t:", t)

        # Book-keeping.
        epoch_actions.append(action)
        epoch_qs.append(q)
        agent.store_transition(obs, action, r, new_obs, done)
        obs = new_obs

        if done:
            # Episode done.
            epoch_episode_rewards.append(episode_reward)
            episode_rewards_history.append(episode_reward)
            epoch_episode_steps.append(episode_step)
            episode_reward = 0.
            episode_step = 0
            epoch_episodes += 1
            episodes += 1

            agent.reset()
            obs = env.reset()

    return agent, env, episode_reward, episode_step, epoch_actions,epoch_qs,epoch_episode_rewards, episode_rewards_history, \
            epoch_episode_steps, epoch_episodes, episodes, t

def train(nb_train_steps, memory, batch_size, t, param_noise_adaption_interval, agent):

    epoch_actor_losses = []
    epoch_critic_losses = []
    epoch_adaptive_distances = []
    for t_train in range(nb_train_steps):
        # Adapt param noise, if necessary.
        if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0:
            distance = agent.adapt_param_noise()
            epoch_adaptive_distances.append(distance)

        cl, al = agent.train()
        epoch_critic_losses.append(cl)
        epoch_actor_losses.append(al)
        agent.update_target_net()
    return agent, epoch_actor_losses, epoch_critic_losses, epoch_adaptive_distances

def evaluate(agent, eval_obs, max_action, eval_env, render_eval, sess, t, eval_episode_rewards_history,
              best_score, data_path, saver_best, nb_eval_episodes, study, max_steps, size_buffer):

    eval_episode_rewards = []
    eval_qs = []
    if eval_env is not None:
        eval_episode_reward = 0.
        for i in range(nb_eval_episodes): # 5 rollouts
            while True:
                eval_action, eval_q = agent.pi(eval_obs, apply_noise=False, compute_Q=True)
                eval_obs, eval_r, eval_done, eval_info = eval_env.step(max_action * eval_action)  # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
                if render_eval:
                    eval_env.render()
                eval_episode_reward += eval_r
                eval_qs.append(eval_q)
               # print("evaluate eval_done : ", eval_done, " eval_episode_reward:", eval_episode_reward)
                if eval_done:
                    eval_obs = eval_env.reset()
                    eval_episode_rewards.append(eval_episode_reward)
                    eval_episode_rewards_history.append(eval_episode_reward)
                    eval_episode_reward = 0.
                    break
        if mpi_mean(eval_episode_rewards) > best_score:
            best_score = mpi_mean(eval_episode_rewards)
            if not os.path.exists(data_path+'tf_save/'):
                os.mkdir(data_path+'tf_save/')
                os.mkdir(data_path+'tf_save/'+'best1_5M/') # optional, to save GEP data at 1.5M
            for file in os.listdir(data_path+'tf_save/'):
                if path.isfile(data_path+'tf_save/'+file): # optional, to save GEP data at 1.5M
                    os.remove(data_path+'tf_save/'+file)
            saver_best.save(sess, data_path+'tf_save/'+ 'best_actor_step'+str(t)+'_score'+str(int(best_score)))

    return agent, eval_episode_rewards, eval_qs, best_score, eval_obs, eval_env, \
            eval_episode_rewards_history


"""Module with helper function to find libraries, load behaviours, ..."""
from __future__ import nested_scopes
from jython_compat import *

import os
import sys
import time

# get main configuration
import config

# to identify behaviours
from behaviour import Behaviour

# private constants
_behaviour_classes = (Behaviour, )

# counters
_agent_id = 0

class World:
    """The World class that is used to communicate with the world
    initialisation script.

    Upon running the world initialisation script, using the L{run_world_script},
    an instance of this class, named 'world' is given to the script. The
    script can use this instance to gather information on how the world is to
    be initialised, and can return the world object and other information to
    the instance that calls this script.
    """
    def __init__(self, library, world_args = None, agents_init = None):
        """Initialises an instance with the neccessary information.

        @param library: name of the behaviour library that is to be used.
        @type library: string
        @param world_args: arguments to be given to the world initialisation
            script.
        @type world_args: string
        @param agents_init: structure containing information to initialise
            the agents.
        @type agents_init: as returned by
            L{POSH.agentinitparser.parse_agent_init_file}
        """
        self._library = library
        self._args = world_args
        if agents_init == None:
            self._agents_init = []
        else:
            self._agents_init = agents_init
        self._creates_agents = False
        self._world = None

    def args(self):
        """Returns the arguments for customised world initialisation.

        If no arguments are given, None is returned.

        @return: arguments for world initialisation.
        @rtype: string or C{None}
        """
        return self._args

    def set(self, x):
        """Sets the world object for use when initialising the agents.

        The world object given to this method is given to the agents upon
        initialisation.

        @param x: The world object
        @type x: any
        """
        self._world = x

    def library(self):
        """Returns the behaviour library name that the agents are to use.

        @return: name of the library to use
        @rtype: string
        """
        return self._library

    def agentsInit(self):
        """Returns the agents initialisation structure.

        @return: agents initialisation structure.
        @rtype: as returned by L{POSH.agentinitparser.parse_agent_init_file}
        """
        return self._agents_init

    def createsAgents(self):
        """Specifies that the agents are created an run by the world
        initialisation script.

        By default, the world initialisation script is only responsible for
        setting up the world, and eventually returning the world object to
        initialise the agents with. Calling this method from the world
        initialisation script indicates that both creation and running the
        agents is performed by the world initialisation script. For this
        purpose, the script can use L{library()} and L{agentsInit()}.
        """
        self._creates_agents = True

# to provide current_time()
if sys.platform == "win32":
    # On Windows, the best timer is time.clock()
    current_time = time.clock
else:
    # On most other platforms the best timer is time.time()
    current_time = time.time

def get_root_path():
    """Returns the root path that POSH is installed in. Assumes that this
    module resides in the root path.

    @return: Root path
    @rtype: string
    """
    return os.path.split(config.__file__)[0]

def _get_library_path(library = None):
    """Returns the path to the behaviour library. If no library is given,
    the the base path of the library is returned.

    @param library: The library to return the path for
    @type library: string
    @return: The base library path or the path to the given library
    @rtype: string
    """
    if library:
        return os.path.join(get_root_path(), config.LIBRARYPATH, library)
    else:
        return os.path.join(get_root_path(), config.LIBRARYPATH)

def _get_plans_path(library):
    """Returns the path to the plans of the given behaviour library.

    @param library: The library to return the path for
    @type library: string
    @return: The path to the plans
    @rtype: string
    """
    return os.path.join(_get_library_path(library), config.PLANSPATH)

def _is_behaviour_class(c):
    """Returns if the given class object is a subclass of a behaviour class.

    @param c: A class object
    @type c: class
    @return: If the given class is a subclass of L{POSH.Behaviour}
    @rtype: bool
    """
    for behaviour_class in _behaviour_classes:
        try:
            if issubclass(c, behaviour_class) and c != behaviour_class:
                return True
        except TypeError:
            # will be raise whenever c is not an object of type classobj
            pass
    return False

def get_plans(library):
    """Returns a list of available plans for the given behaviour library.

    The plans need to be located at library/PLANS and end in PLANENDING.

    @param library: The library to return the list of plans for
    @type library: string
    @return: A list of plans without file ending
    @rtype: sequence of strings
    """
    planpath = _get_plans_path(library)
    try:
        dirlist = os.listdir(planpath)
    except OSError:
        return []
    planfilter = lambda f: \
        (not f[:1] == '.') and \
        f[-len(config.PLANENDING):].lower() == config.PLANENDING.lower() and \
        os.path.isfile(os.path.join(planpath, f))
    planfiles = filter(planfilter, dirlist)
    # remove the endings of the files
    return [filename[:-len(config.PLANENDING)] for filename in planfiles]

def get_plan_file(library, plan):
    """Returns the plan file name for the given library and plan

    @param library: The library that the plan is from
    @type library: string
    @param plan: The name of the plan (without the .lap ending)
    @type plan: string
    @return: The filename with full path of the plan
    @rtype: string
    """
    return os.path.join(_get_plans_path(library),
                        "%s%s" % (plan, config.PLANENDING))

def get_library_file(library, file):
    """Returns the file name for the given library (not for plans, see above)

    @param library: The library that the file is in
    @type library: string
    @param plan: The name of the file (including any needed ending)
    @type plan: string
    @return: The filename with full path
    @rtype: string
    """
    return os.path.join(_get_library_path(library),
                        "%s" % (file))

def is_plan(library, plan):
    """Returns if the given plan of the given library exists.

    This method only checks if the plan file exists, not if its syntax is
    correct.

    @param library: The library that the plan is from
    @type library: string
    @param plan: The name of the plan (without the .lap ending)
    @type plan: string
    @return: If the plan exists (i.e. is a file)
    @rtype: bool
    """
    return os.path.isfile(get_plan_file(library, plan))

def get_libraries():
    """Returns a list of available behaviour libraries.

    The function assumes that any directory in the libraries path that
    does not start with an '.' is a behaviour library.

    @return: List of libraries
    @rtype: sequence of strings
    """
    library_path = _get_library_path()
    dirlist = os.listdir(library_path)
    libraryfilter = lambda d: \
        (not d[:1] == '.') and \
        os.path.isdir(os.path.join(library_path, d))
    return filter(libraryfilter, dirlist)

def is_library(library):
    """Returns if the given library is a valid behaviour library.

    This is the case if the following requirements are met:
        - there is a directory with the given name in LIBRARYPATH
        - the directory can be loaded as a module, which requires it to at least
          contain a __init__.py file

    @param library: Name of the library to check
    @type library: string
    @return: If the library is a valid library
    @rtype: bool
    """
    try:
        lib_module = __import__("%s.%s" % (config.LIBRARYPATH, library))
    except ImportError:
        return False
    return True

def get_behaviours(library, log = None):
    """Returns a sequence of classes, containing all behaviour classes that
    are available in a particular library.

    The method searches the behvaiour subclasses by attempting to import
    all file in the library ending in .py, except for the WORLDSCRIPT, and
    search through all classes they contain to see if they are derived from
    any behaviour class.

    If a log object is given, then logging output at the debug level is
    produced.

    @param library: Name of the library to find the classes for
    @type library: string
    @param log: A log object
    @type log: as from the Python logging module
    @return: The list of classes
    """
    # get list of python files is behaviour library
    if log != None:
        log.debug("Scanning library '%s' for behaviour classes" % library)
    librarypath = _get_library_path(library)
    try:
        dirlist = os.listdir(librarypath)
    except OSError:
        return []
    pyfilter = lambda f: \
        (not f[:1] == '.') and \
        f[-3:].lower() == '.py' and \
        os.path.isfile(os.path.join(librarypath, f))
    pylist = filter(pyfilter, dirlist)
    # try to load the library modules to collect the behaviour classes
    behaviour_classes = []
    for pyfile in pylist:
        if pyfile == config.WORLDSCRIPT:
            continue
        modname = pyfile[:-3]
        # try to load the module
        try:
            behaviour_mod = __import__('%s.%s.%s' % \
                                       (config.LIBRARYPATH, library, modname))
        except Exception:
            # ignore any non-importable python file
            if log != None:
                log.debug("Loading the file '%s' failed" % pyfile)
            continue
        # workaround to get the real module rather than its top-level parent
        # (see __import__ documentation)
        behaviour_mod = getattr(getattr(behaviour_mod, library), modname)
        # add all behaviour class objects in module to list
        mod_objs = [getattr(behaviour_mod, objname) \
                    for objname in dir(behaviour_mod)]
        found_classes = filter(_is_behaviour_class, mod_objs)
        if log != None:
            class_names = ', '.join([c.__name__ for c in found_classes])
            log.debug("Found %d behaviour class(es) in '%s': %s" % \
                      (len(found_classes), pyfile, class_names))
        behaviour_classes += found_classes
    # remove double entries
    behaviour_set = {}
    for behaviour_class in behaviour_classes:
        behaviour_set[behaviour_class] = None
    return behaviour_set.keys()

def default_agent_init(library):
    """Returns the default agent initialisation file filename for the given
    library.

    If no such file exists, and emtpy string is returned.

    @param library: Name of the library
    @type library: string
    @return: Agent initialisation file filename, or '' if not found
    @rtype: string
    """
    agent_init = os.path.join(_get_library_path(library), config.AGENTINIT)
    if os.path.isfile(agent_init):
        return agent_init
    else:
        return ''

def unique_agent_id():
    """Returns a unique agent id string of the form 'Axx', where xx is
    an increasing number, starting from 00.

    If more than 99 agents are created, the string length adjusts to the
    length of the number.

    @return: Unique agent id
    @rtype: string
    """
    global _agent_id
    _agent_id += 1
    return 'A%02d' % (_agent_id - 1)

def default_world_script(library):
    """Returns the default world initialisation script for the given library.

    If no script was found, then an empty string is returned.

    @param library: Name of the library
    @type library: string
    @return: World initialisatoin script filename, of '' if not found
    @rtype: string
    """
    world_script = os.path.join(_get_library_path(library), config.WORLDSCRIPT)
    if os.path.isfile(world_script):
        return world_script
    else:
        return ''

def run_world_script(script_file, library,
                     world_args = None, agents_init = None):
    """Runs the given file to initialise the world and returns the world object
    and if the world initialisation script creates and runs the agents
    itself.

    This method creates an instance of the class L{World} and makes it
    accessible to the world initialisation script under the instance name
    'world'. Using this instance, the world intialisation script can
    cimmunicate with this function.

    @param script_file: The filename of the script to run
    @type script_file: string
    @param library: name of the behaviour library to use by agents
    @type library: string
    @param world_args: arguments given to the world initialisation script
    @type world_args: string
    @param agents_init: agent initialisation information structure
    @type agents_init: as returned by
        L{POSH.agentinitparser.parse_agent_init_file}
    @return: tuple (world object, if script created and ran agents)
    @rtype: (any, bool)
    @raise IOError: If it cannot file the script
    @raise Exception: If the script causes an exception
    """
    w = World(library, world_args, agents_init)
    # world instance is given as local variable
    # note that if the global and local variables (when calling execfile)
    # refer to a different object, then some strange things happen as soon as
    # the script starts importing other modules (don't know why). So both
    # arguments have to refer to the same object!
    variables = {'world' : w}
    execfile(script_file, variables, variables)
    return (w._world, w._creates_agents)

#There must be a better way (& place) to do this...
#Note, because jython (or maybe mason.jar) is currently 1.4, we need to be sure to compile our classes in 1.4 too
def compile_mason_java():
      import os
      ext1='.java'
      ext2='.class'
      dir = os.path.join(get_root_path(), config.MASONPATH)
      dir=get_root_path()+'/platform_files/MASON/'
      java_src=filter((lambda str:str!="__init__"),map((lambda str:str[:len(str)-len(ext1)]),filter((lambda str: str.endswith(ext1)),os.listdir(dir))))
      classes=filter((lambda str:str!="__init__$py"),map((lambda str:str[:len(str)-len(ext2)]),filter((lambda str:str.endswith(ext2)),os.listdir(dir))))

      if java_src!=classes:
            cmd = 'javac -target 1.4 -source 1.4 -cp %smason.jar %s*.java' % (dir,dir)
            os.system(cmd)


import xml.etree.ElementTree as ET
import os
def main(file_path):
    file_path = os.path.expanduser(file_path)
    tree = ET.parse(file_path)
    root = tree.getroot()
    folder, filename, path, source, size, segmented = root[:6]
    # print(root[0])
    print('folder, filename, path, source, size, segmented = ', folder, filename, path, source, size, segmented)
    print()
    print('folder.text, filename.text, path.text, source.text, size.text , segmented.text = ', folder.text, filename.text, path.text, source.text, size.text , segmented.text )
    # print(root['folder'])

    for ob in root.findall('object'):
        name, bndbox = ob[0], ob[4]
        print('name.text, bndbox', name.text, bndbox)
        location = xmin, ymin, xmax, ymax = [int(e.text) for e in bndbox[:]]
        print('location: ', location)

if __name__=='__main__':
    import argparse
    ap = argparse.ArgumentParser()
    ap.add_argument("--file_path", help="file_path")
    args= vars(ap.parse_args())
    main(args["file_path"])


import subprocess

import requests
from flask import Flask
from flask import render_template

app = Flask(__name__)
url = 'https://api.github.com/repos/awesome-jobs/jobs/issues'
app.debug = True

@app.route("/")
def hello():
    return "Hello World!"

@app.route("/about")
def about():
    return """<title>About
</title>
This is about page

"""

def get_processes():
    processes = subprocess.check_output('ps xau'.split())
    return processes.splitlines()

@app.route("/processes")
def processes():
    ps = get_processes()
    return render_template('ps.html', processes=ps)

@app.route('/jobs')
def jobs():
    jobs = []
    res = requests.get(url)
    for j in res.json():
        jobs.append(j['title'])
    return ("
" +
            ''.join(['
' + j['title'] + '
' for j in res.json()]) +
            "
")

if __name__ == "__main__":
    app.run()


# -*- coding:utf8 -*-
__author__ = 'dropu'

import urllib
import time
import json

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from youku.items import YoukuItem
from builtin.mysql import MySQLPool

class YoukuSpider(CrawlSpider):

    name = 'youku'
    allowed_domains = ['soku.com']
    start_urls = [
        'http://www.soku.com/search_video/q_%E9%83%AD%E5%BE%B7%E7%BA%B2%E7%9B%B8%E5%A3%B0_orderby_2_lengthtype_4_hd_1_page_1_pagesize_20_limitdate_0?sfilter=1&noqc=',
    ]

    rules = (
        Rule(SgmlLinkExtractor(allow=('/search_video/q_%E9%83%AD%E5%BE%B7%E7%BA%B2%E7%9B%B8%E5%A3%B0_orderby_2_lengthtype_4_hd_1_page_(\d+)_pagesize_20_limitdate_0\?sfilter\=1\&noqc\=', )), callback='parse_video', follow=True),
    )

    def parse_video(self, response):
        self.mysql = MySQLPool()
        sel = Selector(response)
        videoList = sel.xpath('//div[@class="sk-vlist"]/div[@class="v"]')

        item = YoukuItem()
        videoArray = list()
        for video in videoList:
            videoDict = dict()
            vid = video.xpath('.//div[@class="v-link"]/a/@_log_vid').extract()
            videoInfo = self.requestUrl(vid[0])
            if videoInfo is not False:
                videoInfo = videoInfo[0]
                videoDict['title'] = videoInfo['title']
                videoDict['seo_title'] = videoInfo['title']
                videoDict['thumb'] = videoInfo['urlpic']
                videoDict['length'] = video.xpath('.//span[@class="v-time"]/text()').extract()
                videoDict['tags'] = videoInfo['tags']
                videoDict['actor'] = ''
                videoDict['hd'] = 1
                videoDict['category'] = 1
                videoDict['vid'] = vid
                videoDict['url'] = videoInfo['swf']
                videoDict['crawl'] = 1
                videoDict['status'] = 1
                videoDict['showTime'] = video.xpath('.//span[@class="pub"]/text()').extract()
                videoDict['createTime'] = int(time.time())
                exist = self.checkVideoExist(vid)
                if exist is False:
                    videoArray.append(videoDict)
        item['videoData'] = videoArray
        return item

    # 根据vid查询是否存在此视频
    def checkVideoExist(self, vid):
        result = self.mysql.getOne("SELECT `id` FROM `video` WHERE `vid` = %s", (vid))
        if result is False:
            return False
        else:
            return result['id']

    def parseTime(self, timeStr):
        tempTime = list()
        tempTime.append('小时')
        tempTime.append('天')
        tempTime.append('月')
        tempTime.append('年')

    def requestUrl(self, vid):
        url = 'http://v.youku.com/v_show/id_ITEMID.html'
        url = url.replace('ITEMID', vid)
        request_url = 'http://hi.baidu.com/pub/data/video?url=DETAILURL&asyn=1&_1393068790846=1&qing_request_source=new_request'
        # time.sleep(2)
        request_url = request_url.replace('DETAILURL', url)
        video_info = urllib.urlopen(request_url).read()
        video_info = json.loads(video_info)
        if video_info['errorNo'] == '0':
            return video_info['data']
        else:
            return False


"""
This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start
when invoked.

When ran in development mode (LEMUR_CONFIG=<location of development configuration file. To run both the celery
beat scheduler and a worker simultaneously, and to have jobs kick off starting at the next minute, run the following
command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B

"""
import copy
import datetime
import sys
from datetime import timezone

from celery import Celery
from flask import current_app

from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.users import service as user_service

flask_app = create_app()

def make_celery(app):
    celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
                    broker=app.config['CELERY_BROKER_URL'])
    celery.conf.update(app.config)
    TaskBase = celery.Task

    class ContextTask(TaskBase):
        abstract = True

        def __call__(self, *args, **kwargs):
            with app.app_context():
                return TaskBase.__call__(self, *args, **kwargs)

    celery.Task = ContextTask
    return celery

celery = make_celery(flask_app)

@celery.task()
def fetch_acme_cert(id):
    """
    Attempt to get the full certificate for the pending certificate listed.

    Args:
        id: an id of a PendingCertificate
    """
    log_data = {
        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
    }
    pending_certs = pending_certificate_service.get_pending_certs([id])
    user = user_service.get_by_username('lemur')
    new = 0
    failed = 0
    wrong_issuer = 0
    acme_certs = []

    # We only care about certs using the acme-issuer plugin
    for cert in pending_certs:
        cert_authority = get_authority(cert.authority_id)
        if cert_authority.plugin_name == 'acme-issuer':
            acme_certs.append(cert)
        else:
            wrong_issuer += 1

    authority = plugins.get("acme-issuer")
    resolved_certs = authority.get_ordered_certificates(acme_certs)

    for cert in resolved_certs:
        real_cert = cert.get("cert")
        # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
        pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)

        if real_cert:
            # If a real certificate was returned from issuer, then create it in Lemur and delete
            # the pending certificate
            pending_certificate_service.create_certificate(pending_cert, real_cert, user)
            pending_certificate_service.delete_by_id(pending_cert.id)
            # add metrics to metrics extension
            new += 1
        else:
            failed += 1
            error_log = copy.deepcopy(log_data)
            error_log["message"] = "Pending certificate creation failure"
            error_log["pending_cert_id"] = pending_cert.id
            error_log["last_error"] = cert.get("last_error")
            error_log["cn"] = pending_cert.cn

            if pending_cert.number_attempts > 4:
                error_log["message"] = "Deleting pending certificate"
                send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
                pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
            else:
                pending_certificate_service.increment_attempt(pending_cert)
                pending_certificate_service.update(
                    cert.get("pending_cert").id,
                    status=str(cert.get("last_error"))
                )
                # Add failed pending cert task back to queue
                fetch_acme_cert.delay(id)
            current_app.logger.error(error_log)
    log_data["message"] = "Complete"
    log_data["new"] = new
    log_data["failed"] = failed
    log_data["wrong_issuer"] = wrong_issuer
    current_app.logger.debug(log_data)
    print(
        "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
            new=new,
            failed=failed,
            wrong_issuer=wrong_issuer
        )
    )

@celery.task()
def fetch_all_pending_acme_certs():
    """Instantiate celery workers to resolve all pending Acme certificates"""
    pending_certs = pending_certificate_service.get_pending_certs('all')

    # We only care about certs using the acme-issuer plugin
    for cert in pending_certs:
        cert_authority = get_authority(cert.authority_id)
        if cert_authority.plugin_name == 'acme-issuer':
            if cert.last_updated == cert.date_created or datetime.datetime.now(
                    timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3):
                fetch_acme_cert.delay(cert.id)


#!/usr/bin/env python

from gna.expression.expression_v01 import Expression_v01 as Expression, ExpressionContext_v01 as ExpressionContext
from gna.bundle import execute_bundles
from load import ROOT as R
from gna.configurator import NestedDict, uncertaindict
R.GNAObject

from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument( '--dot', help='write graphviz output' )
args = parser.parse_args()

#
# Indices and their variants
# Each line has a syntax: 'short name', 'long name', 'index variants'
#
indices = [
    ('n', 'num',   ['1', '2']),
    ('a', 'alph',  ['a', 'b']),
    ('z', 'zyx',   ['X', 'Y']),
    ('b', 'bkg',   ['b1', 'b2'])
    ]

#
# Lib is a dictionary, that tells how the intermediate expressions should be named
#
#
lib = dict(
)

# The expression is a forumla
expr = [
        'res=vara * ta() * tb()'
        ]
# In order to parse a formula the Expression needs to know how to iterate over indices
a = Expression(expr, indices=indices)

# Print the expression formula before and after preprocessing
print(a.expressions_raw)
print(a.expressions)

# Parse the formula. The following line will execute the expression as python code and create all the objects, needed for its functioning.
a.parse()
# Assign the names for all the intermediate products, sums, etc.
a.guessname(lib, save=True)
# Print the tree to the terminal
a.tree.dump(True)

print()
# Provide the configuration. The configuration tells the Expression on how to build each variable/output.
cfg = NestedDict(
        ta = NestedDict(
            bundle = dict(name='dummy', version='v01'),
            name = 'ta',
            input = False,
            size = 10,
            debug = False
            ),
        tb = NestedDict(
            bundle = dict(name='dummy', version='v01'),
            name = 'tb',
            input = False,
            size = 10,
            debug = False
            ),
        va = NestedDict(
            bundle = dict(name='dummyvar', version='v01'),
            variables = uncertaindict([('vara', (2, 0.1))], mode='percent'),
            ),
        vb = NestedDict(
            bundle = dict(name='dummyvar', version='v01'),
            variables = uncertaindict([('varb', (3, 0.1))], mode='percent'),
            ),
        )
# Build the expression for given configuration. The output is a context, that contains the inputs and outputs.
context = ExpressionContext( cfg )
a.build(context)

from gna.bindings import OutputDescriptor
# Print inputs
print('Inputs')
print(context.inputs)

# Print outputs
print()
print('Outputs')
print(context.outputs)

if args.dot:
    from gna.graphviz import GNADot, savegraph

    savegraph(next(context.outputs.ta.values(nested=True)), args.dot, rankdir='LR')


import random
from lxml import etree
import train.mainfun
from selenium import webdriver
from time import sleep

from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.webdriver import ChromeOptions

def scroll(driver):
    driver.execute_script("""
        (function () {
            var y = document.body.scrollTop;
            var step = 200;
            window.scroll(0, y);
            function f() {
                if (y < document.body.scrollHeight) {
                    y += step;
                    window.scroll(0, y);
                    setTimeout(f, 50);
                }
                else {
                    window.scroll(0, y);
                    document.title += "scroll-done";
                }
            }
            setTimeout(f, 1000);
        })();
        """)

def get_main_pic(url, name, pathname):
    train.mainfun.mkdir(pathname)

    chrome_options = Options()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--disable-gpu')
    chrome_options = chrome_options
    option = ChromeOptions()
    option.add_experimental_option('excludeSwitches', ['enable-automation'])
    options = option

    browser = webdriver.Chrome(options=option, chrome_options=chrome_options)
    # 输入动漫之家爬取漫画主页(用于·国漫爬取)
    browser.get(url)

    sleep(1)
    browser.find_element_by_xpath('//div[@class="login_tip out"]').click()
    sleep(0.1)
    browser.find_element_by_class_name("red_reset").click()
    sleep(0.1)
    browser.find_element_by_id("mode_2").click()
    sleep(0.1)
    browser.find_element_by_xpath('//div[@class="bar_open_btn"]/a[@class="qd cus"]').click()
    sleep(1)
    scroll(browser)
    # pic_list = WebDriverWait(browser, 0.5).until(
    #     EC.presence_of_all_elements_located((By.XPATH, '//div[@class="comic_wraCon autoHeight"]/a/img'))
    # )
    pic_list = browser.find_elements_by_xpath('//div[@class="comic_wraCon autoHeight"]/a/img')
    # i用于计数
    i = 0
    for pic in pic_list:
        pic_url = pic.get_attribute('src')
        #print(pic_url)
        pic_name = str(i) + '.jpg'
        train.mainfun.get_pic(pic_url, pathname, pic_name)
        i = i + 1
    print(name + "爬取成功")
    browser.close()

# 爬取动漫之家漫画
if __name__ == '__main__':
    # 输入动漫首页
    url = "https://www.dmzj1.com/info/lingzhushimedewusuoweila.html"

    # 获取网址信息
    res = train.mainfun.get_respense(url)
    tree = etree.HTML(res.text)
    # 通过xpath解析获得信息

    # 创建文件夹
    title_name = tree.xpath(
        '//div[@class="comic_deCon"]/h1/a/text()')[0]
    filename = "C:/Users/violet/Desktop/comic/" + title_name
    train.mainfun.mkdir(filename)
    # print(title_name)

    news_list = tree.xpath(
        '//div[@class="tab-content zj_list_con autoHeight"]/ul[@class="list_con_li autoHeight"]/li/a')
    # print(news_list)

    for li in news_list:
        href = li.xpath('./@href')[0]
        name = li.xpath('./span/text()')[0]
        path_name = filename + '/' + name
        get_main_pic(href, name, path_name)
        sleep(0.5)
        # print(href + "  " + name)


import csv
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.linear_model import Lasso
import plotly.graph_objects as go

# since the feature of yellow is similiar to green taxi, using the green as representive
# use the preprocessed dataset to run this python file
newdf = pd.DataFrame()
count = 4
while count < 7:
    # thru three monthes data, break huge data into chunks for reserving memory
    for chunk in pd.read_csv("dataset/g0"+str(count)+".csv", chunksize=10000):

        # keep relevant featrues for pairwise analysis
        pool = chunk[['RateCodeID', 'Pickup_longitude', 'Pickup_latitude',
       'Dropoff_longitude', 'Dropoff_latitude', 'Passenger_count',
       'Trip_distance', 'Extra', 'MTA_tax', 'Tip_amount',
       'Tolls_amount', 'improvement_surcharge', 'Total_amount', 'Payment_type',
       'Trip_type ']].copy()

        # sampling 20% from the data each chunk to save memory
        sampling = pool.sample()
        newdf = newdf.append(sampling)

    count += 1

# since looking for ppl willing to pay high tips, tip is the goal we need to predict
X = newdf.drop('Tip_amount', axis=1)
y= newdf['Tip_amount']

# use lasso to help the regression more precise
lasso = Lasso()
lasso.fit(X, y)

# perform feature selection visualization
kept_cols = [feature for feature, weight in zip(X.columns.values, lasso.coef_) if weight != 0]
figt = go.Figure(
         go.Waterfall(name= "Lasso Coefficients",
                      orientation= "h",
                      y = X.columns.values,
                      x = lasso.coef_))

figt.update_layout(title = "Coefficients of Lasso Regression Model")

figt.show()


class Solution:
    def iterativer(self, root):
        stack, res = [], []
        while root or stack:
            if root:
                stack.append(root)
                root = root.left
            else:
                root = stack.pop()
                res.append(root.val)
                root = root.right
        return res

    def recursive(self, root, res):
        if not root: return
        self.recursive_in_order(root.left, res)
        res.append(root.val)
        self.recursive_in_order(root.right, res)

    def inorderTraversal(self, root):
        res = []
        self.iterative_inorder(root, res)
        return res

def inorder(root):
    cur, pre, res = root, None, []
    while cur:
        if not cur.left:
            res.append(cur.val)
            cur = cur.right
        else:
            pre = cur.left
            while pre.right and pre.right != cur:
                pre = pre.right
            if not pre.right:
                pre.right = cur
                cur = cur.left
            else:
                pre.right = None
                res.append(cur.val)
                cur = cur.right


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data

class MTLCNN_single(nn.Module):
    def __init__(self, texture_labels):
        super(MTLCNN_single, self).__init__()

        # shared conv layers
        self.conv1 = nn.Conv2d(3, 96, kernel_size=11, stride=4)
        # torch.nn.init.xavier_uniform(self.conv1.weight)
        self.bn1 = nn.BatchNorm2d(96)

        self.conv2 = nn.Conv2d(96, 256, kernel_size=5, stride=1, padding=2)
        # torch.nn.init.xavier_uniform(self.conv2.weight)
        self.bn2 = nn.BatchNorm2d(256)

        self.conv3 = nn.Conv2d(256, 384, kernel_size=3, stride=1, padding=1)
        # torch.nn.init.xavier_uniform(self.conv3.weight)

        self.max_pool = nn.MaxPool2d(3, 2)
        self.drop_out = nn.Dropout(p=0.5)

        self.avg_pool = nn.AvgPool2d(13, 1)

        # Texture classification task
        self.fc_texture_1 = nn.Linear(in_features=384, out_features=4096)
        # torch.nn.init.xavier_uniform(self.fc_texture_1.weight)
        self.fc_texture_2 = nn.Linear(in_features=4096, out_features=4096)
        # torch.nn.init.xavier_uniform(self.fc_texture_2.weight)
        self.texture_out = nn.Linear(in_features=4096, out_features=len(texture_labels))

    def forward(self, x):
        if torch.cuda.is_available():
            x = x.float().cuda()
        else:
            x = x.float()

        # shared conv
        x = F.relu(self.bn1(self.conv1(x)))
        x = self.max_pool(x)
        x = F.relu(self.bn2(self.conv2(x)))
        x = self.max_pool(x)
        x = F.relu(self.conv3(x))

        # texture classification

        texture_head = self.avg_pool(x)
        texture_head = texture_head.reshape(-1, 384 * 1)

        texture_head = F.relu(self.fc_texture_1(texture_head))
        texture_head = F.relu(self.fc_texture_2(texture_head))
        texture_head = self.texture_out(texture_head)

        return texture_head


# -*- coding: utf-8 -*-
"""
Created on Sun Dec 15 22:59:17 2019

@author: spand
"""
import numpy as np
import pandas as pd

columns = ['Kind','Name','CountLineCode','CountClassBase', 'CountClassCoupled',
       'CountClassDerived', 'CountDeclMethodAll', 'CountDeclMethod',
       'MaxInheritanceTree', 'PercentLackOfCohesion',
       'CountDeclInstanceMethod', 'CountDeclInstanceVariable']

metrics = ['CountLineCode','CountClassBase', 'CountClassCoupled',
       'CountClassDerived', 'CountDeclMethodAll', 'CountDeclMethod',
       'MaxInheritanceTree', 'PercentLackOfCohesion',
       'CountDeclInstanceMethod', 'CountDeclInstanceVariable']

# Note:
# I first compared NL, if there was a difference I compared coupling,
# inheritance tree and cohesion metrics, if they showed improvement in
# the next version, and the code lines were not increased, there was a
# fault that was corrected, hence 1, otherwise 0.

# Need low IT, low coupling, high cohesion

def func(version1,version2) :

    # Read version 1 and sanitize
    csv1 = pd.read_csv(version1)
    csv1 = csv1[columns]
    csv1.dropna(inplace=True)
    csv1 = csv1[csv1.CountLineCode > 0]

    # Read version 2 and sanitize
    csv2 = pd.read_csv(version2)
    csv2 = csv2[columns]
    csv2.dropna(inplace=True)
    csv2 = csv2[csv2.CountLineCode > 0]

    # Get the names of classes
    names = csv1.Name.tolist()

    # list of our target variable
    ans = []

    # For each class
    for i in names :
        fault_corrected = 0
        v2 = csv2[csv2.Name == i]
        if not v2.empty :
            v1 = csv1[csv1.Name == i]
            # If the lines of code are non-increasing
            if v1.iloc[0]['CountLineCode'] >= v2.iloc[0]['CountLineCode']:
                # and if DIT has become lower
                if v1.iloc[0]['MaxInheritanceTree'] > v2.iloc[0]['MaxInheritanceTree']:
                    fault_corrected = 1
                # or if coupling has decreased
                elif v1.iloc[0]['CountClassCoupled'] > v2.iloc[0]['CountClassCoupled']:
                    fault_corrected = 1
                # or if cohesion has increased
                elif v1.iloc[0]['PercentLackOfCohesion'] > v2.iloc[0]['PercentLackOfCohesion']:
                    fault_corrected = 1

        # If the class has been deleted, mark fault as corrected
        else:
            fault_corrected = 1

        ans.append(fault_corrected)

    # Add a new column Change to the v1 file
    csv1['Change'] = ans

    # save the file to disk
    csv1.to_csv(ver1 + "change.csv")

ver1 = 'flink-release-1.4-preprocessed.csv' #Change the path to version 1
ver2 = 'flink-release-1.5-preprocessed.csv' #Change the path to version 2

func(ver1,ver2)


# -*- coding: utf-8 -*-

class CHurt(object):
    @staticmethod
    def CalculateByStringFunc(killer, beKiller, hurtFuncString):
        str = killer.prop.strength
        agi = killer.prop.agility
        inte = killer.prop.intelligence

        hurt = eval(hurtFuncString)

        # hack
        if hurtFuncString.find("inte"):
            hurt *= (1 - beKiller.prop.inteResist)

        return int(hurt)

    @staticmethod
    def Calculate(killer, beKiller, hurtInfo):
        import math

        if hasattr(hurtInfo, "executeCondition") and hurtInfo.executeCondition > 0.0:
            hpPercent = beKiller.prop.hp / beKiller.prop.hpMax
            if hpPercent <= hurtInfo.executeCondition:
                return math.ceil(beKiller.prop.hp)

        hurtFuncString = hurtInfo.hurtFunc
        hurt = CHurt.CalculateByStringFunc(killer, beKiller, hurtFuncString)

        return math.ceil(hurt)

    @staticmethod
    def Attack(killer, beKiller, hurtInfo):
        hurt = CHurt.Calculate(killer, beKiller, hurtInfo)

        hp = beKiller.prop.hp
        hp -= hurt
        hp = max(hp, 0.0)

        beKiller.prop.hp = hp

        import Net
        import war_pb2
        hurtMsg = war_pb2.Hurt()
        hurtMsg.id = beKiller.objId
        hurtMsg.hp = int(hurt)

        # 发送相关的伤害信息
        if killer.client:
            Net.SendMessage(killer.client, hurtMsg)

        if beKiller.client:
            Net.SendMessage(beKiller.client, hurtMsg)

        import player_pb2
        syncHp = player_pb2.SyncHp()
        syncHp.id = beKiller.objId
        syncHp.hp = int(beKiller.prop.hp)

        beKiller.battle.BroadcastMessage(syncHp)
        return hurt


import csv
import numpy as np
import pandas as pd

def read_RNA(fil):

   # load data into pndas dataframe
   df=pd.read_table('HeLa_RNA_normalized_filtered.txt',index_col=0)

   # get rna names
   names = df.index

   # get data as stupidly as possible
   t0  = df.R1t0  + df.R3t0   # t == 0  hours
   t05 = df.R1t05 + df.R3t05  # t == 0.5 hours
   t1  = df.R1t1  + df.R3t1   # t == 1 hour
   t2  = df.R1t2  + df.R3t2   # etc.
   t8  = df.R1t8  + df.R3t8
   t16 = df.R1t16 + df.R3t16
   t24 = df.R1t24 + df.R3t24
   t30 = df.R1t30 + df.R3t30

   # put them all in a list
   rna = [t0, t05 ,t1 ,t2 ,t8 ,t16 ,t24 ,t30]

   # make it an array and average
   rna = np.array(rna)/2.0

   # transpose so times are columns
   rna = np.transpose( rna )

   return rna ,names

if __name__=="__main__":
   read_RNA('HeLa_RNA_normalized_filtered.txt')


class CountSort:
    # Sorts a sequence of integers.
    def sort(sequence):
        """in-place counting sort"""
        m = max(sequence) + 1
        count = [0] * m               # init with zeros
        for a in sequence:
            count[a] += 1             # count occurences
        i = 0
        for a in range(m):            # emit
            for c in range(count[a]): # - emit 'count[a]' copies of 'a'
                sequence[i] = a
                i += 1
        return (sequence)

##seq = [4, 13, 52, 7, 18, 3, 1, 6]
##print (CountSort.sort(seq))


import random
parent = dict()
rank = dict()

def make_set(vertice):
    parent[vertice] = vertice
    rank[vertice] = 0

def find(vertice):
    if parent[vertice] != vertice:
        parent[vertice] = find(parent[vertice])
    return parent[vertice]

def union(vertice1, vertice2):
    root1 = find(vertice1)
    root2 = find(vertice2)
    if root1 != root2:
        if rank[root1] > rank[root2]:
            parent[root2] = root1
        else:
            parent[root1] = root2
        if rank[root1] == rank[root2]: rank[root2] += 1

def kruskal(graph, e):
    t = ()
    f = 0
    for vertice in graph['vertices']:
        make_set(vertice)
        minimum_spanning_tree = set()
        edges = list(graph['edges'])
        edges.sort()

    for edge in edges:
        weight, vertice1, vertice2 = edge
        if edge == e and f == 1:
            edge = t
        if weight == e[0] and f == 0:
            t = edge
            edge = e
            f = 1
        weight, vertice1, vertice2 = edge

        if find(vertice1) != find(vertice2):
            union(vertice1, vertice2)
            minimum_spanning_tree.add(edge)
    if e in minimum_spanning_tree:
        return sorted(minimum_spanning_tree)
    else:
        return "The edge is not in any MST"

graph = {
'vertices': ['A', 'B', 'C', 'D', 'E', 'F', 'G'],
'edges': list([
(2, 'A', 'B'),
(3, 'A', 'D'),
(3, 'A', 'C'),
(4, 'C', 'B'),
(5, 'D', 'C'),
(3, 'B', 'E'),
(7, 'D', 'F'),
(1, 'C', 'E'),
(8, 'E', 'F'),
(9, 'G', 'F'),
])
}

print("Graph: ")
print(graph)
print()
e = graph['edges'][random.randint(0,len(graph['edges']))]
print ("MST containing edge "+str(e)+":")
print (kruskal((graph), e))


class Solution(object):
    def countBits(self, num):
        """
        :type num: int
        :rtype: List[int]
        """
        results = []
        results.append(0)
        for i in range(1,num+1):
            prev = i >> 1
            lastdigit = i & 1
            results.append(results[prev] + lastdigit)
        return results


def solve(i, d):
    C, I, prices = d
    C = int(C)
    prices = list(map(int, prices.split()))
    for p1, P1 in enumerate(prices[:-1]):
        remains = C-P1
        left = prices[p1+1:]
        if remains in left:
            result = [p1+1, p1+left.index(remains)+2]
            return i, ' '.join(map(str, result))


f=open("data","r")

dict={}

for lines in f:
    words=lines.rstrip("\n").split(" ")

    for word in words:
        if (word  not in dict):
            dict[word]=1
        else:
            dict[word]+=1

print(dict)

#method 1:
#flag=0
#for k ,v in dict.items():
 #   if (v>=flag):
  #      flag=v
#for k,v in dict.items():
 #   if(v==flag):
  #      print((k,v))

#method 2
#high=max(dict,key=dict.get)
#print(high)

#method 3
lst=[]
for v in dict.values():
    lst.append(v)
high=max(lst)

for k,v in dict.items():
    if (v==high):
        print(k,v)


import time
import os

from django.test import TestCase

from ..lockfile import FileLock, FileLocked

class LockTest(TestCase):

    def test_process_killed_force_unlock(self):
        lockfile = 'test.txt.lock'
        with open(lockfile, 'w+') as f:
            f.write('9999999')
        assert os.path.exists(lockfile)
        with FileLock('test.txt'):
            assert True

    def test_force_unlock_in_same_process(self):
        lockfile = 'test.txt.lock'
        with open(lockfile, 'w+') as f:
            f.write(str(os.getpid()))

        with FileLock('test.txt', force=True):
            assert True

    def test_exception_after_timeout(self):
        lockfile = 'test.txt.lock'
        with open(lockfile, 'w+') as f:
            f.write(str(os.getpid()))

        try:
            with FileLock('test.txt', timeout=1):
                assert False
        except FileLocked:
            assert True

    def test_force_after_timeout(self):
        lockfile = 'test.txt.lock'
        with open(lockfile, 'w+') as f:
            f.write(str(os.getpid()))

        timeout = 1
        start = time.time()
        with FileLock('test.txt', timeout=timeout, force=True):
            assert True
        end = time.time()
        assert end - start > timeout


from keras.models import Sequential
import keras
from keras.layers import BatchNormalization
import csv
import os
os.environ["PATH"] += os.pathsep + 'file:///home/gautom/anaconda3/envs/pyenv35/lib/graphviz/'

results = []
with open("noNA.csv") as csvfile:
    reader = csv.reader(csvfile) # change contents to floats
    for row in reader: # each row is a list
        results.append(row)

resultsTwo = []
with open("S_filledNA.csv") as csvfile:
    reader = csv.reader(csvfile) # change contents to floats
    for row in reader: # each row is a list
        resultsTwo.append(row)

def remove_col(arra, index):
    for row in arra:
        del row[index]

    return arra
results = remove_col(results, 0)
results = remove_col(results, 0)

resul = results[1:]
susa = resultsTwo[1:]
for row in susa:
    resul.append(row)

print(len(resul))

fina = []
for row in resul:
    rowa = []
    for val in row:
        print(resul.index(row))
        print(row)
        rowa.append(float(val))
    fina.append(rowa)

print("Coefficients:")
print(results[0])
import random
random.shuffle(fina)
random.seed(128)

res = fina[1:]
test = fina[1:50]
X_train = []
y_train = []
for row in res:
    print(row)
    X_train.append(row[0:11])
    y_train.append(row[11])

X_test = []
y_test = []
for row in test:
    X_test.append(row[0:11])
    y_test.append(row[11])

from keras.layers import Dense

#y_train = [val/100.0 for val in y_train]
#y_test = [val/100.0 for val in y_test]
print(y_train)
print(y_test)
# create model
model = Sequential()
model.add(BatchNormalization())
model.add(Dense(12, input_dim=12, activation='relu'))
model.add(BatchNormalization())
model.add(Dense(16, activation='relu'))
model.add(BatchNormalization())
model.add(Dense(16, activation='relu'))
model.add(BatchNormalization())
model.add(Dense(1))
rms_prop = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
model.compile(loss='mean_squared_error', optimizer=rms_prop, metrics=['mae'])

from keras.utils import plot_model
plot_model(model, to_file='deep_model.png')

import numpy as np
X_test = np.asarray(X_test)
print(X_test.shape)
y_test = np.asarray(y_test)

X_train = np.asarray(X_train)
print(X_train.shape)
y_train = np.asarray(y_train)

history = model.fit(X_train, y_train, epochs=1000, batch_size=100, validation_split=0.1)
# serialize model to JSON
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("model.h5")
print("MODEL SAVED")
def plot_history(history):
    import matplotlib.pyplot as plt
    plt.figure()
    plt.xlabel('Epoch')
    plt.ylabel('Mean Abs Error [1000$]')
    plt.plot(history.epoch, np.array(history.history['mean_absolute_error']),
           label='Train Loss')
    plt.plot(history.epoch, np.array(history.history['val_mean_absolute_error']),
           label = 'Val loss')
    plt.legend()
    plt.show()

plot_history(history)

from sklearn import metrics
import numpy as np
from scipy.signal import savgol_filter
# Predict and measure RMSE
pred = model.predict(X_test)

score = np.sqrt(metrics.mean_squared_error(pred,y_test))
print("Score (RMSE): {}".format(score))

def chart_regression(pred,y,sort=True):
    import pandas as pd
    import matplotlib.pyplot as plt
    t = pd.DataFrame({'pred' : pred, 'y' : y.flatten()})
    yhat = savgol_filter(y, 49, 12)
    if sort:
        t.sort_values(by=['y'],inplace=True)
    a = plt.plot(t['y'].tolist(),label='expected')
    b = plt.plot(t['pred'].tolist(),label='prediction')
    plt.plot(yhat, label='smooth')

    finaArra = []
    finaArra.append(t['y'].tolist())
    finaArra.append(t['pred'].tolist())
    import csv
    with open("3_1000deep_first_col_is_expected_second_col_is_predicted.csv", "w") as f:
        writer = csv.writer(f)
        writer.writerows(finaArra)
    """
    HI PRAYAG

    t['y'].tolist() this is the expected data > BLUE
    t['pred'].tolist() this is the predicted data > Orange

    """

    plt.ylabel('output')
    plt.legend()
    plt.show()
# Plot the chart
chart_regression(pred.flatten(),y_test)

import matplotlib.pyplot as plt
print(history.history.keys())
# summarize history for loss
plt.plot(history.history['loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()


'''
This script is for the decision-tree formulation.
'''

from gurobipy import *

class Primal:
    def __init__(self, data, features, treatment_col, disab_col, pshTr, outcome, prob_t, tree, time_limit, method, y_hat, b_warm, budget_constraint, z_greedy):
        self.data = data
        self.datapoints = data.index
        self.treatment = treatment_col
        self.outcome = outcome
        self.prob_t = prob_t
        self.method = method
        self.y_hat = y_hat
        self.b_warm = b_warm
        self.z_greedy = z_greedy
        self.budget_constraint = budget_constraint
        self.disab = disab_col
        self.pshT = pshTr

        self.treatments_set = data[self.treatment].unique()  # set of possible treatments

        self.features = features

        self.tree = tree

        # Decision Variables
        self.b = 0
        self.p = 0
        self.w = 0
        self.z = 0
        self.zeta = 0

        # Gurobi model
        self.model = Model('FlowOPT')
        # self.model.params.Threads = 1
        self.model.params.TimeLimit = time_limit

        # NEW ADDITIONS TO SPEED UP SOLVER
        # Root and Node Method - Barrier (2) vs DualSimplex (1) to solve LP Relaxation at each node
        self.model.setParam('Method',2)
        self.model.setParam('NodeMethod',2)
        self.model.setParam('Crossover',0) # Cross Over speeds up solving for Barrier

    ###########################################################
    # Create the master problem
    ###########################################################
    def create_primal_problem(self):

        ############################### define variables

        self.b = self.model.addVars(self.tree.Nodes, self.features, vtype=GRB.BINARY, name='b')
        self.p = self.model.addVars(self.tree.Nodes + self.tree.Terminals, vtype=GRB.BINARY, name='p')
        self.w = self.model.addVars(self.tree.Nodes + self.tree.Terminals, self.treatments_set, vtype=GRB.CONTINUOUS, lb=0,
                                    name='w')
        self.zeta = self.model.addVars(self.datapoints, self.tree.Nodes + self.tree.Terminals, self.treatments_set,
                                       vtype=GRB.CONTINUOUS, lb=0, name='zeta')
        self.z = self.model.addVars(self.datapoints, self.tree.Nodes + self.tree.Terminals, vtype=GRB.CONTINUOUS, lb=0,
                                    name='z')

        ############################### define constraints

        ## ADDING THE WARM START BRANCHING VARIABLES
        # if self.b_warm is not None:
        #     self.model.addConstrs(self.b[k] == self.b_warm[k] for k in self.b_warm.keys())

        # z[i,n] = z[i,l(n)] + z[i,r(n)] + zeta[i,n]    forall i, n in Nodes
        for n in self.tree.Nodes:
            n_left = int(self.tree.get_left_children(n))
            n_right = int(self.tree.get_right_children(n))
            self.model.addConstrs(
                (self.z[i, n] == self.z[i, n_left] + self.z[i, n_right] + quicksum(self.zeta[i, n, k] for k in self.treatments_set)) for i in self.datapoints)

        # z[i,l(n)] <= sum(b[n,f], f if x[i,f]<=0)    forall i, n in Nodes
        for i in self.datapoints:
            self.model.addConstrs((self.z[i, int(self.tree.get_left_children(n))] <= quicksum(
                self.b[n, f] for f in self.features if self.data.at[i, f] <= 0)) for n in self.tree.Nodes)

        # z[i,r(n)] <= sum(b[n,f], f if x[i,f]=1)    forall i, n in Nodes
        for i in self.datapoints:
            self.model.addConstrs((self.z[i, int(self.tree.get_right_children(n))] <= quicksum(
                self.b[n, f] for f in self.features if self.data.at[i, f] == 1)) for n in self.tree.Nodes)

        # sum(b[n,f], f) + p[n] + sum(p[m], m in A(n)) = 1   forall n in Nodes
        self.model.addConstrs(
            (quicksum(self.b[n, f] for f in self.features) + self.p[n] + quicksum(
                self.p[m] for m in self.tree.get_ancestors(n)) == 1) for n in
            self.tree.Nodes)

        # p[n] + sum(p[m], m in A(n)) = 1   forall n in Terminals
        self.model.addConstrs(
            (self.p[n] + quicksum(
                self.p[m] for m in self.tree.get_ancestors(n)) == 1) for n in
            self.tree.Terminals)

        #self.model.addConstr(self.p[1] == 0)

        # sum(sum(b[n,f], f), n) <= branching_limit
        # self.model.addConstr(
        #     (quicksum(
        #         quicksum(self.b[n, f] for f in self.features) for n in self.tree.Nodes)) <= self.branching_limit)

        # zeta[i,n] <= w[n,T[i]] for all n in N+L, i
        for n in self.tree.Nodes + self.tree.Terminals:
            for k in self.treatments_set:
                self.model.addConstrs(
                    self.zeta[i, n, k] <= self.w[n, k] for i in self.datapoints)

        # sum(w[n,k], k in treatments) = p[n]
        self.model.addConstrs(
            (quicksum(self.w[n, k] for k in self.treatments_set) == self.p[n]) for n in
            self.tree.Nodes + self.tree.Terminals)

        # 2k
        """for n in self.tree.Nodes + self.tree.Terminals:
            for i in self.datapoints:
                self.model.addConstr(quicksum(self.zeta[i, n, k] for k in self.treatments_set) == self.p[n])"""

        for n in self.tree.Terminals:
            self.model.addConstrs(quicksum(self.zeta[i, n, k] for k in self.treatments_set) == self.z[i, n] for i in self.datapoints)

        self.model.addConstrs(self.z[i, 1] ==1 for i in self.datapoints)

        #self.model.addConstr(self.b[1, 'V1'] == 1)
        #self.model.addConstr(self.b[2, 'V2'] == 1)

        # add budget constraints
        for k in self.treatments_set:
            if int(k) != 0:
                max_treated = self.budget_constraint[int(k)] * len(self.data)
                self.model.addConstr(
                    quicksum(
                        self.zeta[i, n, k] for i in self.datapoints for n in self.tree.Nodes + self.tree.Terminals) <= max_treated)

        ########################
        # Disability Constraint #
        # instead of doing mode.addConstr, it is better from memory perspective to just set
        # both upper (ob) and lower (lb) bounds to 0. Here I set all zetas for nondisabled individuals
        # to 0 for pshTr [2,3]
        for k in self.pshT:
            for i in self.datapoints:
                for n in self.tree.Nodes + self.tree.Terminals:
                    if self.data.at[i, self.disab] == 0:
                        self.zeta[i, n, k].ub = 0
        ########################

        # PRESPECIFYING THE INNER NODES' VALUES
        if self.b_warm is not None:
            for k, v in self.b_warm.items():
                if v == 0:
                    self.b[k].ub = 0
                    self.b[k].lb = 0
                else:
                    self.b[k].ub = 1
                    self.b[k].lb = 1

        if self.z_greedy is not None:
            for k, v in self.z_greedy.items():
                if int(k[0]) in self.datapoints:
                    self.z[k].ub = int(v)
                    self.z[k].lb = int(v)

        for n in self.tree.Nodes:
            # inner nodes cannot be treatment nodes
            self.p[n].ub = 0
            self.p[n].lb = 0

            # inner nodes cannot be treatment nodes
            for k in self.treatments_set:
                self.w[n, k].ub = 0

                for i in self.datapoints:
                    self.zeta[i, n, k].ub = 0

        # SOS Constraint - Specify at most one variable in a set can be a value other than 0
        for n in self.tree.Nodes:
            self.model.addSOS(GRB.SOS_TYPE1, [self.b[b] for b in self.b if b[0] == n]) # Only one of the features at each splitting node can be non-zero

        # define objective function
        obj = LinExpr(0)
        for i in self.datapoints:
            for n in self.tree.Nodes + self.tree.Terminals:
                for k in self.treatments_set:
                    if self.method == 'DM':
                        obj.add(self.zeta[i, n, k]*(self.data.at[i, self.y_hat[int(k)]]))
                    elif self.method == 'DR':
                        obj.add(self.zeta[i, n, k]*(self.data.at[i, self.y_hat[int(k)]]))
                        treat = self.data.at[i, self.treatment]
                        if int(treat) == int(k):
                            obj.add(self.zeta[i, n, k]*(self.data.at[i, self.outcome] - self.data.at[i, self.y_hat[int(k)]]) * self.data.at[i, self.prob_t])
                    elif self.method == 'IPW':
                        treat = self.data.at[i, self.treatment]
                        if int(treat) == int(k):
                            obj.add(self.zeta[i, n, k]*(self.data.at[i, self.outcome])*self.data.at[i, self.prob_t])

        # NEED TO CHANGE MULTIPLICATION TO DIVISION OF IPW

        self.model.setObjective(obj, GRB.MAXIMIZE)


from ..adapters.mapping import MapAdapter
from ..client import from_tree, record_history

tree = MapAdapter({})
client = from_tree(tree)

def test_history():
    "Very basic exercise of history"
    with record_history() as history:
        repr(client)  # trigger a request
    assert history.requests
    assert history.responses


import numpy as np

class Boundary:

    #initializer #1 - Dirichlet
    def __init__(self, ind_x=None, ind_y=None, value=None, bc_type=None):

        self.ind_x = ind_x
        self.ind_y = ind_y
        self.value = value
        if isinstance(value, float): self.type = 'dirichlet'
        elif isinstance(value, list): self.type = 'neumann'

    # @classmethod
    # def Dirichlet(cls, ind_x: list, ind_y: list, value: float) -> 'Boundary':
    #     return cls(ind_x, ind_y, value, 'dirichlet')
    #
    # @classmethod
    # def Neumann(cls, ind_x: list, ind_y: list, value: list) -> 'Boundary':
    #     return cls(ind_x, ind_y, value, 'neumann')


#!/usr/bin/env python3

import argparse
import logging
import json
from google.cloud import translate
logger = logging.getLogger(__name__)

def translate_text(source="en-US", target="fr", text="Example Application", project_id="rosy-griffin-312113"):
    """Translating Text."""

    client = translate.TranslationServiceClient()

    location = "global"

    parent = f"projects/{project_id}/locations/{location}"

    # Detail on supported types can be found here:
    # https://cloud.google.com/translate/docs/supported-formats
    response = client.translate_text(
        request={
            "parent": parent,
            "contents": [text],
            "mime_type": "text/plain",  # mime types: text/plain, text/html
            "source_language_code": f"{source}",
            "target_language_code": f"{target}",
        }
    )

    return response.translations

def find(dict_var, key):
    for k, v in dict_var.items():
        if k == key:
            if len(v) > 0:
                dict_var[k] = "XXX"
            yield v
        elif isinstance(v, dict):
            for id_val in find(v, key):
                yield id_val

def replace_item(obj, key, replace_value):
    for k, v in obj.items():
        if isinstance(v, dict):
            obj[k] = replace_item(v, key, replace_value)
    if key in obj:
        if len(obj[key]) >0:
            try:
                translation = translate_text(text=obj[key], target=ARGS.target)
                obj[key] = translation[0].translated_text

            except Exception as e:
                logger.warning(e)

    return obj

def main():
    schema = json.load(ARGS.file)
    replace_item(schema,"description", "XXX")
    replace_item(schema,"title", "XXX")
    print(json.dumps(schema,indent=4))

if __name__ == "__main__":

    logging.basicConfig(level=logging.WARNING)

    parser = argparse.ArgumentParser(
        description="Translate the titles and/or descriptions in a JSON-SCHEMA file"
    )
    parser.add_argument(
        "-v", "--verbose", help="print verbose output to console", action="store_true"
    )
    parser.add_argument(
        "-d", "--debug", help="turn on debug for asyncio", action="store_true"
    )

    parser.add_argument(
        "-s", "--source", help="source language", default="en-US"
    )
    parser.add_argument(
        "-t", "--target", help="target language", default="fr"
    )

    parser.add_argument(
        "-f", "--file", help="JSON-SCHEMA file to translate", type=argparse.FileType("r"),
        required=True,
    )

    ARGS = parser.parse_args()

    if ARGS.verbose:
        logging.getLogger().setLevel(logging.INFO)

    logger.info("Start")
    # translations = translate_text(source=ARGS.source, target=ARGS.target)
    # # Display the translation for each input text provided
    # for translation in translations:
    #     print("Translated text: {}".format(translation.translated_text))
    main()
    logger.info("Done")


import  requests
from bs4 import BeautifulSoup
import pandas as pd
from datetime import date

def parseData():
    ## Initial data
    url = 'https://www.curs.md/ro/curs_valutar_banci'
    response = requests.get(url)
    parse_html = BeautifulSoup(response.text, 'html.parser')
    ##

    ## --- > Bank name
    clas_bank_name = parse_html.find(id="tabelBankValute").select('a')

    bank_list = []
    for elem in clas_bank_name:
        s2 = elem.get_text()
        bank_list.append(s2)

    bank_list_new = [x.replace('\n', '') for x in bank_list]
    bank_list_new = [x.replace(' ', '') for x in bank_list_new]
    bank_list_name =[]
    count = 0
    for number in bank_list_new:
        count+=1
        bank_list_name.append(number)
        #print(str(count)+ str(' ') +number)

    #USD Vinzare / Cumparare

    data_USD = parse_html.body.findChildren(class_='column-USD column-ind-0')
    set_of_data_USD = []
    for resul in data_USD:

        usd_text = resul.get_text()
        set_of_data_USD.append(usd_text)
    del set_of_data_USD[0]

    usd_cumparare = set_of_data_USD[::2]
    usd_vinzare = []

    for i in range(len(usd_cumparare)): #перебираем элементы списка (в пределах длины первого списка)
        if set_of_data_USD[i] == usd_cumparare[i]: #сравниваем элементы первого списка с элементами второго списка
            del set_of_data_USD[i] #добавляем в пустой список элемент из первого, если он меньше
            usd_vinzare.append(set_of_data_USD[i])

    del usd_cumparare[0]
    del usd_vinzare[0]

    ### end

    # Eur vinzare / cumparare

    data_EUR = parse_html.body.findChildren(class_='column-EUR column-ind-1')
    set_of_data_EUR = []
    for resul in data_EUR:

        eur_text = resul.get_text()
        set_of_data_EUR.append(eur_text)
    del set_of_data_EUR[0]

    eur_cumparare = set_of_data_EUR[::2]
    eur_vinzare = []

    for i in range(len(eur_cumparare)): #перебираем элементы списка (в пределах длины первого списка)
        if set_of_data_EUR[i] == eur_cumparare[i]: #сравниваем элементы первого списка с элементами второго списка
            del set_of_data_EUR[i] #добавляем в пустой список элемент из первого, если он меньше
            eur_vinzare.append(set_of_data_EUR[i])

    del eur_cumparare[0]
    del eur_vinzare[0]

    ## End

    #Rub vinzare/cumparare

    data_RUB = parse_html.body.findChildren(class_='column-RUB column-ind-2')
    set_of_data_RUB = []
    for resul in data_RUB:

        rub_text = resul.get_text()
        set_of_data_RUB.append(rub_text)
    del set_of_data_RUB[0]

    rub_cumparare = set_of_data_RUB[::2]
    rub_vinzare = []

    for i in range(len(rub_cumparare)): #перебираем элементы списка (в пределах длины первого списка)
        if set_of_data_RUB[i] == rub_cumparare[i]: #сравниваем элементы первого списка с элементами второго списка
            del set_of_data_RUB[i] #добавляем в пустой список элемент из первого, если он меньше
            rub_vinzare.append(set_of_data_RUB[i])

    del rub_cumparare[0]
    del rub_vinzare[0]

    ## end

    ## RON vinzare / cumparare

    #Rub vinzare/cumparare

    data_RON = parse_html.body.findChildren(class_='column-RON column-ind-3')
    set_of_data_RON = []
    for resul in data_RON:

        ron_text = resul.get_text()
        set_of_data_RON.append(ron_text)
    del set_of_data_RON[0]

    ron_cumparare = set_of_data_RON[::2]
    ron_vinzare = []
    date_now = []
    for i in range(len(ron_cumparare)): #перебираем элементы списка (в пределах длины первого списка)
        if set_of_data_RON[i] == ron_cumparare[i]: #сравниваем элементы первого списка с элементами второго списка
            del set_of_data_RON[i] #добавляем в пустой список элемент из первого, если он меньше
            ron_vinzare.append(set_of_data_RON[i])
            date_now.append(date.today())

    del ron_cumparare[0]
    del ron_vinzare[0]

    ## end
    desired_width=320

    pd.set_option('display.width', desired_width)

    pd.set_option('display.max_columns',10)

    table1 = (list(zip(bank_list_name,usd_cumparare,usd_vinzare,eur_cumparare,eur_vinzare, rub_cumparare, rub_vinzare,ron_cumparare,ron_vinzare,date_now)))

    df = pd.DataFrame(table1,columns=['Denumirea Bancii',
                                      'Cumparare USD',  'Vinzare USD',
                                      'Cumparare EUR',  'Vinzare EUR',
                                      'Cumparare RUB',  'Vinzare RUB',
                                      'Cumparare RON',  'Vinzare RON',
                                      'Date'])

    return df

vallet_data = parseData()


""" 3rd edition of IEC standard 61400-1 """
def fVref(WT_class='I'):
    if WT_class== 'I':
          Vref = 50.0
    elif WT_class== 'II':
          Vref = 42.5
    elif WT_class=='III':
        Vref = 37.5
    else:
        raise Exception('Unknown wind turbine class '+IEC_class)
    return Vref

def fIref(IEC_class='A'):
    if IEC_class=='A':
        Iref=0.16
    elif IEC_class=='B':
        Iref=0.14
    elif IEC_class=='C':
        Iref=0.12
    else:
        raise Exception('Unknown class '+IEC_class)
    return Iref

def sigma1_ETM(WS, IEC_class='A'):
    # "sigma1" from extreme turbulence model
    Iref = fIref(IEC_class)
    c    = 2.
    Vave = 10.
    return  c * Iref * (0.072 * (Vave / c + 3.) * (WS / c - 4.) + 10.)

def sigma1_NTM(WS, IEC_class='A'):
    # Normal turbulence model
    # Function sigma, offshore
    #   sigma1 = fE_sigma(U_10_hub) + 1.28*fD_sigma(U_10_hub)
    Iref = fIref(IEC_class)
    return Iref * (0.75 * WS + 5.6)

def ETM(WS, IEC_class='A'):
    # Turbulence intensity from extreme turbulence model
    return sigma1_ETM(WS, IEC_class)/ WS

def NTM(WS, IEC_class='A'):
    # Turbulence intensity from normal turbulence model
    return sigma1_NTM(WS, IEC_class)/ WS

def Lambda(zhub, IECedition=3):
    """
    IEC length scale.  Lambda = 0.7*min(Zhat,zhub)
    Where: Zhat = 30,60 for IECedition = 2,3, respectively.
    """
    if IECedition <= 2:
        return 0.7 * min(30, zhub)
    return 0.7 * min(60, zhub)

def KaimalSpectrum(f, zhub, uhub, Model='ETM', IEC_class='A' ):
    """IEC Kaimal spectrum model.
    See IEC 61400-3 Appendix B.2
        S_k(f) = \frac{4 \sigma_k^2 L_k/V }{(1+6 f L_k/V)^{5/3}}
        k=0,1,2 = longi, lateral, upward
    """
    if Model == 'ETM':
        sigma1 = sigma1_ETM(uhub, IEC_class)
    elif Model == 'NTM':
        sigma1 = sigma1_NTM(uhub, IEC_class)

    S_k = np.zeros(3,len(f))
    sigma_k = sigma1       * np.array([1.0,  0.8,  0.5 ])
    L_k     = Lambda(zhub) * np.array([8.10, 2.70, 0.66])
    for k in [0,1,2]: # loop on components
        S_k[k] = (4 * sigma[k]**2 * L_k[k]/uhub)/( 1 + 6 * f * L_k[k]/uhub)**(5/3)
    return S_k

def VonKarmanSpectrum(f, zhub, uhub, Model='ETM', IEC_class='A'):
    r"""IEC Von-Karman spectral model
    The form of this model is,
    .. math::
          S_u(f) = \frac{4 \sigma^2/\hat{f}}{(1+71(f/\hat{f})^2)^{5/6}}
          S_v(f) = S_w(f) = (1+189(f/\hat{f})^2\frac{2\sigma^2/\hat{f}}{(1+71 (f/\hat{f})^2)^{11/6}}
    Where,
      :math:`\hat{f} = \bar{u}_{hub}/\Lambda`
    """
    if Model == 'ETM':
        sigma1 = sigma1_ETM(uhub, IEC_class)
    elif Model == 'NTM':
        sigma1 = sigma1_NTM(uhub, IEC_class)

    spec    = np.zeros(3,len(f))
    sig2 = 4 * sigma1 ** 2
    L_u  = 3.5 * Lambda(zhub)/uhub
    dnm = 1 + 71 * (f * L_u) ** 2 # TODO 71 or 70.8...?
    S_k[0] =     sig2 * L_u/(dnm)**(5./6)
    S_k[1] = sig2 / 2 * L_u/(dnm)**(11./6) * (1 + 189 * (f * L_u) ** 2)
    S_k[2] = S_k[1]
    return spec

if __name__=='__main__':
    import numpy as np
    import matplotlib.pyplot as plt
    ws=np.linspace(0,30,100)
    fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8)
    fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20)
    ax.plot(ws,ETM(ws,'A')*100, '-', label='ETM A', c='k')
    ax.plot(ws,ETM(ws,'B')*100, '--', label='ETM B', c='k')
    ax.plot(ws,ETM(ws,'C')*100, '.-', label='ETM C', c='k')
    ax.plot(ws,NTM(ws,'A')*100, '-', label='NTM A',c=[0.5,0.5,0.5])
    ax.plot(ws,NTM(ws,'B')*100, '--', label='NTM B',c=[0.5,0.5,0.5])
    ax.plot(ws,NTM(ws,'C')*100, '-.', label='NTM C',c=[0.5,0.5,0.5])
    ax.set_ylim([0,88])
    ax.set_xlim([0,30])
    ax.set_xlabel('Wind speed [m/s]')
    ax.set_ylabel('Turbulence intensity [%]')
    ax.legend()
    ax.tick_params(direction='in')
    plt.show()


import os
import time

# THIS SCRIPT ITERATE THE 1ST PRIZE SCORE AND REMOVE DUPLICATES!

def iterateNumbers(f, dirname):

    fname = os.path.join('MERGED', f)

    if not os.path.exists(dirname):
        os.makedirs(dirname)

    zerowrite = open(dirname + "/0series.txt", 'w')
    onewrite = open(dirname + "/1series.txt", 'w')
    twowrite = open(dirname + "/2series.txt", 'w')
    threewrite = open(dirname + "/3series.txt", 'w')
    fourwrite = open(dirname + "/4series.txt", 'w')
    fivewrite = open(dirname + "/5series.txt", 'w')
    sixwrite = open(dirname + "/6series.txt", 'w')
    sevenwrite = open(dirname + "/7series.txt", 'w')
    eightwrite = open(dirname + "/8series.txt", 'w')
    ninewrite = open(dirname + "/9series.txt", 'w')

    # for line in enumerate(f):
    zerolist = list()
    onelist = list()
    twolist = list()
    threelist = list()
    fourlist = list()
    fivelist = list()
    sixlist = list()
    sevenlist = list()
    eightlist = list()
    ninelist = list()

    with open(fname) as file:
        for row in file:
            row = row.rstrip()
            if int(row[:1]) == 0:
                zerolist.append(row)
            elif int(row[:1]) == 1:
                onelist.append(row)
            elif int(row[:1]) == 2:
                twolist.append(row)
            elif int(row[:1]) == 3:
                threelist.append(row)
            elif int(row[:1]) == 4:
                fourlist.append(row)
            elif int(row[:1]) == 5:
                fivelist.append(row)
            elif int(row[:1]) == 6:
                sixlist.append(row)
            elif int(row[:1]) == 7:
                sevenlist.append(row)
            elif int(row[:1]) == 8:
                eightlist.append(row)
            elif int(row[:1]) == 9:
                ninelist.append(row)

    zerolist.sort()
    onelist.sort()
    twolist.sort()
    threelist.sort()
    fourlist.sort()
    fivelist.sort()
    sixlist.sort()
    sevenlist.sort()
    eightlist.sort()
    ninelist.sort()

    for val in zerolist:
        zerowrite.write(val + '\n')
    for val in onelist:
        onewrite.write(val + '\n')
    for val in twolist:
        twowrite.write(val + '\n')
    for val in threelist:
        threewrite.write(val + '\n')
    for val in fourlist:
        fourwrite.write(val + '\n')
    for val in fivelist:
        fivewrite.write(val + '\n')
    for val in sixlist:
        sixwrite.write(val + '\n')
    for val in sevenlist:
        sevenwrite.write(val + '\n')
    for val in eightlist:
        eightwrite.write(val + '\n')
    for val in ninelist:
        ninewrite.write(val + '\n')

    file.close()
    zerowrite.close()
    onewrite.close()
    twowrite.close()
    threewrite.close()
    fourwrite.close()
    fivewrite.close()
    sixwrite.close()
    sevenwrite.close()
    eightwrite.close()
    ninewrite.close()

######################################################################################

def main():
    start = time.time()

    iterateNumbers('results1stprizeremoveduplicates.txt', '1STPRIZE')
    iterateNumbers('results2ndprizeremoveduplicates.txt', '2NDPRIZE')
    iterateNumbers('results3rdprizeremoveduplicates.txt', '3RDPRIZE')

    end = time.time()
    print(str(end - start) + ' seconds')

if __name__ == "__main__":
    main()

######################################################################################


import torch
import torch.nn as nn
import torch.nn.functional as F

class Event_tagger(nn.Module):
    def __init__(self,
                 word_to_ix,
                 tag_to_ix,
                 ix_to_tag,
                 weights_matrix,
                 batch_size,
                 word_embed_dim=300,
                 bidirectional=True,
                 gru_num_layers=2,
                 dropout=.5):
        '''
            initialize models
            batch_size      - size of batches for traininig
            word_embed_dim  - dimension of word embeddings
            gru_num_layers - number of gru layers
            dropout         - rate for dropout layer
        '''
        super(Event_tagger, self).__init__()

        # dictionaries
        self.word_to_ix = word_to_ix
        self.tag_to_ix = tag_to_ix
        self.ix_to_tag = ix_to_tag

        # parameters
        self.batch_size = batch_size
        self.word_embed_dim = word_embed_dim
        self.gru_hidden_dim = self.word_embed_dim
        self.gru_num_layers = gru_num_layers
        self.dropout = dropout

        self.num_directions = 2 if bidirectional else 1

        self.word_embeds, num_embeds = create_embed_layer(weights_matrix, True)

        self.gru = nn.GRU(self.word_embed_dim,
                          self.gru_hidden_dim,
                          bidirectional=bidirectional,
                          num_layers=self.gru_num_layers,
                          batch_first=True)
        self.dense = nn.Linear(self.gru_hidden_dim * self.num_directions,
                               len(self.tag_to_ix))

    def forward(self, words_batch):
        '''
            words_batch - contains indices of words in current batch with shape
                        (batch_size, num_words_in_sentence)

            creates word level word embeddings from words_batch

            runs Bi-directional gru over input word representation to get
            final word representation which is fed to linear layer and softmax
            activation function to generate probability distribution for event
            tag set
        '''
        # create word-level word embeddings
        word_embed_word_level = words_batch.view(-1)
        word_embed_word_level = self.word_embeds(word_embed_word_level)
        batch_sent_embed = word_embed_word_level.view(
            words_batch.shape[0], -1, word_embed_word_level.shape[-1])

        # create final word representation from gru
        gru_out, _ = self.gru(batch_sent_embed)

        # get probabilities for event tag
        tag_space = self.dense(gru_out)
        tag_scores = F.log_softmax(tag_space, dim=2)
        return tag_scores


import matplotlib.pyplot as plt
import numpy as np
from stompy.spatial import field

import os
opj=os.path.join

from stompy.spatial import interp_coverage
import composite_field_v01

from composite_field_v01 import mbf
dem_dir="tiles_5m_20170501"
os.path.exists(dem_dir) or os.mkdir(dem_dir)

from multiprocessing import Pool

def f(args):
    fn,xxyy,res = args
    if not os.path.exists(fn):
        mbf.to_grid(dx=res,dy=res,bounds=xxyy).write_gdal(fn)

if __name__ == '__main__':
    p = Pool(2)
    res=5.0
    total_bounds=[576000., 596000., 4137000., 4152000.]
    tile_x=tile_y=1000.0

    total_bounds[0] = np.floor(total_bounds[0]/tile_x) * tile_x
    total_bounds[1] = np.ceil(total_bounds[1]/tile_x) * tile_x
    total_bounds[2] = np.floor(total_bounds[2]/tile_y) * tile_y
    total_bounds[3] = np.ceil(total_bounds[3]/tile_y) * tile_y

    calls=[]
    for x0 in np.arange(total_bounds[0],total_bounds[1],tile_x):
        for y0 in np.arange(total_bounds[2],total_bounds[3],tile_y):
            xxyy=(x0,x0+tile_x,y0,y0+tile_y)
            fn=os.path.join(dem_dir,"%.0f_%.0f.tif"%(x0,y0))
            calls.append( [fn,xxyy,res] )

    p.map(f, calls )


from ..forms import *

def set_accounts_queries(request, set_query):
    if request.method == 'POST':
        # CREAR FORMULARIO NUEVO
        form = Account_Form(data=request.POST)

        if form.is_valid():
            # LEER FORMULARIO
            data = form.cleaned_data

            # VARIABLES
            userCui = request.POST['userCui']
            userCui = 'null' if len(userCui) == 0 else '"'+userCui+'"'

            # NOMBRE COMERCIAL
            userBusiness = request.POST['userBusiness']
            userBusiness = 'null' if len(
                userBusiness) == 0 else '"'+userBusiness+'"'

            # INFORMACIÓN DE LA CUENTA
            description = request.POST.get('description', '')
            accountId = request.POST['accountId']
            isSingle = request.POST.get('isSingle', '0').replace('on', '1')
            isDollar = data.get('isdollar', False)
            isDollar = '1' if isDollar else '0'
            accountType = request.POST['accountType']
            interest = request.POST['interest']
            description = request.POST['description']
            plan = request.POST['plan'].replace('-', '/')

            # INSERTAR EN TIPO DE CUENTA
            if accountType == 'saving':
                set_query(
                    f'INSERT INTO SavingAccount VALUES ({accountId}, {interest}); ')
                set_query(
                    f'INSERT INTO AccountType(id, saving) VALUES ({accountId}, {accountId}); ')
            elif accountType == 'timedSaving':
                set_query(
                    f'INSERT INTO TimedSavingAccount VALUES ({accountId}, {interest}, "{plan}"); ')
                set_query(
                    f'INSERT INTO AccountType(id, timedSaving) VALUES ({accountId}, {accountId}); ')
            elif accountType == 'monetary':
                set_query(
                    f'INSERT INTO MonetaryAccount VALUES ({accountId}, "{description}"); ')
                set_query(
                    f'INSERT INTO AccountType(id, monetary) VALUES ({accountId}, {accountId}); ')

            # INSERTAR EN CUENTA
            set_query(
                f'INSERT INTO Account VALUES ({accountId}, 1, 0, {isSingle}, 0, 0, {isDollar}, {userCui}, {userBusiness}, 15, 0); ')


import math
from BlindersPy3 import Blinders
from BlindersPy3 import FitType

fitdict={
    "fitrange":[50,500],
    "npar":5,
    "initial_values":[1000000,64.4, 0.333, 10, 0.1],
    "parnames": ["N", "tau", "A", "R", "phi"],
    "blindingPhrase":"ding!!!",
}
fitdict['blindingLib'] = Blinders(FitType.Omega_a, str(fitdict['blindingPhrase']))

def fitfunc(x,p):
    norm  = p[0]
    life  = p[1]
    asym  = p[2]
    R     = p[3]
    phi   = p[4]

    time  = x[0]
    omega = fitdict['blindingLib'].paramToFreq(R)

    return norm * math.exp(-time/life) * (1 - asym*math.cos(omega*time + phi))


graph = [
  [1,3,1],
  [1,5,1],
  [4,2,1]
]

graph2 = [
    [1,2],
    [5,6],
    [1,1]]

def minPathSum(grid):
    memo = [[0]*(len(grid[0])) for i in range(len(grid))]

    memo[0][0] = grid[0][0]
    for i in range(1,len(grid[0])):
        memo[0][i] = memo[0][i-1] + grid[0][i]
    for i in range(1, len(grid)):
        memo[i][0] = memo[i-1][0] + grid[i][0]

    print(memo)
    for i in range(1, len(grid)):
        for j in range(1, len(grid[0])):
            print(i,j)
            memo[i][j] = min(memo[i-1][j], memo[i][j-1]) + grid[i][j]
            print(memo)
    print(memo)
    return memo[len(grid)-1][len(grid[0])-1]

print(minPathSum(graph2))