# Definition for a binary tree node.
class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution:
    def constructMaximumBinaryTree(self, nums: List[int]) -> TreeNode:
        St = []
        for num in nums:
            node = TreeNode(num)
            while St and St[-1].val < num:
                node.left = St.pop()
            if St:
                St[-1].right = node
            St.append(node)
        return St[0]

0601-0700/0654-Maximum Binary Tree/0654-Maximum Binary Tree.py


#Binary Search
#Given a sorted array arr[] of n elements, write a function to search a given element x in arr[]
#recursive

def binSearch(arr, l, r, elem):
  if r >= l:
    mid = l + (r - l)//2

    if arr[mid] == elem:
      return mid

    elif arr[mid] > elem:
      return binSearch(arr, l, mid-1, elem)

    else:
      return binSearch(arr, mid+1, r, elem)

  else:
    return -1

arr = [ 2, 3, 4, 10, 40 ]
elem = 10

result = binSearch(arr, 0, len(arr)-1, elem)

if result != -1:
    print ("Element is present at index %d" % result)
else:
    print ("Element is not present in array")


__all__ = [
    # Utilities
    'CapabilityTemplate',
    'add',
    # Base services
    'bird',
    'dhclient',
    'dhcpd',
    'firstboot',
    'gateway',
    'jail',
    'linux',
    'named',
    'openssh',
    'pf',
    'pflate',
    'pflog',
    'resolvconf',
    'sshd',
    'trust',
    'zfs'
]

import frieze
import importlib
import io
import os
import sys
import tarfile

import mako.template
import pkg_resources as pkg

from openarc import staticproperty

class CapabilityTemplate(object):

    # Resource expectations.
    cores  = None
    memory = None

    # Mounts to be fed into the capability's container
    mounts = []

    # Ports to be redirected from outside
    ports = []

    # Config files that need to be set in the container
    config = []

    # The site in which we would like this capability to be run. None for any site.
    affinity = None

    # Application can exist inside a jail
    jailable = True

    # Package to install. If set to None, nothing is installed. If set and
    # custom_pkg=False, a naive call to pkg install <package> is issued. If set
    # and custom_pkg=True, pkg add <path/to/txz> is issued.
    package = None

    # Knobs used by job system to adjust the operation of this capability. Knobs
    # not in this list cannot be set on the capability
    knobs = []

    # Resource directory (for internal bookkeeping)
    resource_path = 'frieze.capability.resources'

    def __init__(self):
        self.set_knobs = {}
        self.cmd_count = 0

    def next_cmd_cfg_name(self):
        cfg_name = f'{self.name}_cmd_{self.cmd_count}'
        self.cmd_count += 1
        return cfg_name

    def generate_cfg_files(self, host, __exclude__=[]):
        rv = {}
        __exclude__.append('__init__.py')
        __exclude__.append('__pycache__')
        try:
            cmd_cnt = 0
            cap_cfgs = pkg.resource_listdir(self.resource_path, self.name)
            for cfg in [cfg for cfg in cap_cfgs if cfg not in __exclude__]:
                cfg_raw = pkg.resource_string(f'{self.resource_path}.{self.name}', cfg)
                if cfg_raw[0:2]=='#!':
                    cfg_name = self.next_cmd_cfg_name()
                else:
                    cfg_1l = bytearray()
                    for b in cfg_raw:
                        if b=='\n'.encode()[0]:
                            break
                        cfg_1l.append(b)
                    cfg_name = bytes(cfg_1l[2:]).decode().split(' ')[0]

                try:
                    rv[cfg_name] = mako.template.Template(cfg_raw.decode()).render(host=host)
                except UnicodeDecodeError:
                    rv[cfg_name] = cfg_raw
                except Exception as e:
                    print(f"Error processing template [{cfg_name}] for capability [{self.name}]")
                    raise
        except FileNotFoundError:
            pass
        return rv

    @staticproperty
    def name(cls):
        return cls.__name__

    def setknob(self, knob, value):
        knobs = getattr(self, 'set_knobs', {})
        if not knobs:
            setattr(self, 'set_knobs', knobs)
        if knob in self.knobs:
            self.set_knobs[knob] = value
        else:
            raise OAError("[%s] is not a permitted knob for %s" % (knob, self.name))
        return self

    @property
    def setknobs_exist(self):
        return len(self.set_knobs)>0

    def startcmd(self, os, fib, *args):
        """Return command to be used when starting this capability without
        the system job management infrastructure"""
        from ..osinfo import OSFamily
        extra_prms = ' '.join(args)
        return {
            OSFamily.FreeBSD: "setfib %s service %s restart %s" % (fib.value, self.name, extra_prms)
        }[os.family]

def add(searchdir):
    """Add user-defined capabilities in searchdir. Pop up one directory and
    use import_modules"""
    canonical_path = os.path.expanduser(searchdir)
    sys.path.append(canonical_path)

    capmod = importlib.import_module('frieze.capability')
    modules = os.listdir(canonical_path)
    for module_name in modules:
        # Push module information into frieze.capability, and fix resource_path
        try:
            module = importlib.import_module(module_name)
        except ModuleNotFoundError:
            continue

        for m in dir(module):
            elem = getattr(module, m)
            if type(elem)==type and isinstance(elem(), CapabilityTemplate):
                setattr(capmod, elem.__name__, elem)
                setattr(elem, 'resource_path', f'{module_name}.resources')

    importlib.invalidate_caches()

## Service definitions

class bird(CapabilityTemplate):
    package = 'bird'
    jailable = False

class dhcpd(CapabilityTemplate):
    package = 'isc-dhcp44-server'
    jailable = False
    knobs = [
        'ifaces'
    ]

class dhclient(CapabilityTemplate):
    jailable = False

class firstboot(CapabilityTemplate):

    def generate_cfg_files(self, host, bootstrap=False):

        if bootstrap:
            exclude = []
        else:
            exclude = [
                'bootstrap.sh',
                'configinit.payload',
                'resetimage.sh',
            ]

        return super().generate_cfg_files(host, __exclude__=exclude)

    def generate_bootstrap_tarball(self):
        """Generate tarball of commands to be executed by frieze_configinit, which
        executes this payload on first boot"""
        rv = {
            frieze.capability.ConfigFile.PRE_COMMAND_LIST : [
                'sysrc -f /etc/rc.conf firstboot_pkgs_list="bash nano ca_root_nss openssh-portable"',
                'sysrc -f /etc/rc.conf sshd_enable="NO"',
                'sysrc -f /etc/rc.conf.local openssh_enable="YES"',
                'mkdir -p /usr/local/etc/ssh',
                'printf "TrustedUserCAKeys /usr/local/etc/ssh/ca.pub\nPasswordAuthentication no\nChallengeResponseAuthentication no\nPermitRootLogin prohibit-password\n" > /usr/local/etc/ssh/sshd_config'
            ]
        }

        return\
            frieze.capability.ConfigInit(rv)\
            .generate()\
            .decode()

class gateway(CapabilityTemplate): pass

class jail(CapabilityTemplate):

    def generate_cfg_files(self, host):
        """Generate regular files, and then add in fstabs for individual jails """
        rv = super().generate_cfg_files(host, __exclude__=['jail-fstab', 'jail-zfs-skeleton', 'jail-configinit'])

        pkg_name = 'frieze.capability.resources.%s' % self.name
        jail_zfs_template    = pkg.resource_string(pkg_name, 'jail-zfs-skeleton').decode()
        jail_fstab_template  = pkg.resource_string(pkg_name, 'jail-fstab').decode()
        jail_cfinit_template = pkg.resource_string(pkg_name, 'jail-configinit').decode()

        for container in host.containers:
            fstab_file = f'/usr/local/jails/{container.sysname}.fstab'
            rv[self.next_cmd_cfg_name()] = mako.template.Template(jail_zfs_template).render(container=container, host=host)
            rv[fstab_file] = mako.template.Template(jail_fstab_template).render(container=container, host=host)

            # Create config payload
            cfinit_file = f'{container.jaildir}/root/cfinit'
            rv[cfinit_file] = mako.template.Template(jail_cfinit_template).render(container=container, host=host)

        return rv

class linux(CapabilityTemplate): pass

class named(CapabilityTemplate):
    package = 'bind912'

    def generate_cfg_files(self, host):
        rv = {}
        zones = {}

        # Load some resources
        pkg_name = 'frieze.capability.resources.%s' % self.name
        zone_template        = pkg.resource_string(pkg_name, 'zone.db').decode()
        revzone_template     = pkg.resource_string(pkg_name, 'revzone.db').decode()
        named_local_template = pkg.resource_string(pkg_name, 'named.conf.local').decode()

        # Track reverse zones
        networks = dict()

        # Generate forward lookup files
        for site in host.site.domain.site:
            filename = "/usr/local/etc/namedb/dynamic/%s.db" % site.zone
            zones[site.zone] = filename
            rv[filename] = mako.template.Template(zone_template).render(zonecontainer=site, host=host)

            # Break forward lookups into class C's and assign revzones
            for dhost in site.host:
                try:
                    networks[dhost.revzone]
                except KeyError:
                    networks[dhost.revzone] = []
                networks[dhost.revzone].append({
                    'fqdn'   : dhost.fqdn,
                    'ip_ext' : dhost.ip4().split('.')[-1]
                })

        for deployment in host.site.domain.deployment:
            filename = "/usr/local/etc/namedb/dynamic/%s.db" % deployment.zone
            zones[deployment.zone] = filename
            rv[filename] = mako.template.Template(zone_template).render(zonecontainer=deployment, host=host)

            for container in deployment.containers:
                try:
                    networks[deployment.revzone]
                except KeyError:
                    networks[deployment.revzone] = []

                networks[deployment.revzone].append({
                    'fqdn'   : container.fqdn,
                    'ip_ext' : '.'.join(reversed(container.ip4().split('.')[2:]))
                })

        for nw, zonehosts in networks.items():
            filename = "/usr/local/etc/namedb/dynamic/%s.db" % nw
            zones[nw] = filename
            rv[filename] = mako.template.Template(revzone_template).render(revzone=nw, zonehosts=zonehosts, host=host)

        # Generate named.conf.local
        rv['/usr/local/etc/namedb/named.conf.local'] =\
            mako.template.Template(named_local_template).render(zones=zones)

        return {**rv, **super().generate_cfg_files(host, __exclude__=['zone.db', 'revzone.db', 'named.conf.local'])}

class pf(CapabilityTemplate): pass

class pflate(CapabilityTemplate): pass

class pflog(CapabilityTemplate): pass

class openssh(CapabilityTemplate):
    package = 'openssh-portable'

class resolvconf(CapabilityTemplate): pass

class sshd(CapabilityTemplate): pass

class trust(CapabilityTemplate):

    def generate_cfg_files(self, host):
        rv = {}

        # Generate trusts
        if type(host)!=frieze.Container:
            return rv

        from .server import TrustType, CertAction, CertFormat

        pkg_name = 'frieze.capability.resources.%s' % self.name
        crt_template  = pkg.resource_string(pkg_name, 'chain.crt')
        priv_template = pkg.resource_string(pkg_name, 'private.pem')

        cap = host.capability
        if cap.capability_alias and cap.capability_alias.is_external:
            print(f'======>Generating trusts for [{cap.fqdn}]')
            aliases = [ca.fqdn for ca in cap.capability_alias]
            cert =\
                cap.deployment.domain\
                .trust(trust_type=TrustType.LETSENCRYPT)\
                .issue_certificate(
                    aliases,
                    CertAction.REMOTE_ACCESS,
                    cert_format=CertFormat.PEM
                )

            rv[cap.truststore(pubkey=True)]  = mako.template.Template(crt_template).render(cap=cap, cert=cert)
            rv[cap.truststore(pubkey=False)] = mako.template.Template(priv_template).render(cap=cap, cert=cert)

        return rv

class zfs(CapabilityTemplate):

    def generate_cfg_files(self, host):

        rv = super().generate_cfg_files(host, __exclude__=['zpool-init'])

        pkg_name = 'frieze.capability.resources.%s' % self.name
        zpool_template = pkg.resource_string(pkg_name, 'zpool-init').decode()

        for bs in host.block_storage:
            try:
                rv[frieze.capability.ConfigFile.PRE_COMMAND_LIST]
            except KeyError:
                rv[frieze.capability.ConfigFile.PRE_COMMAND_LIST] = []
            rv[frieze.capability.ConfigFile.PRE_COMMAND_LIST].append(
                mako.template.Template(zpool_template).render(blockstore=bs, host=host)
            )

        return rv


"""
Data model for Products
"""
import boto3
import os
import datetime
import uuid

from common.utility.dynamodb import (
  to_dynamodb_json,
  from_dynamodb_json
)

PRODUCTS_TABLE_NAME = os.environ.get("PRODUCTS_TABLE_NAME")

class Products:
  def __init__(self, id: str = None, **kwargs):
    _data = { **kwargs, "id": id }

    if not id:
      _data["id"] = str(uuid.uuid4())
      _data["creation_date"] = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat()

    # Directly access __dict__ to avoid infinite loop cause of overwritten __setattr__
    self.__dict__.update({
      "_data": _data,
      "_delete_flag": False
    })

  def __repr__(self):
    return f"Product(id={self.id})"

  def __getattr__(self, key):
    return self._data.get(key, None)

  def __setattr__(self, key, value):
    self._data[key] = value

  def __getitem__(self, key):
    return self._data.get(key, None)

  def __setitem__(self, key, value):
    self._data[key] = value

  def delete(self):
    # __setattr__ would be called otherwise
    self.__dict__["_delete_flag"] = True

  def commit(self):
    if self._delete_flag:
      try:
        dynamodb = boto3.client("dynamodb")
        dynamodb.delete_item(
          TableName=PRODUCTS_TABLE_NAME,
          Key={
            "id": to_dynamodb_json(self.id)
          }
        )
      except Exception as e:
        print(f"Cant delete product {str(self)} due to {str(e)}.")
        raise e
    else:
      exp_names = {}
      exp_values = {}
      exp_set = {}
      exp_remove = []

      for k, v in self._data.items():
        if k == "id":
          continue

        if v == None:
          exp_remove.append(f"#{k}")
          exp_names[f"#{k}"] = f"{k}"
        else:
          exp_set[f"#{k}"] = f":{k}"
          exp_names[f"#{k}"] = f"{k}"
          exp_values[f":{k}"] = v

      set_expression = ""
      for k, v in exp_set.items():
        if set_expression == "":
          set_expression = f"SET {k} = {v}"
        else:
          set_expression = f"{set_expression},{k} = {v}"

      remove_expression = f" REMOVE {','.join(exp_remove)}" if exp_remove else ""

      update_expression = set_expression + remove_expression

      try:
        dynamodb = boto3.client("dynamodb")
        dynamodb.update_item(
          TableName=PRODUCTS_TABLE_NAME,
          Key={"id": to_dynamodb_json(self.id)},
          UpdateExpression=update_expression,
          ExpressionAttributeValues={ k: to_dynamodb_json(v) for k, v in exp_values.items() },
          ExpressionAttributeNames=exp_names,
          ReturnValues="NONE"
        )
      except Exception as e:
        print(f"Cant update products {str(self)} due to {str(e)}.")
        raise e

  def as_dict(self, populated=False):
    if not populated:
      return self._data

    populated_dict = {}
    for k, v in self._data.items():
      populated_dict[k] = v
    return populated_dict

  @staticmethod
  def from_dict(inp):
    return Products(**inp)

  @staticmethod
  def get_by_id(id):
    try:
      dynamodb = boto3.client("dynamodb")
      product_dict = dynamodb.get_item(
        TableName=PRODUCTS_TABLE_NAME,
        Key={ "id": to_dynamodb_json(id) }
      )["Item"]

      return Products.from_dict({ k: from_dynamodb_json(v) for k, v in product_dict.items() })
    except Exception as e:
      print(f"Cant get product with id={id} due to {str(e)}.")
      raise e


import os
import numpy as np
import pandas as pd
from collections import Counter
import matplotlib.pyplot as plt
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
from mmbench.color_utils import (
    print_title, print_subtitle, print_text, print_result)
from mmbench.dataset import get_train_full_data, get_test_full_data
from mmbench.workflow.predict import get_predictor
from mmbench.plotting import plot_bar

def benchmark_baseline(datasetdir, outdir):
    """ Train and test a baseline model on euaims

    Parameters
    ----------
    datasetdir: str
        the path to the euaims associated data.
    outdir: str
        the destination folder.
    """
    dataset = "euaims"
    print_title(f"GET MODELS LATENT VARIABLES: {dataset}")
    benchdir = outdir
    if not os.path.isdir(benchdir):
        os.mkdir(benchdir)
    print_text(f"Benchmark directory: {benchdir}")

    print_subtitle("Loading data...")
    modalities = ["clinical", "rois"]
    print_text(f"modalities: {modalities}")
    data_train, meta_train_df = get_train_full_data(
        dataset, datasetdir, modalities, residualize=False)
    data_test, meta_test_df = get_test_full_data(
        dataset, datasetdir, modalities, residualize=False)
    meta_test_df["asd"] = meta_test_df["asd"].apply(lambda x: x - 1)
    meta_train_df["asd"] = meta_train_df["asd"].apply(lambda x: x - 1)
    print_text([(key, arr.shape) for key, arr in data_test.items()])
    print_text(meta_test_df)
    print_text([(key, arr.shape) for key, arr in data_train.items()])
    print_text(meta_train_df)
    meta_test_file = os.path.join(
        benchdir, f"latent_meta_test_{dataset}.tsv")
    meta_train_file = os.path.join(
        benchdir, f"latent_meta_train_{dataset}.tsv")
    meta_test_df.to_csv(meta_test_file, sep="\t", index=False)
    meta_train_df.to_csv(meta_train_file, sep="\t", index=False)
    print_result(f"train metadata: {meta_train_file}")
    print_result(f"test metadata: {meta_test_file}")

    print_subtitle("Training a classification model...")
    print_text("model: LogisticRegression")
    X_train, X_test = (data_train["rois"].numpy(), data_test["rois"].numpy())
    y_train, y_test = (meta_train_df.asd.values, meta_test_df.asd.values)
    y_train = y_train.astype(int)
    y_test = y_test.astype(int)
    X_train = X_train
    X_test = X_test
    print(f"train: {X_train.shape} - {y_train.shape}")
    print(f"test: {X_test.shape} - {y_test.shape}")
    logreg = LogisticRegression(max_iter=400, solver="saga", l1_ratio=0.5)
    parameters = {
        "penalty": ["l2", "l1", "elasticnet"],
        "C": np.logspace(-4, 1, 6)}
    clf = GridSearchCV(logreg, parameters, cv=5, scoring="balanced_accuracy",
                       return_train_score=True, n_jobs=-1)
    clf.fit(X_train, y_train)
    print("Tuned Hyperparameters:", clf.best_params_)
    print("Accuracy (CV validation):", clf.best_score_)
    logreg = clf.best_estimator_
    score = logreg.score(X_test, y_test)
    print("Accuracy (test):", score)


#!/usr/bin/env python

from __future__ import print_function
from lofarstation.stationdata import ACCData
from casacore.measures import measures
import glob
import sys
import numpy as np

CasA = measures().direction("J2000", "23h23m26s", "+58d48m00s")
#CygA = measures().direction("J2000", "19h59m28.3566s", "+40d44m02.096s")

def beam_from_acc_dir(accdir, rcu, outnpy):
    dyn_spec = []
    for accfile in sorted(glob.glob("{}/*_acc_*.dat".format(accdir))):
        acc = ACCData(accfile, station_name="SE607", rcu_mode=rcu, direction=CasA)
        dyn_spec.append(acc.data.reshape((acc.data.shape[0],-1)).mean(axis=1).real)
        print(accfile)
    dyn_spec = np.array(dyn_spec)
    np.save(outnpy, dyn_spec)

def main():
    if len(sys.argv) != 4:
        print("Usage: {} <acc_directory> <rcu> <outnpy>".format(sys.argv[0]), file=sys.stderr)
        sys.exit()
    dirname, rcu, outfile = sys.argv[1:]
    rcu = int(rcu)
    beam_from_acc_dir(dirname, rcu, outfile)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Sep 28 11:45:54 2022

@author: Mathieu
"""

import pandas as pd
import math
import numpy as np
import itertools as ite

def haversine(coord1, coord2):
    R = 6372800  # Earth radius in meters
    lat1, lon1 = coord1
    lat2, lon2 = coord2

    phi1, phi2 = math.radians(lat1), math.radians(lat2)
    dphi       = math.radians(lat2 - lat1)
    dlambda    = math.radians(lon2 - lon1)

    a = math.sin(dphi/2)**2 + \
        math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2

    return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))

ville = pd.read_csv('C:/Users/Mathieu/Desktop/Villes_marchant.csv')

ville_tab = pd.DataFrame(columns=('Ville', 'Latitude', 'Longitude'))

for i in ville.index:
    information = ville.iloc[i].str.split(';')
    data = pd.DataFrame({'Ville':information[0][0], 'Latitude':information[0][5], 'Longitude':information[0][6]}, index=(0, 1, 2))
    ville_tab = pd.concat([ville_tab, data[0:1]], ignore_index=True)

ville_tab['Latitude'] = ville_tab['Latitude'].astype(float)
ville_tab['Longitude'] = ville_tab['Longitude'].astype(float)

distance = round(haversine(ville_tab.iloc[1][[1,2]].values, ville_tab.iloc[0][[1,2]].values)/1000)
print("L'ecart entre", ville_tab.iloc[1][0], "et" , ville_tab.iloc[0][0], "est de",  distance, 'km')

####################################################################################################################
#meilleur chemin pour les 5 meilleures villes méthode explosion combinatoire

print(ville_tab[0:5])

test_distance = pd.DataFrame(columns=('Ville1', 'Ville2', 'Distance'), index=np.arange(0, 5*5))

print(ville_tab['Ville'][0])
for j in ville_tab.index:
    for i in ville_tab.index:
        test_distance['Ville1'][j] = ville_tab['Ville'][j]
        test_distance['Ville2'][j] = ville_tab['Ville'][i]
        test_distance['Distance'][j] = round(haversine(ville_tab.iloc[j][[1,2]].values, ville_tab.iloc[i][[1,2]].values)/1000)

test_distance['Distance'] = test_distance['Distance'].replace(0, np.nan).dropna()
#print(test_distance['Distance'])
print(test_distance[test_distance['Distance'] == min(test_distance['Distance'])])

distance = 20000000

liste = np.arange(0, len(ville_tab[0:5]))

permut = list(ite.permutations(liste))

df = pd.DataFrame({'Trajet':permut, 'Distance':1})
for i in range(len(df)):
    d = 0
    k = df['Trajet'][i]
    for j in k:
        if(j == k[-1]):
           d = d + round(haversine(ville_tab.iloc[j][[1,2]].values, ville_tab.iloc[k[0]][[1,2]].values)/1000)
        else:
            d = d + round(haversine(ville_tab.iloc[j][[1,2]].values, ville_tab.iloc[j+1][[1,2]].values)/1000)
    df['Distance'][i] = d + round(haversine(ville_tab.iloc[k[-1]][[1,2]].values, ville_tab.iloc[k[0]][[1,2]].values)/1000)
print(df[df['Distance'] == min(df['Distance'])])

for i in permut:
    d = 0
    for j in i:
        if(j == i[-1]):
           d = d + round(haversine(ville_tab.iloc[j][[1,2]].values, ville_tab.iloc[i[0]][[1,2]].values)/1000)
        else:
            d = d + round(haversine(ville_tab.iloc[j][[1,2]].values, ville_tab.iloc[j+1][[1,2]].values)/1000)

    if(d<=distance):
        distance = d
        trajet = i

distance = distance #+ round(haversine(ville_tab.iloc[trajet[-1]][[1,2]].values, ville_tab.iloc[trajet[0]][[1,2]].values)/1000)

print(trajet, distance)
print(trajet[-1], trajet[0], round(haversine(ville_tab.iloc[trajet[-1]][[1,2]].values, ville_tab.iloc[trajet[0]][[1,2]].values)/1000) )

d = 0.0
print(type(d))

j = (0, 4, 3, 2, 1)
for k in range(len(j)):
        if(j[k] == j[-1]):
           d = d + round(haversine(ville_tab.iloc[j[-1]][[1,2]].values, ville_tab.iloc[j[0]][[1,2]].values)/1000)
           print(ville_tab['Ville'][j[-1]], ville_tab['Ville'][j[0]])
        else:
            d = d + round(haversine(ville_tab.iloc[j[k]][[1,2]].values, ville_tab.iloc[j[k+1]][[1,2]].values)/1000)
            print( ville_tab['Ville'][j[k]], ville_tab['Ville'][j[k+1]])
        print(d)

print(d)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 12 15:43:06 2017

@author: amine
"""

#data preprocessing

#importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

#importing the dataset
dataset = pd.read_csv('kddcup.data_10_percent_corrected')

#change Multi-class to binary-class
dataset['normal.'] = dataset['normal.'].replace(['back.', 'buffer_overflow.', 'ftp_write.', 'guess_passwd.', 'imap.', 'ipsweep.', 'land.', 'loadmodule.', 'multihop.', 'neptune.', 'nmap.', 'perl.', 'phf.', 'pod.', 'portsweep.', 'rootkit.', 'satan.', 'smurf.', 'spy.', 'teardrop.', 'warezclient.', 'warezmaster.'], 'attack')

x = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 41].values

#encoding categorical data
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_x_1 = LabelEncoder()
labelencoder_x_2 = LabelEncoder()
labelencoder_x_3 = LabelEncoder()
x[:, 1] = labelencoder_x_1.fit_transform(x[:, 1])
x[:, 2] = labelencoder_x_2.fit_transform(x[:, 2])
x[:, 3] = labelencoder_x_3.fit_transform(x[:, 3])
onehotencoder_1 = OneHotEncoder(categorical_features = [1])
x = onehotencoder_1.fit_transform(x).toarray()
onehotencoder_2 = OneHotEncoder(categorical_features = [4])
x = onehotencoder_2.fit_transform(x).toarray()
onehotencoder_3 = OneHotEncoder(categorical_features = [70])
x = onehotencoder_3.fit_transform(x).toarray()
labelencoder_y = LabelEncoder()
y = labelencoder_y.fit_transform(y)

#splitting the dataset into the training set and test set
from sklearn.cross_validation import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.3, random_state = 0)

#feature scaling
from sklearn.preprocessing import StandardScaler
sc_x = StandardScaler()
x_train = sc_x.fit_transform(x_train)
x_test = sc_x.transform(x_test)

# Importing the Keras libraries and packages
import keras
from keras.models import Sequential
from keras.layers import Dense

# Initialising the ANN
classifier = Sequential()

# Adding the input layer and the first hidden layer
classifier.add(Dense(output_dim = 60, init = 'uniform', activation = 'relu', input_dim = 118))

#Adding a second hidden layer
classifier.add(Dense(output_dim = 60, init = 'uniform', activation = 'relu'))

#Adding a third hidden layer
classifier.add(Dense(output_dim = 60, init = 'uniform', activation = 'relu'))

# Adding the output layer
classifier.add(Dense(output_dim = 1, init = 'uniform', activation = 'sigmoid'))

# Compiling the ANN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

# Fitting the ANN to the Training set
classifier.fit(x_train, y_train, batch_size = 10, nb_epoch = 20)

# Predicting the Test set results
y_pred = classifier.predict(x_test)
y_pred = (y_pred > 0.5)

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)

#the performance of the classification model
print("the Accuracy is: "+ str((cm[0,0]+cm[1,1])/(cm[0,0]+cm[0,1]+cm[1,0]+cm[1,1])))
recall = cm[1,1]/(cm[0,1]+cm[1,1])
print("Recall is : "+ str(recall))
print("False Positive rate: "+ str(cm[1,0]/(cm[0,0]+cm[1,0])))
precision = cm[1,1]/(cm[1,0]+cm[1,1])
print("Precision is: "+ str(precision))
print("F-measure is: "+ str(2*((precision*recall)/(precision+recall))))
from math import log
print("Entropy is: "+ str(-precision*log(precision)))


import random

# Board dimensions
BOARD_SIZE = 8

# Ship configurations
SHIP_LENGTHS = [2, 3, 4]
NUM_SHIPS = len(SHIP_LENGTHS)

# Function to initialize the game board
def initialize_board():
    board = []
    for _ in range(BOARD_SIZE):
        row = []
        for _ in range(BOARD_SIZE):
            row.append('O')
        board.append(row)
    return board

# Function to randomly place ships on the board
def place_ships(board):
    ships = []
    for length in SHIP_LENGTHS:
        ship = []
        while True:
            x = random.randint(0, BOARD_SIZE - 1)
            y = random.randint(0, BOARD_SIZE - 1)
            orientation = random.choice(['horizontal', 'vertical'])
            positions = []

            if orientation == 'horizontal':
                if y + length <= BOARD_SIZE:
                    positions = [(x, y+i) for i in range(length)]
            else:
                if x + length <= BOARD_SIZE:
                    positions = [(x+i, y) for i in range(length)]

            if positions and all(board[x][y] == 'O' for x, y in positions):
                ship = positions
                break

        for x, y in positions:
            board[x][y] = 'S'
        ships.append(ship)
    return ships

# Function to print the board
def print_board(board):
    print('   ' + ' '.join([str(i) for i in range(BOARD_SIZE)]))
    print('  ' + '-' * (BOARD_SIZE * 2 + 1))
    for i in range(BOARD_SIZE):
        print(str(i) + ' | ' + ' '.join(board[i]))

# Function to check if all ships have been sunk
def check_game_over(ships):
    for ship in ships:
        if ship:
            return False
    return True

# Main game loop
def play_battleship():
    board = initialize_board()
    ships = place_ships(board)

    print("Welcome to Battleship!")
    print("Sink all the ships to win the game.")

    while True:
        print("\n====================================")
        print_board(board)
        print("Take a shot!")
        x = input("Enter the X coordinate (0-7):\n")
        y = input("Enter the Y coordinate (0-7):\n")

        if not x.isdigit() or not y.isdigit():
            print("Error: Invalid coordinates. Please enter numeric values.")
            continue

        x = int(x)
        y = int(y)

        if x < 0 or x >= BOARD_SIZE or y < 0 or y >= BOARD_SIZE:
            print("Error: Invalid coordinates. Please enter values within the board range (0-8).")
            continue

        if board[x][y] == 'S':
            print("Hit!")
            board[x][y] = 'X'
            for ship in ships:
                if (x, y) in ship:
                    ship.remove((x, y))
                    if not ship:
                        ships.remove(ship)
                        print("You sunk a ship!")
                        if check_game_over(ships):
                            print("Congratulations! You won!")
                            return
        elif board[x][y] == 'X':
            print("You've already hit that spot!")
        else:
            print("Miss!")
            board[x][y] = 'M'

# Start the game
play_battleship()


import argparse

from mp3icloud.main import main

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Tiny script for audio ID3v2 metatag normalization, which allows imported to Apple Music Library files preserve sort order.')

    parser.add_argument('directory', type=str, help='Directory with mp3 files. E.g. "/home/f1nnix/alice_audiobook"')
    parser.add_argument('artist', type=str, help='Artist set for all tracks. E.g. "Lewis Carroll"')
    parser.add_argument('album', type=str, help='Album to set for all books, e.g. "Alice in Wonderland"')
    parser.add_argument('genre', type=str, help='Genre to set for all books, e.g. "Audiobook"')
    parser.add_argument('--use-filenames', dest='use_filenames', action='store_true', default=False,
                        help="Preserve original filename as title tag")

    args = parser.parse_args()

    main(args)


from __future__ import with_statement
import re
import os
from dotenv import load_dotenv
import discord

def customLobbyMenu(author, gameId):
    view = discord.ui.View()
    view.add_item(discord.ui.Button(label="Join "+str(author)+"\'s game", url=f"https://www.aoe2lobby.com/j/{gameId}"))
    #view.add_item(discord.ui.Button(label="Join "+str(author)+"\'s game", url="http://"+str(os.getenv('SERVERHTTP'))+"?gameId="+str(gameId)+"&spectate=0"))
    view.add_item(discord.ui.Button(label="Spectate", url=f"https://www.aoe2lobby.com/w/{gameId}"))
    #view.add_item(discord.ui.Button(label="Spectate", url="http://"+str(os.getenv('SERVERHTTP'))+"?gameId="+str(gameId)+"&spectate=1"))
    return view

def isCommand(message):
    aoe2de_pattern = re.compile('''aoe2de:\/\/0\/[0-9]+''')
    return aoe2de_pattern.findall(message.content)

async def respond(message):
        try:
            matchId = re.compile('[0-9][0-9][0-9][0-9]+').findall(message.content)[0]
        except IndexError as exc:
            return
        await message.channel.send(view = customLobbyMenu(message.author.name, matchId))


import sys
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5 import uic
import datetime

from_class = uic.loadUiType("Test7.ui")[0]

class WindowClass(QMainWindow, from_class):
    def __init__(self):
        super().__init__()
        self.setupUi(self)

        self.tableWidget.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)
        self.btnAdd.clicked.connect(self.Add)

    def Add(self):
        row = self.tableWidget.rowCount()
        self.tableWidget.insertRow(row)
        self.tableWidget.setItem(row, 0, QTableWidgetItem(self.editName.text()))
        self.tableWidget.setItem(row, 1, QTableWidgetItem(self.editGender.text()))
        self.tableWidget.setItem(row, 2, QTableWidgetItem(self.editBirthday.text()))

if __name__ == "__main__":
    app = QApplication(sys.argv)
    myWindows = WindowClass()
    myWindows.show()

    sys.exit(app.exec_())


from flask import Flask, request, render_template
import sys

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/move')
def move():
    data = request.args.get('data')
    print(
        'MOVE from:{} data:{}'.format(request.remote_addr, data),
        file=sys.stderr
    )
    return ('', 204)

if __name__ == '__main__':
    app.run(debug=True)


from django.urls import path
from rest_framework_jwt.views import (
    obtain_jwt_token as create_token,
    refresh_jwt_token as refresh_token,
    verify_jwt_token as verify_token
)
from . import views

urlpatterns = [
    path('register_user', views.RegisterUser.as_view(), name='register_user'),
    path('users', views.UserListView.as_view(), name='users_list'),
    path('create_jwt_token', create_token, name='create_jwt_token'),
    path('refresh_jwt_token', refresh_token, name='refresh_jwt_token'),
    path('verify_jwt_token', verify_token, name='verify_jwt_token')
]


import pygame
from pygame.sprite import Sprite

class Ship(Sprite):
    def __init__(self, ai_settings, screen):
        super(Ship, self).__init__()
        self.screen = screen
        self.ai_settings = ai_settings
        # Загрузка изображения корабля и получение прямоугольника.
        self.image = pygame.image.load('images/ship.bmp')
        self.rect = self.image.get_rect()
        self.screen_rect = screen.get_rect()
        # Каждый новый корабль появляется у нижнего края экрана.
        self.rect.centerx = self.screen_rect.centerx
        self.rect.bottom = self.screen_rect.bottom
        self.center = float(self.rect.centerx)

        # Флаг перемещения
        self.moving_right = False
        self.moving_left = False

    def update(self):
        if self.moving_right and self.rect.right < self.screen_rect.right:
            self.center += self.ai_settings.ship_speed_factor
        if self.moving_left and self.rect.left > 0:
            self.center -= self.ai_settings.ship_speed_factor

        self.rect.centerx = self.center

    def blitme(self):
        self.screen.blit(self.image, self.rect)

    def center_ship(self):
        # Размещает корабль в центре нижней стороны
        self.center = self.screen_rect.centerx


from datetime import datetime
from django.db.models import FilteredRelation, Q, F
from .utils import MemberHistoryLine
from .models import MemberTransaction, Account, AccountType, Aid, Meeting

def get_accounts(*, used_for):
    if used_for.upper() == "MEMBER":
        used_for_list = [Account.MEMBER, Account.BOTH]
    elif used_for.upper() == "ORG":
        used_for_list = [Account.ORG, Account.BOTH]
    else:
        raise ValueError("Wrong argument value. Only Member and Org are allowed!")
    return [
        account.title
        for account in (
            Account.objects.filter(used_for__in=used_for_list).only("title").all()
        )
    ]

class MemberFinance(object):
    def __init__(self, email):
        qs = (
            MemberTransaction.objects.select_related(
                "meeting",
                "member",
                "member__user",
                "account",
            )
            .filter(member__user__email=email)
            .only(
                "meeting__date",
                "account__title",
                "amount",
                "member__id",
                "member__user",
            )
            .all()
        )

        self._member_accounts = [
            AccountType.ASSISTANCE,
            AccountType.SAVINGS,
            AccountType.SCHOLAR_SAVINGS,
            AccountType.PROJECT,
            AccountType.SANCTION,
        ]

        # Logged In Member
        self._member = qs[0].member
        # Transactions
        self._transactions = qs
        # Member's aids
        self._aids = self._member.aids.all() if qs else []

        self._date_format = "%m/%m/%Y"

        # History Table
        history = list(
            Meeting.objects.annotate(
                A=FilteredRelation(
                    "absences", condition=Q(absences__member=self._member.id)
                ),
                S=FilteredRelation(
                    "sanctions", condition=Q(sanctions__member=self._member.id)
                ),
            )
            .filter(date__lte=datetime.today())
            .values(
                "date",
                "A__absence_reason",
                "A__justified",
                "A__sanctioned",
                "S__sanction_reason",
                "S__amount",
                "S__sanction_type",
                "S__title",
                "S__executed_or_paid",
            )
            .all()
        )

        self._history = {}

        for line in history:
            date = line["date"].strftime(self._date_format)
            values = {
                key.split("__")[1]: (value or None)
                for key, value in line.items()
                if key != "date"
            }
            self._history[date] = values

    @property
    def account_balances(self):
        balances = {title: 0.0 for title in self._member_accounts}
        for transaction in self._transactions:
            balances[transaction.account.title] += float(transaction.amount)
        balances["absences"] = len(
            [1 for _, line in self._history.items() if line["absence_reason"]]
        )
        balances["assistance_aid"] = len(
            [1 for aid in self._aids if aid.aid_type == Aid.ASSISTANCE]
        )
        balances["happiness_aid"] = len(
            [1 for aid in self._aids if aid.aid_type == Aid.HAPPINESS]
        )
        return balances

    @property
    def history(self):
        date_format = "%m/%m/%Y"
        base_dict = {title: 0 for title in self._member_accounts}
        base_dict.update(
            {
                "absence": False,
                "absence_reason": "",
                "justified": "",
                "sanctioned": "",
                "sanction_reason": "",
                "amount": "",
                "sanction_type": "",
                "title": "",
                "executed_or_paid": "",
            }
        )
        for transaction in self._transactions:
            date = transaction.meeting.date.strftime(self._date_format)
            if date not in self._history:
                # Normally, this must never happen
                self._history[date] = base_dict.copy()
            self._history[date][transaction.account.title] = float(transaction.amount)

        return (
            self._history.items()
        )  # [MemberHistoryLine(date=date, **values) for date, values in his.items()]

"""
from ade4.selectors import MemberFinance as MF
m = MF(email='johnsmith@f4.com')
q = m.queryset
m.account_balances

for date, vals in d.items():
    print(date)
    for c, v in vals.items():
        print('\t',c,': ',v)

me = Meeting.objects.filter(date__lte=datetime(2022, 6, 21)).all().values('date', 'absences').extra(where=[f"member_id={member.id} OR member_id IS NULL"]).all()
me = Meeting.objects.filter(date__lte=datetime.today()).all().values('date', 'absences').extra(where=[f"member_id={member.id} OR member_id IS NULL"]).all()

SELECT "ade4_meeting"."date", "ade4_absence"."reason", "ade4_absence"."member_id"
FROM "ade4_meeting" LEFT OUTER JOIN "ade4_absence" ON
    ("ade4_meeting"."id" = "ade4_absence"."meeting_id")
WHERE ("ade4_meeting"."date" < 2022-06-21 AND (member_id=1 OR member_id IS NULL))
ORDER BY "ade4_meeting"."date" DESC

me = Meeting.objects.annotate(
pizzas_vegetarian=FilteredRelation(
'absences__member_id', condition=Q(absences__member_id=member.id),)
).filter(date__lte=datetime.today()).all().values('date', 'absences').all()

me = Meeting.objects.annotate(
    A=FilteredRelation('absences', condition=Q(absences__member=member.id))
).filter(date__lte=datetime.today()).all().values('date', 'A__reason').all()

me = Meeting.objects.annotate(
    A=FilteredRelation('absences', condition=Q(absences__member=member.id)),
    S=FilteredRelation('sanctions', condition=Q(sanctions__member=member.id))
).filter(date__lte=datetime.today()).all().values('date', 'A__reason', 'S__amount').all()

me = Meeting.objects.annotate(
    A=FilteredRelation('absences', condition=Q(absences__member=member.id)),
    S=FilteredRelation('sanctions', condition=Q(sanctions__member=member.id))
).filter(date__lte=datetime.today()).values('date', 'A__reason', 'S__amount').annotate(
    reason=F('A__reason'), amount=F('S__amount')
)

"""


from tensorflow import python as tf
import numpy as np
import os
import matplotlib.pyplot as plt
from random import shuffle
from label_vehicles import label_vehicle
import pandas as pd
import glob
import cv2
from preprocessing import preprocessing
from rotate import rotate
from rotate_horizontal import rotate_horizontal_and_resize
from preparation import create_mask_of_tag
from load_model import pretty_cm, transform_image, evaluation_indices, report

def main(path):
    print("Working on it, please have patience")
    src_dir = 'data2/' + path
    df = pd.read_csv('data2/val_csv.csv', sep=',')
    br = 0

    for filename in glob.glob(os.path.join(src_dir, '*.jpg')):
        im = cv2.imread(filename)
        name = filename.replace(src_dir, '')
        img_name = name.replace('.jpg', '')
        img_name = img_name.replace('\\', '')
        preprocessed = preprocessing(im, img_name, df)
        create_mask_of_tag(preprocessed, img_name, df)
        br += 1
        print(br)
    src_dir = 'data2/masked_val/small vehicle'
    dst_dir = 'data2/masked_rotated_h_val/'
    for filename in glob.glob(os.path.join(src_dir, '*.jpg')):
        im = cv2.imread(filename)
        name = filename.replace(src_dir, '')
        tag_name = name.replace('.jpg', '')
        tag_name = tag_name.replace('\\', '')
        rot = rotate(im, tag_name, df)
        rot_h = rotate_horizontal_and_resize(rot, (30, 75))
        tag_id_new = int(tag_name)
        selection = df[df.tag_id == tag_id_new]

        for idx, row in selection.iterrows():
            if str(row.general_class) == "small vehicle":
                add_dst = 'small vehicle/'
                if str(row.sub_class) == "sedan":
                    cv2.imwrite(dst_dir + add_dst + 'sedan/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "hatchback":
                    cv2.imwrite(dst_dir + add_dst + 'hatchback/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "minivan":
                    cv2.imwrite(dst_dir + add_dst + 'minivan/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "van":
                    cv2.imwrite(dst_dir + add_dst + 'van/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "jeep":
                    cv2.imwrite(dst_dir + add_dst + 'jeep/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "pickup":
                    cv2.imwrite(dst_dir + add_dst + 'pickup/' + str(row.tag_id) + '.jpg', rot_h)
        br += 1
        print(br)

    # preparing data for predictions
    size = (30, 75)
    X_eval = list()
    y_eval = list()
    X_tag = list()

    # hatchback
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/hatchback/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/hatchback/' + files[i], size))
        y_eval.append(0)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # jeep
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/jeep/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/jeep/' + files[i], size))
        y_eval.append(1)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # minivan
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/minivan/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/minivan/' + files[i], size))
        y_eval.append(2)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # pickup
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/pickup/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/pickup/' + files[i], size))
        y_eval.append(3)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # sedan
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/sedan/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/sedan/' + files[i], size))
        y_eval.append(4)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # van
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/van/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/van/' + files[i], size))
        y_eval.append(5)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # stacking the arrays
    X_eval = np.vstack(X_eval)

    labels_index = {0: "hatchback", 1: "jeep", 2: "minivan", 3: "pickup", 4: "sedan", 5: "van"}

    # load the model
    cnn_classifier = tf.keras.models.load_model('data2/vehicle_classification_model_dropout.h5')

    cnn_pred = cnn_classifier.predict_classes(X_eval, batch_size=32)

    pretty_cm(cnn_pred, y_eval, labels_index)
    correctly_classified_indices, misclassified_indices = evaluation_indices(cnn_pred, y_eval)

    plt.figure(figsize=(36, 6))
    shuffle(correctly_classified_indices)
    plt.show()

    for plot_index, good_index in enumerate(correctly_classified_indices[0:5]):
        plt.subplot(1, 5, plot_index + 1)
        plt.imshow(X_eval[good_index])
        plt.title('Predicted: {}, Actual: {}'.format(labels_index[cnn_pred[good_index]],
                                                     labels_index[y_eval[good_index]]), fontsize=5)
    plt.show()

    print(X_tag)
    print(list(cnn_pred))
    print(y_eval)
    predicted_dict = dict(zip(X_tag, list(cnn_pred)))
    actual_dict = dict(zip(X_tag, y_eval))

    print(len(X_tag))
    print(len(list(cnn_pred)))
    print(len(y_eval))

    df = pd.read_csv('data2/val_csv.csv', sep=',')
    path = 'data2/val'
    src_dir = path
    dst_dir = 'data2/labeled_predicted_val'
    for filename in glob.glob(os.path.join(src_dir, '*.jpg')):
        im = cv2.imread(filename)
        name = filename.replace(src_dir, '')
        img_name = name.replace('.jpg', '')
        img_name = img_name.replace('\\', '')
        labeled_img_p = label_vehicle(im.copy(), img_name, df, predicted_dict)
        cv2.imwrite(dst_dir + name, labeled_img_p)
        labeled_img_a = label_vehicle(im.copy(), img_name, df, actual_dict)
        cv2.imwrite('data2/labeled_actual_val' + name, labeled_img_a)

    report(y_eval, list(cnn_pred), labels_index)
    print("Done")

if __name__ == "__main__":
    main('val')


# coding: utf-8

# ## Build a simple trading strategy

# In[3]:

import pandas as pd
import matplotlib.pyplot as plt
get_ipython().magic('matplotlib inline')

# ### 1. Munging the stock data and add two columns - MA10 and MA50

# In[4]:

import pandas as pd
import matplotlib.pyplot as plt
get_ipython().magic('matplotlib inline')

#import MS's stock data, add two columns - MA10 and MA50
#use dropna to remove any "Not a Number" data
ms = pd.DataFrame.from_csv('../data/microsoft.csv')
ms['MA10'] = ms['Close'].rolling(10).mean()
ms['MA50'] = ms['Close'].rolling(50).mean()
ms = ms.dropna()
ms.head()

# ### 2. Add "Shares" column to make decisions base on the strategy

# In[7]:

#Add a new column "Shares", if MA10>MA50, denote as 1 (long one share of stock), otherwise, denote as 0 (do nothing)

ms['Shares'] = [1 if ms.loc[ei, 'MA10']>ms.loc[ei, 'MA50'] else 0 for ei in ms.index]

# In[8]:

#Add a new column "Profit" using List Comprehension, for any rows in fb, if Shares=1, the profit is calculated as the close price of
#tomorrow - the close price of today. Otherwise the profit is 0.

#Plot a graph to show the Profit/Loss

ms['Close1'] = ms['Close'].shift(-1)
ms['Profit'] = [ms.loc[ei, 'Close1'] - ms.loc[ei, 'Close'] if ms.loc[ei, 'Shares']==1 else 0 for ei in ms.index]
ms['Profit'].plot()
plt.axhline(y=0, color='red')

# ### 3. Use .cumsum() to display our model's performance if we follow the strategy

# In[10]:

#Use .cumsum() to calculate the accumulated wealth over the period

ms['wealth'] = ms['Profit'].cumsum()
ms.tail()

# In[11]:

#plot the wealth to show the growth of profit over the period

ms['wealth'].plot()
plt.title('Total money you win is {}'.format(ms.loc[ms.index[-2], 'wealth']))

# ## You can create your own simple trading strategy by copying the codes above and modify the codes accordingly using the data of Microsoft (microsoft.csv).


import os
import cv2
import json
import numpy as np
from os.path import join
from shutil import copyfile
from shapely.geometry import Point, LineString, Polygon
from shapely.ops import linemerge, unary_union, polygonize

from src.utils import filename_without_ext

from mosaic.mosaic_api import SaltData
from mosaic.math_fit import fit_third_order

MOSAIC_PATH = '/workdir/data/mosaic/pazzles_6013.csv'

left = 0
right = 1
top = 2
bottom = 3
img_side_len = 101

BIG_NUMBER = 1000
SMALL_NUMBER = 1e-3

def find_points(mask, x_shift=0, y_shift=0):
    # Find points where mask change class on edges
    mask = mask > 0
    mask = mask.astype(np.int)
    n = mask.shape[1]
    edges = [mask[:, 0+x_shift], mask[:, -1-x_shift],
             mask[0+y_shift, :], mask[-1-y_shift, :]]
    diffs = [np.diff(edge, n=1) for edge in edges]
    pos = [np.argwhere(diff>0)+1 for diff in diffs]
    neg = [np.argwhere(diff<0)+1 for diff in diffs]
    pos = [[int(x) for x in p] for p in pos]
    neg = [[int(x) for x in n] for n in neg]
    if mask[0, 0] > 0:
        for i in [left, top]:
            pos[i] = [0] + pos[i]
    if mask[-1, 0] > 0:
        pos[bottom] = [0] + pos[bottom]
        neg[left] = [n] + neg[left]
    if mask[0, -1] > 0:
        pos[right] = [0] + pos[right]
        neg[top] = [n] + neg[top]
    if mask[-1, -1] > 0:
        for i in [right, bottom]:
            neg[i] = [n] + neg[i]
    return(pos, neg)

def is_closed(pos_s, neg_s):
    return len(pos_s) == 1 and pos_s[0] == 0\
            and len(neg_s) == 1 and neg_s[0] == img_side_len

def is_filled(pos, neg):
    return all([is_closed(p, n) for (p, n) in zip(pos, neg)])

def is_empty(mask):
    return np.max(mask) == 0

def is_side_closed(pos, neg, side):
    return is_closed(pos[side], neg[side])

def len_side_closed(mask, side):
    if side == left:
        s = mask[:, 0]
    elif side == right:
        s = mask[:, -1]
    elif side == top:
        s = mask[0, :]
    else:
        s = mask[-1, :]
    return np.count_nonzero(s)

def is_terminal(mask):
    # Check if the given mask is a terminal tile
    return len_side_closed(mask, top) < len_side_closed(mask, bottom)/2

def is_lines(pos, neg):
    # Check if the given mask contains only verlical lines
    return pos[top] == pos[bottom] and neg[top] == neg[bottom]

def can_be_matched(top_tile, bottom_tile):
    # Check if two masks can be connected with lines
    top_tile_pos, top_tile_neg = top_tile
    bottom_tile_pos, bottom_tile_neg = bottom_tile
    return len(bottom_tile_pos[top]) == len(top_tile_pos[bottom])\
            and len(bottom_tile_neg[top]) == len(top_tile_neg[bottom])

def tile_diff(tile, idx, side, pos_type=True, depth=2):
    # Calculate derivative on tiles edges
    pos, neg = find_points(tile)
    pos_crop, neg_crop = find_points(tile, x_shift=depth, y_shift=depth)
    if side in [left, right]:
        dx = depth
        if pos_type:
            if len(pos_crop[side]) != len(pos[side]):
                return SMALL_NUMBER
            dy = pos_crop[side][idx] - pos[side][idx]
        else:
            if len(neg_crop[side]) != len(neg[side]):
                return SMALL_NUMBER
            dy = neg_crop[side][idx] - neg[side][idx]
    else:
        dy = depth
        if pos_type:
            if len(pos_crop[side]) != len(pos[side]):
                return BIG_NUMBER
            dx = pos_crop[side][idx] - pos[side][idx]
        else:
            if len(neg_crop[side]) != len(neg[side]):
                return BIG_NUMBER
            dx = neg_crop[side][idx] - neg[side][idx]
    if dx == 0:
        return SMALL_NUMBER
    else:
        return dy/dx

def poly_3(a, b, c, d, x):
    return x*(x*(a*x+b)+c)+d

def can_be_corner(tile1, tile2, left):
    '''Check if two tiles in a rectangular pattern can have a missed corner between

    The tiles relation (left=False):

    |corn?|tile2| Where corn? - examined tile.
    |tile1|-----| The scheme is reflected in case left=True
    '''
    pos1, neg1 = find_points(tile1)  # points of the bottom-center tile
    pos2, neg2 = find_points(tile2)  # points of the top-side tile
    ret = None
    if left:
        x_match = img_side_len + 1
        k_x = 0
        for k, x in enumerate(neg1[top]):
            if 0 < x < img_side_len and x < x_match:
                x_match = x
                k_x = k
        y_match = -1
        k_y = 0
        for k, y in enumerate(pos2[right]):
            if 0 < y < img_side_len and y > y_match:
                y_match = y
                k_y = k
        if x_match != img_side_len + 1 and y_match != -1:
            d1 = tile_diff(tile1, k_x, top)
            d2 = tile_diff(tile2, k_y, right)
            ret = [(img_side_len, x_match, d1), (y_match, 0, d2)]
    else:
        x_match = -1
        k_x = 0
        for k, x in enumerate(pos1[top]):
            if 0 < x < img_side_len and x > x_match:
                x_match = x
                k_x = k
        y_match = -1
        k_y = 0
        for k, y in enumerate(pos2[left]):
            if 0 < y < img_side_len and y > y_match:
                y_match = y
                k_y = k
        if x_match != -1 and y_match != -1:
            d1 = tile_diff(tile1, k_x, top)
            d2 = tile_diff(tile2, k_y, left)
            ret = [(img_side_len, x_match, d1),
                   (y_match, img_side_len, d2)]
    return ret

def pair_len(pair):
    return pair[1] - pair[0] - 1

def draw_v_pair(pair, strip, l):
    # Drow polligons between two masks from test
    if l > 0:
        roi_points = [(0, 0), (img_side_len, 0),
                      (img_side_len*l, img_side_len), (0, img_side_len)]
        roi_poly = Polygon(roi_points)
        top_tile = find_points(pair[0])
        bottom_tile = find_points(pair[1])
        top_tile_pos, top_tile_neg = top_tile
        bottom_tile_pos, bottom_tile_neg = bottom_tile
        v_shift = l * img_side_len

        square_points = [(0, 0), (img_side_len, 0), (img_side_len, v_shift), (0, v_shift)]
        square_poly = Polygon(square_points)
        lines = []
        for i in range(len(top_tile_pos[bottom])):
            line = LineString([(top_tile_pos[bottom][i], 0),
                               (bottom_tile_pos[top][i], v_shift),
                               (bottom_tile_neg[top][i], v_shift),
                               (top_tile_neg[bottom][i], 0)])
            lines.append(line)

        merged = linemerge([square_poly.boundary, *lines])
        borders = unary_union(merged)
        polygons = []
        for poly in polygonize(borders):
            polygons.append(poly)
        masks = [mask_for_polygons([polygons[i]], (v_shift, img_side_len))
                 for i in range(0, len(polygons), 2)]
        mask = (np.any(masks, axis=0)*255).astype(np.uint8)
        return mask
    return None

def divide(mask, ni, nj):
    # Divide a mask into 101x101 px tiles
    step_i = mask.shape[0]//ni
    step_j = mask.shape[1]//nj
    crops = [[[] for _j in range(nj)] for _i in range(ni)]
    for i in range(ni):
        for j in range(nj):
            crop = mask[i*step_i:(i+1)*step_i,
                        j*step_j:(j+1)*step_j]
            crops[i][j] = crop
    return crops

# From https://michhar.github.io/masks_to_polygons_and_back/
def mask_for_polygons(polygons, im_size):
    """Convert a polygon or multipolygon list back to
       an image mask ndarray"""
    img_mask = np.zeros(im_size, np.uint8)
    if not polygons:
        return img_mask
    # function to round and convert to int
    int_coords = lambda x: np.array(x).round().astype(np.int32)
    exteriors = [int_coords(poly.exterior.coords) for poly in polygons]
    interiors = [int_coords(pi.coords) for poly in polygons
                 for pi in poly.interiors]
    cv2.fillPoly(img_mask, exteriors, 255)
    cv2.fillPoly(img_mask, interiors, 0)
    return img_mask

def find_fit_corners(tile1, tile2, left):
    # Draw missed corners polygons
    c = can_be_corner(tile1, tile2, left)
    ret = None
    if c is not None:
        x1 = c[0][0]
        x2 = c[1][0]
        y1 = c[0][1]
        y2 = c[1][1]
        pol = fit_third_order(x1, x2, y1, y2, c[0][2], c[1][2])
        if pol is not None:
            pol_points = []
            if left:
                pol_points.append((min(y1,y2), min(x1, x2)))
            else:
                pol_points.append((min(y1,y2), max(x1, x2)))
            pol_points_a = []
            for x_i in range(min(x1, x2), max(x1, x2)+1):
                y_i = int(poly_3(*pol, x_i))
                pol_points_a.append((int(y_i), int(x_i)))
            if all([0<=x[0]<=img_side_len for x in pol_points_a])\
                    and all([0<=x[1]<=img_side_len for x in pol_points_a]):
                pol_points.extend(pol_points_a)
            if left:
                pol_points.append((max(y1, y2), max(x1, x2)))
                pol_points.append((img_side_len, 0))
            else:
                pol_points.append((max(y1,y2), min(x1, x2)))
                pol_points.append((img_side_len, img_side_len))

            square_points = [(0, 0), (img_side_len, 0),
                             (img_side_len, img_side_len),
                             (0, img_side_len)]
            square_poly = Polygon(square_points)
            line = LineString(pol_points)

            merged = linemerge([square_poly.boundary, line])
            borders = unary_union(merged)
            polygons = []
            for poly in polygonize(borders):
                polygons.append(poly)
            if len(polygons) > 1:
                return mask_for_polygons([polygons[1]],
                                         (img_side_len, img_side_len))

def fix_holes(mask):
    _, thresh = cv2.threshold(mask, 127, 255, 0)
    _, contour,hier = cv2.findContours(thresh, cv2.RETR_CCOMP,
                                       cv2.CHAIN_APPROX_SIMPLE)

    for cnt in contour:
        cv2.drawContours(mask, [cnt], 0, 255, -1)
    return mask

def postprocess(mean_prediction_dir, postprocessed_prediction_dir):
    '''Make the postprocessing

    Args:
        mean_prediction_dir (str): Path to a dir with mean predictions
        postprocessed_prediction_dir (str): Path to save postprocessed tiles
    '''

    def find_pairs(strip):
        # Return a list of indexes of train images pairs in a vertical strip
        pairs = []
        opening = None
        for i, s in enumerate(strip):
            if s[0] in saltdata.id2mask:
                if opening is not None:
                    pairs.append((opening, i))
                opening = i
        return pairs

    def pair_match(strip, pairs):
        # Return a list of matched pair images from train
        new_pairs = []
        for pair in pairs:
            top_tile = find_points(saltdata.id2mask[strip[pair[0]][0]])
            bottom_tile = find_points(saltdata.id2mask[strip[pair[1]][0]])
            if can_be_matched(top_tile, bottom_tile):
                new_pairs.append(pair)
        return new_pairs

    def find_pairs_pred(strip):
        # Return a list of indexes of train images pairs in a vertical strip
        pairs = []
        opening = None
        for i, s in enumerate(strip):
            if s[0] in saltdata.id2pred\
                    and np.count_nonzero(saltdata.id2pred[s[0]])>0:
                if opening is not None:
                    pairs.append((opening, i))
                opening = i
        return pairs

    def pair_match_pred(strip, pairs):
        # Return a list of matched pair images from train
        new_pairs = []
        for pair in pairs:
            if strip[pair[0]][0] in saltdata.id2pred_cor:
                top_tile = find_points(saltdata.id2pred_cor[strip[pair[0]][0]])
            else:
                top_tile = find_points(saltdata.id2pred[strip[pair[0]][0]])
            if strip[pair[1]][0] in saltdata.id2pred_cor:
                bottom_tile = find_points(saltdata.id2pred_cor[strip[pair[1]][0]])
            else:
                bottom_tile = find_points(saltdata.id2pred[strip[pair[1]][0]])
            if can_be_matched(top_tile, bottom_tile):
                new_pairs.append(pair)
        return new_pairs

    def process_strip(strip, term_level=-1):
        # Postprocess a vertical column of tiles
        pairs = pair_match(strip, find_pairs(strip))
        for pair in pairs:
            empty = [is_empty(saltdata.id2mask[strip[k][0]]) for k in pair]
            l = pair_len(pair)
            if l > 0:
                ids = [strip[i][0] for i in range(pair[0]+1, pair[1])]
                if not any(empty):
                    mask = draw_v_pair((saltdata.id2mask[strip[pair[0]][0]],
                                        saltdata.id2mask[strip[pair[1]][0]]), strip, l)
                    pred_cors = divide(mask, l, 1)
                    for i in range(l):
                        saltdata.add_pred_cor(ids[i], pred_cors[i][0])

        can_be_copied = [-1] * len(strip)
        for i, tile in enumerate(strip):
            tile_id = tile[0]
            if tile_id in saltdata.id2mask:
                tile_mask = saltdata.id2mask[tile_id]
                if is_lines(*find_points(tile_mask))\
                        and not is_empty(tile_mask):
                    for j in range(i+1, len(strip)):
                        if strip[j][0] not in saltdata.id2pred_cor:
                            can_be_copied[j] = i
        for i in range(len(can_be_copied)):
            if can_be_copied[i] >= 0:
                saltdata.add_pred_cor(strip[i][0], saltdata.id2mask[strip[can_be_copied[i]][0]])

        can_be_copied_up = [-1] * len(strip)
        for i in range(len(strip)-1, -1, -1):
            tile_id = strip[i][0]
            if tile_id in saltdata.id2mask:
                tile_mask = saltdata.id2mask[tile_id]
                if is_lines(*find_points(tile_mask))\
                        and not is_empty(tile_mask):
                    for j in range(i-1, term_level, -1):
                        is_term = False
                        j_id = strip[j][0]
                        if j_id in saltdata.id2pred and not is_empty(saltdata.id2pred[j_id]):
                            is_term = is_terminal(saltdata.id2pred[j_id])
                        elif j_id in saltdata.id2mask:
                            is_term = True
                        if j_id not in saltdata.id2pred_cor and not is_term:
                            can_be_copied_up[j] = i
                        else:
                            break

        for i in range(len(can_be_copied_up)):
            if can_be_copied_up[i] >= 0:
                saltdata.add_pred_cor(strip[i][0], saltdata.id2mask[strip[can_be_copied_up[i]][0]])
        pairs = pair_match_pred(strip, find_pairs_pred(strip))
        for pair in pairs:
            l = pair_len(pair)
            if l > 0:
                ids = [strip[i][0] for i in range(pair[0]+1, pair[1])]
                not_in_masks = [id not in saltdata.id2mask for id in ids]
                if all(not_in_masks):

                    m1 = get_tile_mask(strip[pair[0]][0])
                    m2 = get_tile_mask(strip[pair[1]][0])
                    if m1 is not None and m2 is not None:
                        mask = draw_v_pair((m1, m2), strip, l)
                        pred_cors = divide(mask, l, 1)
                        for i in range(l):
                            if ids[i] not in saltdata.id2mask:
                                saltdata.add_pred_cor(ids[i], pred_cors[i][0])

    def zero_under_salt(strip, term_level):
        # Set all tiles to zero under completely filled bottom edge of a tile
        can_be_filled = [False] * len(strip)
        for i in range(term_level, len(strip)):
            tile_id = strip[i][0]
            if tile_id in saltdata.id2mask:
                tile_mask = get_tile_mask(tile_id)
                if is_side_closed(*find_points(tile_mask), bottom)\
                        and not is_empty(tile_mask):
                    for j in range(i+1, len(strip)):
                        can_be_filled[j] = True

        for i in range(len(can_be_filled)):
            fill_id = strip[i][0]
            if can_be_filled[i] and fill_id in saltdata.id2pred:
                saltdata.add_pred_cor(fill_id, np.zeros((img_side_len, img_side_len), dtype=np.uint8))
            elif can_be_filled[i] and fill_id in saltdata.id2pred_cor:
                saltdata.add_pred_cor(fill_id, np.zeros((img_side_len, img_side_len), dtype=np.uint8))

    def get_tile_mask(tile_id):
        # Return mask by id wherever it is
        if tile_id in saltdata.id2mask:
            return saltdata.id2mask[tile_id]
        elif tile_id in saltdata.id2pred_cor:
            return saltdata.id2pred_cor[tile_id]
        elif tile_id in saltdata.id2pred:
            return saltdata.id2pred[tile_id]

    def process_mosaic(mosaic):
        # Process mosaics with 3 tiles or more
        if len(mosaic) > 2:
            # Find the terminal level
            is_term_mat = np.zeros_like(mosaic.array, dtype=np.bool)
            for i in range(mosaic.array.shape[0]):
                for j in range(mosaic.array.shape[1]):
                    tile_id = mosaic.array[i,j]
                    if tile_id is not None:
                        tile_id = tile_id
                        tile_mask = get_tile_mask(tile_id)
                        is_term_mat[i, j] = is_terminal(tile_mask)
            term_level = 0
            for i in range(mosaic.array.shape[0]):
                h_strip = mosaic.array[i,:][np.newaxis, :]
                is_term = [False] * h_strip.shape[1]
                for j, tile in enumerate(h_strip[0]):
                    if tile is not None:
                        tile_id = tile
                        if tile_id in saltdata.id2mask:
                            is_term[j] = (is_term_mat[i, j]\
                                          and np.count_nonzero(saltdata.id2mask[tile_id]) > (img_side_len**2)/20)
                        elif tile_id in saltdata.id2pred:
                            is_term[j] = (is_term_mat[i, j]\
                                          and np.count_nonzero(saltdata.id2pred[tile_id]) > (img_side_len**2)/20)
                if any(is_term):
                    term_level = i
                    break
            # Fix vertical columns of tiles
            for i in range(mosaic.array.shape[1]):
                strip = mosaic.array[:,i][:, np.newaxis]
                process_strip(strip, term_level)

            # Add missed corners
            for i in range(mosaic.array.shape[0]):
                for j in range(mosaic.array.shape[1]):
                    if True:#is_term_mat[i, j]:
                        if mosaic.array[i,j] is not None:
                            # Left
                            if i+1 < mosaic.array.shape[0] and j+1 < mosaic.array.shape[1]:

                                if mosaic.array[i+1, j+1] is not None:
                                    l_id = mosaic.array[i, j+1]
                                    if (l_id in saltdata.id2pred\
                                            and np.count_nonzero(get_tile_mask(l_id)) < img_side_len**2/1000)\
                                            or l_id in saltdata.id2pred_cor:

                                        tile1 = get_tile_mask(mosaic.array[i,j])
                                        tile2 = get_tile_mask(mosaic.array[i+1, j+1])

                                        fix = find_fit_corners(tile2, tile1, True)
                                        if fix is not None:
                                            if l_id not in saltdata.id2pred_cor:
                                                saltdata.add_pred_cor(l_id, fix)
                                            else:
                                                saltdata.id2pred_cor[l_id] = saltdata.id2pred_cor[l_id] + fix
                            # Right
                            if i+1 < mosaic.array.shape[0] and j-1 > 0:
                                if mosaic.array[i+1, j-1] is not None:
                                    if mosaic.array[i+1, j] is not None:
                                        r_id = mosaic.array[i, j-1]
                                        if (r_id in saltdata.id2pred\
                                                and np.count_nonzero(get_tile_mask(r_id)) < img_side_len**2/1000)\
                                                or r_id in saltdata.id2pred_cor:
                                            tile1 = get_tile_mask(mosaic.array[i,j])
                                            tile2 = get_tile_mask(mosaic.array[i+1, j-1])

                                            fix = find_fit_corners(tile2, tile1, False)
                                            if fix is not None:
                                                if r_id not in saltdata.id2pred_cor:
                                                    saltdata.add_pred_cor(r_id, fix)
                                                else:
                                                    saltdata.id2pred_cor[r_id] = saltdata.id2pred_cor[mosaic.array[i, j-1]] + fix

            for i in range(mosaic.array.shape[1]):
                strip = mosaic.array[:,i][:, np.newaxis]
                zero_under_salt(strip, term_level)

    # Fill holes in images
    for f_name in os.listdir(mean_prediction_dir):
        mask = cv2.imread(join(mean_prediction_dir, f_name),
                          cv2.IMREAD_GRAYSCALE)
        mask = fix_holes(mask)
        cv2.imwrite(join(postprocessed_prediction_dir, f_name), mask)

    # Postprocess mosaics
    saltdata = SaltData(mosaic_csv_path=MOSAIC_PATH,
                        pred_dir=postprocessed_prediction_dir)
    for mosaic_id in saltdata.mosaics.mosaic_id2mosaic:
        process_mosaic(saltdata.mosaics.mosaic_id2mosaic[mosaic_id])

    # Save corrected tiles
    for f_name in os.listdir(postprocessed_prediction_dir):
        f_id = filename_without_ext(f_name)
        if f_id in saltdata.id2pred_cor:
            cv2.imwrite(join(postprocessed_prediction_dir, f_name),
                        saltdata.id2pred_cor[f_id])


#!/usr/bin/python3

#Este programa desacumula os jsons, isto é: compara um arquivo com o subsequente e deleta qualquer timestamp menor do que o timestamp máximo encontrado no arquivo anterior, tanto para os dados de navegação quanto para o de telemetria. Deve-se especificar o endereço dos dados acumulados de entrada e o endereço dos dados desacumulados de saída. Deve-se também especificar uma data de início. No primeiro arquivo, começando com "0.", o programa irá deletar qualquer dado anterior à 00:00 da data especificada.

#Exemplo: python3 preprocess.py -i /home/felipeadachi/Dados/cumulating/cumulating2-13.03.19 -o /home/felipeadachi/Dados/non-cumulating/13.03.19 -d 2019,3,13

import json
import os
import collections
import datetime
import argparse
import numpy as np
from attributes import which_topast,past_to_get

input_1_filename = 'nonein.json'
output_filename = 'noneout.json'

parser = argparse.ArgumentParser(description='Preprocess Raw JSON files')

required = parser.add_argument_group('required named arguments')

required.add_argument('-i','--input', help='Location of local data to create database from', required=True)
required.add_argument('-o','--output', help='Output path', required=True)
required.add_argument('-d','--date', help='Date to decumulate', required=True)

args = vars(parser.parse_args())

date=args['date'].split(',')
folder_path_in = args['input']
folder_path_out = args['output']

which_todelta = [['deapth']]

#                                    Y   M  D
date_to_start=int(datetime.datetime(2019,1,19).strftime('%s'))*1000

date_to_start=int(datetime.datetime(int(date[0]),int(date[1]),int(date[2])).strftime('%s'))*1000

#---------------------#---------------------#---------------------#---------------------#---------------------#

def main():

    i=0
    number_of_files = len(os.listdir(folder_path_in))
    list_of_files = [None]*number_of_files
    print(number_of_files)
    for data_file in sorted(os.listdir(folder_path_in)):

        numfile=data_file.partition(".")[0]
        print(numfile)
        list_of_files[int(numfile)]=data_file
        print(numfile+">"+data_file)

    print(list_of_files)

    while i < number_of_files:
        json_current_content= read_json_content(folder_path_in + '/' + list_of_files[i])
        if (i != 0):
            json_preceding_content = read_json_content(folder_path_in + '/' + list_of_files[i-1])
            json_file_content = (json_preceding_content,json_current_content)
            print("max timestamps:")
            json_content_out = content_modifier(*json_file_content)
        else:
            json_content_out= check_start_date(json_current_content)

        json_content_out=motorsep(json_content_out)

        json_content_out=has_elapsed(json_content_out)

        json_content_out = delta(json_content_out)

        #json_content_out = past_commands(json_content_out,which_topast)

        #del json_content_out['0']
        if not os.path.exists(folder_path_out):
            os.makedirs(folder_path_out)
        write_json_content(folder_path_out + '/' + list_of_files[i],json_content_out)
        i += 1

        #json_file_content=read_json_content(folder_path_in+'/'+data_file)
        #del json_file_content['0']

        #write_json_content(folder_path_out+'/'+data_file,json_file_content)

        #json_file_content = None

        #json_file_content = read_json_content(folder_path_in+'/'+data_file)

        #if json_file_content is None:
            #exit(0)
        #print(data_file)
        #output_filename=folder_path_out+'/'+data_file
        #write_json_content(output_filename, json_file_content)

#---------------------#---------------------#---------------------#---------------------#---------------------#

def check_start_date(current_content):

    for i in range(0,2):

        data_list = current_content[str(i)]['data']

        l=0

        while l < len(data_list):

            data = data_list[l]
            cur_timestamp = data['timestamp']

            if cur_timestamp < date_to_start:
                #print("current timestamp:",cur_timestamp,"type:",type(cur_timestamp))
                #print("date_to_start:",date_to_start,"type:",type(date_to_start))

                del data_list[l]

            else:
                l=l+1

    return current_content

#---------------------#---------------------#---------------------#---------------------#---------------------#

def content_modifier(content_1, content_2):

    timestamps_nav = list()
    timestamps_telemetry = list()

    for data_nav in content_1['0']['data']:
        timestamps_nav.append(data_nav['timestamp'])

    max_timestamp_nav_preceding = max(timestamps_nav)
    print(max_timestamp_nav_preceding)
    data_list_nav = content_2['0']['data']

    l = 0

    while l < len(data_list_nav):

        data_nav = data_list_nav[l]
        cur_timestamp_nav = data_nav['timestamp']

        if cur_timestamp_nav <= max_timestamp_nav_preceding:
            del data_list_nav[l]

        else:
            l = l + 1

    for data_telemetry in content_1['1']['data']:
        timestamps_telemetry.append(data_telemetry['timestamp'])

    max_timestamp_telemetry_preceding = max(timestamps_telemetry)
    print(max_timestamp_telemetry_preceding)
    data_list_telemetry = content_2['1']['data']

    l = 0

    while l < len(data_list_telemetry):

        data_telemetry = data_list_telemetry[l]
        cur_timestamp_telemetry = data_telemetry['timestamp']

        if cur_timestamp_telemetry <= max_timestamp_telemetry_preceding:
            del data_list_telemetry[l]

        else:
            l = l + 1

    return content_2

#---------------------#---------------------#---------------------#---------------------#---------------------#

def motorsep(content_1):

    data_list = content_1['0']['data']

    i = 0
    while i < len(data_list):
        #print(i)
        data = data_list[i]
        nomotors=False
        nomtargs=False
        #try:
        #    motor1,motor2,motor3 = (data['motors'].split(","))

        #except KeyError:
        #    nomotors=True

        #if nomotors==False:
        #    data['motor1']=int(motor1)
        #    data['motor2']=int(motor2)
        #    data['motor3']=int(motor3)

        try:
            mtarg1,mtarg2,mtarg3 = data['mtarg'].split(",")
        except KeyError:
            print("key error mtarg:",data['timestamp'])
            nomtargs=True

        if nomtargs==False:
            data['mtarg1']=int(mtarg1)
            data['mtarg2']=int(mtarg2)
            data['mtarg3']=int(mtarg3)

        i += 1

    return content_1

#---------------------#---------------------#---------------------#---------------------#---------------------#

def read_json_content(filename):
    with open(filename, 'r') as f:
        return json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(f.read(os.stat(filename).st_size))

#---------------------#---------------------#---------------------#---------------------#---------------------#
def delta(content):

    data_list=content['0']['data']

    i=0

    while (i<len(data_list)):

        data=data_list[i]
        for attribute in which_todelta:
            if i==0 or data['has_elapsed']==1:
                data[attribute[0]+'_delta'] = np.nan
                try:
                    prev=data[attribute[0]]
                except KeyError:
                    print("key error:",attribute)
            else:
                data_prev=data_list[i-1]
                if(attribute==["yaw"]):
                    #print("aqui")
                    #Quando o rov passa de 340 para 10 graus por exemplo, o delta seria calculado como 330 graus, quando na verdade ele deslocou apenas 20 graus
                    if abs(float(data[attribute[0]])-float(data_prev[attribute[0]]))>360-abs(float(data[attribute[0]])-float(data_prev[attribute[0]])):
                        data[attribute[0]+'_delta'] = float("%.2f" % round(360-(abs(float(data[attribute[0]])-float(data_prev[attribute[0]]))),2))
                        #print("aqui:",data[attribute[0]],"timestamp:",data['timestamp'])

                    else:
                        data[attribute[0]+'_delta'] = float("%.2f" % round(float(data[attribute[0]])-float(data_prev[attribute[0]]),2))

                else:
                    try:
                        if data[attribute[0]]==data_prev[attribute[0]]:
                            data[attribute[0]+'_delta'] = data_prev[attribute[0]+'_delta']
                        else:
                            data[attribute[0]+'_delta'] = float("%.2f" % round((float(data[attribute[0]])-float(data_prev[attribute[0]])),2))

                    except KeyError:
                        print("key error:",attribute)
        i+=1

    return content

#---------------------#---------------------#---------------------#---------------------#---------------------#

def has_elapsed(content):

    data_list = content['0']['data']

    i=0

    while (i<len(data_list)):

        data=data_list[i]
        if i==0:
            data['has_elapsed']=1
        if i >= 1:
            if (data['timestamp']-data_list[i-1]['timestamp'])>=500:
                data['has_elapsed']=1
            else:
                data['has_elapsed']=0
        i+=1
    return content

#---------------------#---------------------#---------------------#---------------------#---------------------#

def past_commands(content,which_topast):

    data_list=content['0']['data']

    i=0

    while (i<len(data_list)):

        #get_token retorna o número de vezes permitido para pegar os comandos passados, com base nos flags "has_elapsed" de cada timestep
        token=get_token(i,data_list)
        #print("token for timestep:",i,"is:",token)
        data=data_list[i]
        tkn={}

        for att in which_topast:
                    tkn[att[0]]=token

        for past in past_to_get:

            if i<past:
                for att in which_topast:
                    data[att[0]+'_t'+str(past)]=np.nan

            else:

                for att in which_topast:

                    try:
                        if tkn[att[0]]<=0:
                            data[att[0]+'_t'+str(past)]=np.nan
                        else:
                            data[att[0]+'_t'+str(past)]=data_list[i-past][att[0]]
                            tkn[att[0]]-=1
                            #print("token for:",motor,"in tstep:",i,"is:",tkn[motor[0]])
                    except KeyError:
                        data[att[0]+'_t'+str(past)]=np.nan
                        tkn[att[0]]-=1
        i+=1

    return content

def get_token(i,data_list):
    j=0
    token=0
    while j<max(past_to_get):
        if data_list[i-j]['has_elapsed'] == 0:
            token+=1
        else:
            return token

        j+=1
    return token

#---------------------#---------------------#---------------------#---------------------#---------------------#

def write_json_content(filename, content):
    with open(filename, 'w') as f:
        f.write(json.dumps(content,indent=4))

#---------------------#---------------------#---------------------#---------------------#---------------------#

if(__name__ == '__main__'):
    main();


from pprint import pprint

from twisted.web.server import Site
from twisted.web import server
from twisted.web.resource import Resource
from twisted.internet import reactor, defer, threads
from twisted.web.static import File
from string import Template

import cPickle
import time
import os
import socket

def wait(seconds, result=None):
    d = defer.Deferred()
    reactor.callLater(seconds, d.callback, result)
    return d

class Load(Resource):

    def final(self, message, request):
        request.finish()

    #@defer.inlineCallbacks

    def crap(self, request, name, num):
        x = ''
        pprint(str(num))
        for n in range(num):
            with open('c:\\' + name + '_' + str(n) + '.vbo', 'rb') as f:
                x = cPickle.load(f)
                request.write(x)
                pprint("sending cloud #" + str(n))
                #yield wait(0.0)

    def render_GET(self, request):
        isLeaf = True
        cloudName = request.received_headers['cloudName']

        # check that file cloudName exists
        # nyi

        # find out how many vbo's there are
        with open('c:\\' + cloudName) as f:
            numVBO = int(f.readline())

        if numVBO > 0:
            # extract filename
            fileName = cloudName[0:cloudName.rfind('.')]
            d = threads.deferToThread(self.crap, request, fileName, numVBO)
            d.addCallback(lambda _: request.finish())

            return server.NOT_DONE_YET

        #x += cPickle.load(f)
        #print(type(x))

        #equest.finish()
        #return server.NOT_DONE_YET
        #return x

        #return -1;

class UploadVBO(Resource):
    def render_GET(self, request):
        return ''

    def render_POST(self, request):
        cloudName = request.received_headers['x-cloudName']
        cloudSeq = request.received_headers['x-cloudSequence']
        cloudArrayByteLength = request.received_headers['X-cloudArrayByteLength']
        cloudNumPoints = request.received_headers['X-cloudNumPoints']
        pprint(cloudName + ' ' + cloudSeq + ' ' + cloudArrayByteLength + ' ' + cloudNumPoints)

        #pprint(request.__dict__)
        #pprint(request.content.read())

        with open('c:\\' + cloudName + '_' + cloudSeq + '.vbo', 'wb') as f:
            d = request.content.read()
            cPickle.dump(d, f, cPickle.HIGHEST_PROTOCOL)

        return ''

## this completes the upload process by writing .pointcloud file
## right now there's on TotalSequence but more will come
class CreatePointCloudFile(Resource):
    def render_GET(self, request):
        return ''

    def render_POST(self, request):
        cloudName = request.received_headers['x-cloudName']
        cloudTotalSeq = request.received_headers['X-cloudTotalSequence']
        pprint("writing " + cloudName + ".pointcloud, # of VBOs = " + cloudTotalSeq)

        # write .pointcloud file
        with open('c:\\' + cloudName + '.pointcloud', 'w') as f:
            f.write(str(cloudTotalSeq))

        return 1

class QueryPickPoint(Resource):
    def render_GET(self, request):
       isLeaf = True
       RayOriginX = request.received_headers['rayoriginx']
       RayOriginY = request.received_headers['rayoriginy']
       RayOriginZ = request.received_headers['rayoriginz']
       RayDirX = request.received_headers['raydirx']
       RayDirY = request.received_headers['raydiry']
       RayDirZ = request.received_headers['raydirz']

       HOST = '127.0.0.1'
       PORT = 9000
       s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
       s.connect((HOST, PORT))
       #s.sendall('1,0.849,-2.379,0.445,0.019,0.569,-0.822')

       strQuery = '1,%s,%s,%s,%s,%s,%s' % (RayOriginX,RayOriginY,RayOriginZ,RayDirX,RayDirY,RayDirZ)
       pprint(strQuery)
       s.sendall(strQuery)

       data = s.recv(1024)
       s.close()

       responseData = repr(data).replace('\'','') # need to get rid of single quotes returned from octree backend
       pprint(responseData)
       request.write(responseData)

       request.finish()
       return server.NOT_DONE_YET

class PointCloudListing(Resource):
    isLeaf = True

    def getChild(self, name, request):
        if name == '':
            return self
        return resource.Resource.geChild(self, name, request)

    def render_GET(self, request):
        allFiles = os.listdir("c:\\")
        cloudFiles = [f for f in allFiles if f.find(".pointcloud") > 0]

        s = []
        for f in cloudFiles:
            s.append("" + f + "
")

        template = open("index.template", "r")
        data = template.read().replace("<--pointclouds-->", '\n'.join(s))

        return data

class PointCloud(Resource):
    isLeaf = True

    def render_GET(self, request):
        template = open("view.html", "r")
        cloudName = request.postpath[0]

        t = Template(template.read())
        html = ''

        if cloudName == '':
            html = t.substitute(entry='return;')
        else:
            html = t.substitute(entry='startServer("' + cloudName + '");')

        return html

#root = Resource()
root = File("C:\\Users\\mer\\Documents\\GitHub\\bobcat")

root.putChild('upload', UploadVBO())
root.putChild('load', Load())
root.putChild('findpickpoint', QueryPickPoint())
root.putChild('finalize', CreatePointCloudFile())
root.putChild('contentlist', PointCloudListing())
root.putChild('pointcloud', PointCloud())

factory = Site(root)
reactor.listenTCP(8080, factory)
reactor.run()


# Question: Given a matrix (2x2 array), traverse it in a spiral
# # Solution: Track the offset of the inner level of the spiral we're at, keep track of the directions
# Difficulty: Medium

def spiralOrder(matrix):

    # Initially our direction needs to be right and our offset 0
    direction = "R"
    i = j = offset = 0
    result = []
    if not matrix: return matrix
    m, n = len(matrix) - 1, len(matrix[0]) - 1

    # Loop through the number of values in the matrix
    for _ in range(len(matrix) * len(matrix[0])):
        result += [matrix[i][j]]

        # If the direction is right keep incrementing j (the horizantal index)
        # once j reaches the end (the length of the horizantal - the offset), switch directions
        # accordingly; similar approaches for when traversing down and left
        if direction == "R":
            if j == n - offset: direction = "D"
            else: j += 1
        if direction == "D":
            if i == m - offset: direction = "L"
            else: i += 1
        if direction == "L":
            if j == offset: direction = "U"
            else: j -= 1

        # If we're traversing upwards we need to increment the offset once i is 1 more than the offset,
        # this is because if i == offset + 1 then i is right below the offset and we want to start traversing
        # an inner spiral of the matrix. We also increment j in this case and indicate the next move is to the right
        if direction == 'U':
            if i == offset + 1:
                direction = "R"
                offset += 1
                j += 1
            else: i -= 1

    return result


#!/usr/bin/env python
# coding: utf-8

# # GPU Light gradient boosting trained on TF-IDF reduced 50 dimensions
#
# 1. Same emotion dataset from [NLP-dataset](https://github.com/huseinzol05/NLP-Dataset)
# 2. Same splitting 80% training, 20% testing, may vary depends on randomness
# 3. Same regex substitution '[^\"\'A-Za-z0-9 ]+'
#
# ## Example
#
# Based on Term-frequency Inverse document frequency
#
# After that we apply SVD to reduce the dimensions, n_components = 50

# In[8]:

import re
import time

import numpy as np
import sklearn.datasets
from sklearn import metrics, pipeline
from sklearn.cross_validation import train_test_split
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.model_selection import StratifiedKFold

import lightgbm as lgb

# In[2]:

def clearstring(string):
    string = re.sub("[^\"'A-Za-z0-9 ]+", "", string)
    string = string.split(" ")
    string = filter(None, string)
    string = [y.strip() for y in string]
    string = " ".join(string)
    return string

# because of sklean.datasets read a document as a single element
# so we want to split based on new line
def separate_dataset(trainset):
    datastring = []
    datatarget = []
    for i in range(len(trainset.data)):
        data_ = trainset.data[i].split("\n")
        # python3, if python2, just remove list()
        data_ = list(filter(None, data_))
        for n in range(len(data_)):
            data_[n] = clearstring(data_[n])
        datastring += data_
        for n in range(len(data_)):
            datatarget.append(trainset.target[i])
    return datastring, datatarget

# In[3]:

trainset_data = sklearn.datasets.load_files(container_path="data", encoding="UTF-8")
trainset_data.data, trainset_data.target = separate_dataset(trainset_data)

# In[4]:

train_X, test_X, train_Y, test_Y = train_test_split(
    trainset_data.data, trainset_data.target, test_size=0.2
)

# In[5]:

decompose = pipeline.Pipeline(
    [("count", TfidfVectorizer()), ("svd", TruncatedSVD(n_components=50))]
).fit(trainset_data.data)

# In[6]:

params_lgb = {
    "max_depth": 27,
    "learning_rate": 0.03,
    "verbose": 50,
    "early_stopping_round": 200,
    "metric": "multi_logloss",
    "objective": "multiclass",
    "num_classes": len(trainset_data.target_names),
    "device": "gpu",
    "gpu_platform_id": 0,
    "gpu_device_id": 0,
}

# In[10]:

train_X = decompose.transform(train_X)
test_X = decompose.transform(test_X)

# In[11]:

d_train = lgb.Dataset(train_X, train_Y)
d_valid = lgb.Dataset(test_X, test_Y)
watchlist = [d_train, d_valid]
t = time.time()
clf = lgb.train(params_lgb, d_train, 100000, watchlist, early_stopping_rounds=200, verbose_eval=100)
print(round(time.time() - t, 3), "Seconds to train lgb")

# In[12]:

print(
    metrics.classification_report(
        test_Y, np.argmax(clf.predict(test_X), axis=1), target_names=trainset_data.target_names
    )
)

# In[13]:

clf.save_model("lgb-tfidf-svd50.model")

# In[ ]:


"""
Remove RasterAggregatedLayer and Related objects given RasterAggregatedLayer ids.
Note: Use the 'list_raster_layers' command to obtain the RasterAggregatedLayer ids.
"""
from django.core.management.base import BaseCommand
from django.conf import settings
from ...models import RasterAggregatedLayer

WGS84_SRID = settings.WGS84_SRID

class Command(BaseCommand):
    help = __doc__

    def add_arguments(self, parser):
        parser.add_argument("-i", "--ids",
                            type=int,
                            default=None,
                            required=True,
                            nargs="+",
                            help="RasterAggregatedLayer ids to delete.")

    def handle(self, *args, **options):
        for layer in RasterAggregatedLayer.objects.order_by("id"):
            self.stdout.write("Removing RasterAggregatedLayer: [{}] {}...".format(layer.id, layer.name))
            # Delete related DataModel Objects
            layer.pixels().delete()

            # delete Layer
            layer.delete()

            self.stdout.write("Done!")


import numpy as np
from utils import *

class Prediction:

    def __init__(self, objective, name=None, is_NN=False, class_weight=None):
        self.objective = objective
        self.name = name
        self.is_NN = is_NN
        self.class_weight = class_weight

    def fit(self, model, x, y=None, print_result=False, thr_1 = 0.5):

        if self.is_NN:
            # validation for neural network models :
            if 'pandas' in str(type(x)):
                self.prediction = model.predict(x.values)
            else:
                self.prediction = model.predict(x)
        else:
            # validation for sklearn models, catboost, xgboost and lightgbm :
            if 'binary_proba' in self.objective:
                if 'pandas' in str(type(x)):
                    self.prediction = model.predict_proba(x.values)[:, 1]
                else:
                    self.prediction = model.predict_proba(x)[:, 1]
            else:
                if 'pandas' in str(type(x)):
                    self.prediction = model.predict(x.values)
                else:
                    self.prediction = model.predict(x)

        if y is not None:
            # calculate metrics if y_true is provided
            if self.objective in ['binary', 'text_binary'] and self.is_NN:
                self.prediction = np.argmax(self.prediction, axis=1).reshape(-1)
            else:
                self.prediction = self.prediction.reshape(-1)
            self.acc_test, self.f1_test, self.recall_test, self.pre_test, self.roc_auc_test = calcul_metric_binary(y,
                                                                                self.prediction,
                                                                                print_result, thr_1)

    def get_prediction(self):
        return self.prediction

    def get_scores(self):
        return self.acc_test, self.f1_test, self.recall_test, self.pre_test, self.roc_auc_test


class Dish:
    def __init__(self,name,price,decription,type):
        self.name=name
        self.price=price
        self.decription=decription
        self.type=type
        self.quantity=0

    def showDish(self):
        print("Name:{}".format(self.name))
        print("price:{}||Decription:{}".format(self.price,self.decription))
        print("Type:{}".format(self.type))
        print("===========================")

dish1=Dish("Dal Makhhni",100,"Launch","Indian Veg")
dish2=Dish("chicken masala",450,"launch","Indian Non-veg")
dish3=Dish("Butter Checken",500,"launch","Indian Non-veg")
dish4=Dish("Butter Panner",350,"launch","Indian Veg")

dishes=[dish1,dish2,dish3,dish4]
# dish1.showDish()
# dish2.showDish()
# dish3.showDish()
# dish4.showDish()

cart=[]
total_dish=0

def addtocart(dish):
    total_dish=total_dish + quantity